ECS service discovery hell

eni-analysis.py

import pandas as pd

def read_eni_cap(filename):
    return pd.read_csv(filename,header=None, sep='\t', names=['name', 'old', 'new']).set_index('name')

eni_cap = read_eni_cap('instance-types.txt')

def read_enis(filename):
    enis = pd.read_json(filename, orient='records')
    return enis.join(enis.Attachment.apply(pd.Series), rsuffix='.attach').set_index('NetworkInterfaceId')

def read_ecs(filename, eni_cap_dict):
    ecs = pd.read_json(filename, orient='records', lines=True)
    instance = ecs.attributes.apply(lambda t: pd.Series({x['name']: x.get('value') for x in t}))['ecs.instance-type']
    ecs = ecs.join(instance).set_index('ec2InstanceId')
    ecs['ecs.instance-type.num_enis'] = ecs['ecs.instance-type'].apply(eni_cap_dict.get)
    return ecs

def get_used_enis(ecs, enis):
    eni_instances = enis[enis['Status.attach'] == 'attached']['InstanceId'].value_counts().rename('eni_count')
    return ecs.join(eni_instances)

def utilization(ecs_eni, ecs):
    return ecs_eni.eni_count.sum()/ecs['ecs.instance-type.num_enis'].sum()

instance-types.txt

a1.medium	2	4	4
a1.large	3	10	10
a1.xlarge	4	15	15
a1.2xlarge	4	15	15
a1.4xlarge	8	30	30
a1.metal	8	30	30
c1.medium	2	6	IPv6 not supported
c1.xlarge	4	15	IPv6 not supported
c3.large	3	10	10
c3.xlarge	4	15	15
c3.2xlarge	4	15	15
c3.4xlarge	8	30	30
c3.8xlarge	8	30	30
c4.large	3	10	10
c4.xlarge	4	15	15
c4.2xlarge	4	15	15
c4.4xlarge	8	30	30
c4.8xlarge	8	30	30
c5.large	3	10	10
c5.xlarge	4	15	15
c5.2xlarge	4	15	15
c5.4xlarge	8	30	30
c5.9xlarge	8	30	30
c5.12xlarge	8	30	30
c5.18xlarge	15	50	50
c5.24xlarge	15	50	50
c5.metal	15	50	50
c5d.large	3	10	10
c5d.xlarge	4	15	15
c5d.2xlarge	4	15	15
c5d.4xlarge	8	30	30
c5d.9xlarge	8	30	30
c5d.12xlarge	8	30	30
c5d.18xlarge	15	50	50
c5d.24xlarge	15	50	50
c5d.metal	15	50	50
c5n.large	3	10	10
c5n.xlarge	4	15	15
c5n.2xlarge	4	15	15
c5n.4xlarge	8	30	30
c5n.9xlarge	8	30	30
c5n.18xlarge	15	50	50
c5n.metal	15	50	50
cc2.8xlarge	8	30	IPv6 not supported
cr1.8xlarge	8	30	IPv6 not supported
d2.xlarge	4	15	15
d2.2xlarge	4	15	15
d2.4xlarge	8	30	30
d2.8xlarge	8	30	30
f1.2xlarge	4	15	15
f1.4xlarge	8	30	30
f1.16xlarge	8	50	50
g2.2xlarge	4	15	IPv6 not supported
g2.8xlarge	8	30	IPv6 not supported
g3s.xlarge	4	15	15
g3.4xlarge	8	30	30
g3.8xlarge	8	30	30
g3.16xlarge	15	50	50
g4dn.xlarge	3	10	10
g4dn.2xlarge	3	10	10
g4dn.4xlarge	3	10	10
g4dn.8xlarge	4	15	15
g4dn.12xlarge	8	30	30
g4dn.16xlarge	15	50	50
h1.2xlarge	4	15	15
h1.4xlarge	8	30	30
h1.8xlarge	8	30	30
h1.16xlarge	15	50	50
hs1.8xlarge	8	30	IPv6 not supported
i2.xlarge	4	15	15
i2.2xlarge	4	15	15
i2.4xlarge	8	30	30
i2.8xlarge	8	30	30
i3.large	3	10	10
i3.xlarge	4	15	15
i3.2xlarge	4	15	15
i3.4xlarge	8	30	30
i3.8xlarge	8	30	30
i3.16xlarge	15	50	50
i3.metal	15	50	50
i3en.large	3	10	10
i3en.xlarge	4	15	15
i3en.2xlarge	4	15	15
i3en.3xlarge	4	15	15
i3en.6xlarge	8	30	30
i3en.12xlarge	8	30	30
i3en.24xlarge	15	50	50
i3en.metal	15	50	50
inf1.xlarge	4	10	10
inf1.2xlarge	4	10	10
inf1.6xlarge	8	30	30
inf1.24xlarge	15	50	50
m1.small	2	4	IPv6 not supported
m1.medium	2	6	IPv6 not supported
m1.large	3	10	IPv6 not supported
m1.xlarge	4	15	IPv6 not supported
m2.xlarge	4	15	IPv6 not supported
m2.2xlarge	4	30	IPv6 not supported
m2.4xlarge	8	30	IPv6 not supported
m3.medium	2	6	IPv6 not supported
m3.large	3	10	IPv6 not supported
m3.xlarge	4	15	IPv6 not supported
m3.2xlarge	4	30	IPv6 not supported
m4.large	2	10	10
m4.xlarge	4	15	15
m4.2xlarge	4	15	15
m4.4xlarge	8	30	30
m4.10xlarge	8	30	30
m4.16xlarge	8	30	30
m5.large	3	10	10
m5.xlarge	4	15	15
m5.2xlarge	4	15	15
m5.4xlarge	8	30	30
m5.8xlarge	8	30	30
m5.12xlarge	8	30	30
m5.16xlarge	15	50	50
m5.24xlarge	15	50	50
m5.metal	15	50	50
m5a.large	3	10	10
m5a.xlarge	4	15	15
m5a.2xlarge	4	15	15
m5a.4xlarge	8	30	30
m5a.8xlarge	8	30	30
m5a.12xlarge	8	30	30
m5a.16xlarge	15	50	50
m5a.24xlarge	15	50	50
m5ad.large	3	10	10
m5ad.xlarge	4	15	15
m5ad.2xlarge	4	15	15
m5ad.4xlarge	8	30	30
m5ad.8xlarge	8	30	30
m5ad.12xlarge	8	30	30
m5ad.16xlarge	15	50	50
m5ad.24xlarge	15	50	50
m5d.large	3	10	10
m5d.xlarge	4	15	15
m5d.2xlarge	4	15	15
m5d.4xlarge	8	30	30
m5d.8xlarge	8	30	30
m5d.12xlarge	8	30	30
m5d.16xlarge	15	50	50
m5d.24xlarge	15	50	50
m5d.metal	15	50	50
m5dn.large	3	10	10
m5dn.xlarge	4	15	15
m5dn.2xlarge	4	15	15
m5dn.4xlarge	8	30	30
m5dn.8xlarge	8	30	30
m5dn.12xlarge	8	30	30
m5dn.16xlarge	15	50	50
m5dn.24xlarge	15	50	50
m5n.large	3	10	10
m5n.xlarge	4	15	15
m5n.2xlarge	4	15	15
m5n.4xlarge	8	30	30
m5n.8xlarge	8	30	30
m5n.12xlarge	8	30	30
m5n.16xlarge	15	50	50
m5n.24xlarge	15	50	50
p2.xlarge	4	15	15
p2.8xlarge	8	30	30
p2.16xlarge	8	30	30
p3.2xlarge	4	15	15
p3.8xlarge	8	30	30
p3.16xlarge	8	30	30
p3dn.24xlarge	15	50	50
r3.large	3	10	10
r3.xlarge	4	15	15
r3.2xlarge	4	15	15
r3.4xlarge	8	30	30
r3.8xlarge	8	30	30
r4.large	3	10	10
r4.xlarge	4	15	15
r4.2xlarge	4	15	15
r4.4xlarge	8	30	30
r4.8xlarge	8	30	30
r4.16xlarge	15	50	50
r5.large	3	10	10
r5.xlarge	4	15	15
r5.2xlarge	4	15	15
r5.4xlarge	8	30	30
r5.8xlarge	8	30	30
r5.12xlarge	8	30	30
r5.16xlarge	15	50	50
r5.24xlarge	15	50	50
r5.metal	15	50	50
r5a.large	3	10	10
r5a.xlarge	4	15	15
r5a.2xlarge	4	15	15
r5a.4xlarge	8	30	30
r5a.8xlarge	8	30	30
r5a.12xlarge	8	30	30
r5a.16xlarge	15	50	50
r5a.24xlarge	15	50	50
r5ad.large	3	10	10
r5ad.xlarge	4	15	15
r5ad.2xlarge	4	15	15
r5ad.4xlarge	8	30	30
r5ad.8xlarge	8	30	30
r5ad.12xlarge	8	30	30
r5ad.16xlarge	15	50	50
r5ad.24xlarge	15	50	50
r5d.large	3	10	10
r5d.xlarge	4	15	15
r5d.2xlarge	4	15	15
r5d.4xlarge	8	30	30
r5d.8xlarge	8	30	30
r5d.12xlarge	8	30	30
r5d.16xlarge	15	50	50
r5d.24xlarge	15	50	50
r5d.metal	15	50	50
r5dn.large	3	10	10
r5dn.xlarge	4	15	15
r5dn.2xlarge	4	15	15
r5dn.4xlarge	8	30	30
r5dn.8xlarge	8	30	30
r5dn.12xlarge	8	30	30
r5dn.16xlarge	15	50	50
r5dn.24xlarge	15	50	50
r5n.large	3	10	10
r5n.xlarge	4	15	15
r5n.2xlarge	4	15	15
r5n.4xlarge	8	30	30
r5n.8xlarge	8	30	30
r5n.12xlarge	8	30	30
r5n.16xlarge	15	50	50
r5n.24xlarge	15	50	50
t1.micro	2	2	IPv6 not supported
t2.nano	2	2	2
t2.micro	2	2	2
t2.small	3	4	4
t2.medium	3	6	6
t2.large	3	12	12
t2.xlarge	3	15	15
t2.2xlarge	3	15	15
t3.nano	2	2	2
t3.micro	2	2	2
t3.small	3	4	4
t3.medium	3	6	6
t3.large	3	12	12
t3.xlarge	4	15	15
t3.2xlarge	4	15	15
t3a.nano	2	2	2
t3a.micro	2	2	2
t3a.small	2	4	4
t3a.medium	3	6	6
t3a.large	3	12	12
t3a.xlarge	4	15	15
t3a.2xlarge	4	15	15
u-6tb1.metal	5	30	30
u-9tb1.metal	5	30	30
u-12tb1.metal	5	30	30
u-18tb1.metal	15	50	50
u-24tb1.metal	15	50	50
x1.16xlarge	8	30	30
x1.32xlarge	8	30	30
x1e.xlarge	3	10	10
x1e.2xlarge	4	15	15
x1e.4xlarge	4	15	15
x1e.8xlarge	4	15	15
x1e.16xlarge	8	30	30
x1e.32xlarge	8	30	30
z1d.large	3	10	10
z1d.xlarge	4	15	15
z1d.2xlarge	4	15	15
z1d.3xlarge	8	30	30
z1d.6xlarge	8	30	30
z1d.12xlarge	15	50	50
z1d.metal	15	50	50

list-ecs-instances.sh

#! /usr/bin/env bash

get_clusters() {
    region="$1"
    >&2 echo "get cluster region=$region cluster=$cluster"
    aws ecs list-clusters --region "$region" | jq -c .clusterArns | jq -r 'join(" ")'
}

list_instances() {
    region="$1"
    cluster="$2"
    >&2 echo "list instances region=$region cluster=$cluster"
    instances=$(aws ecs list-container-instances \
                    --region "$region" \
                    --cluster "$cluster" | jq .containerInstanceArns)
    instances=$(echo $instances | jq -r 'join(" ")')
    echo "$instances"
}

describe_instances() {
    region="$1"
    cluster="$2"
    instance="$3"
    dest="regions/$region/ecs/$(basename $cluster)-$(basename $instance).json"
    >&2 echo "describe instances region=$region cluster=$cluster dest=$dest"
    aws ecs describe-container-instances \
        --region "$region" \
        --cluster "$cluster" \
        --container-instances $(echo $instance)  | jq -c .containerInstances[] > "$dest"
}

fetch_instance_data() {
    region="$1"
    >&2 echo "------------------------"
    >&2 echo "fetch instance data $region"
    rm -rf regions/"$region/ecs/" || 0
    mkdir -p "regions/$region/ecs"
    clusters=$(get_clusters "$region") || echo "empty"
    for cluster in $(echo "$clusters")
    do
        instances=$(list_instances "$region" "$cluster")
        IFS=' ' read -r -a array <<< "$instances"
        i=0
        for instance in "${array[@]}"
        do
            describe_instances "$region" "$cluster" "$instance" &
            ((i++))
            if [ $(($i % 10)) -eq 0 ]; then
                >&2 echo "Sleeping!"
                sleep 2
            fi
        done
        cat $(echo regions/$region/ecs/$(basename $cluster)*.json) > regions/$region/ecs/$(basename $cluster).json
    done

}

fetch_instance_data "eu-west-2"
fetch_instance_data "eu-west-1"
fetch_instance_data "ap-northeast-2"
fetch_instance_data "ap-northeast-1"
fetch_instance_data "ap-southeast-1"
fetch_instance_data "ap-southeast-2"
fetch_instance_data "us-east-1"
fetch_instance_data "us-east-2"
fetch_instance_data "us-west-1"
fetch_instance_data "us-west-2"

list-enis.sh

#! /usr/bin/env bash

list_enis() {
    region="$1"
    mkdir -p "regions/$region"
    echo "Fetching ENIs region=$region"
    aws ec2 describe-network-interfaces \
        --region "$region" | jq .NetworkInterfaces > "regions/$region/enis.json"
    sleep 1
}

list_enis "eu-west-2"
list_enis "eu-west-1"
list_enis "ap-northeast-2"
list_enis "ap-northeast-1"
list_enis "ap-southeast-1"
list_enis "ap-southeast-2"
list_enis "us-east-1"
list_enis "us-east-2"
list_enis "us-west-1"
list_enis "us-west-2"

results.py

root = '**'

eni_ap_southeast_1 = read_enis(root+'regions/ap-southeast-1/enis.json')
ecs_ap_southeast_1 = read_ecs(root+'regions/ap-southeast-1/ecs/production1.json', eni_cap.old.to_dict())

eni_eu_west_1 = read_enis(root+'regions/eu-west-1/enis.json')
ecs_eu_west_1 = read_ecs(root+'regions/eu-west-1/ecs/production1.json', eni_cap.old.to_dict())

eni_us_east_1 = read_enis(root+'regions/us-east-1/enis.json')
ecs_us_east_1 = read_ecs(root+'regions/us-east-1/ecs/production1.json', eni_cap.old.to_dict())

eni_us_east_2 = read_enis(root+'regions/us-east-2/enis.json')
ecs_us_east_2 = read_ecs(root+'regions/us-east-2/ecs/production1.json', eni_cap.old.to_dict())

eni_us_west_2 = read_enis(root+'regions/us-west-2/enis.json')
ecs_us_west_2 = read_ecs(root+'regions/us-west-2/ecs/production1.json', eni_cap.old.to_dict())

ecs_us_east_1_staging = read_ecs(root+'regions/us-east-1/ecs/staging1.json', eni_cap.old.to_dict())

print(utilization(get_used_enis(ecs_ap_southeast_1, eni_ap_southeast_1), ecs_ap_southeast_1))
print('eu-west-1', utilization(get_used_enis(ecs_eu_west_1, eni_eu_west_1), ecs_eu_west_1))
print('us-east-1', utilization(get_used_enis(ecs_us_east_1, eni_us_east_1), ecs_us_east_1))
print('us-east-2', utilization(get_used_enis(ecs_us_east_2, eni_us_east_2), ecs_us_east_2))
print('us-west-2', utilization(get_used_enis(ecs_us_west_2, eni_us_west_2), ecs_us_west_2))