gcp-aws-vpn.create.py

gcp-aws-vpn.create.py

#!/usr/bin/env python3
# Stand up a VPN between GCP & AWS
# Assumes you're CLI auth'd to both as default
# Based on https://cloud.google.com/architecture/build-ha-vpn-connections-google-cloud-aws
# Yes it's ugly AF but basically working!
# Usage: ./setup-vpn.py --shared-secret-0=xxxxxx --shared-secret-1=aaaaa --shared-secret-2=bbbb --shared-secret-3=cccc
# You'll need to to pip[env] install beautifulsoup4 click boto3 lxml

import subprocess
import json
from bs4 import BeautifulSoup
import os
import boto3
import botocore.exceptions
import click
import time
import concurrent.futures
from pprint import pprint


THREADPOOL_MAX_WORKERS = 20

DELETE_GCP_GATEWAY = False
DELETE_GCP_TUNNELS = False
CREATE_GCP_GATEWAY = True
CREATE_GCP_TUNNELS = True
CREATE_AWS = True

GCP_PROJECT_ID = "muy-proj-id"

GCP_NETWORK = "default"

GCP_REGION = "australia-southeast1"
GCP_SIDE_INFRA_NAME = "aws-xxx-dev"
GCP_SIDE_ASN = 65010

GCP_SIDE_INFRA_NAME = "aws-beem-dev"
GCP_SIDE_TUNNEL_NAMES = f"{GCP_SIDE_INFRA_NAME}-{GCP_NETWORK}-tunnel"
GCP_PEER_GATEWAY_NAME = f"{GCP_SIDE_INFRA_NAME}-{GCP_NETWORK}-peer-gw"
GCP_CLOUD_ROUTER_NAME = f"{GCP_SIDE_INFRA_NAME}-{GCP_NETWORK}-router"
GCP_CLOUD_ROUTER_INTERFACE_NAMES = f"{GCP_SIDE_INFRA_NAME}-{GCP_NETWORK}-int"
AWS_SIDE_ASN = 65011

# Used for mocks/default network
GCP_VPN_GW_NAME = f"{GCP_SIDE_INFRA_NAME}-{GCP_NETWORK}-vpn"

AWS_SIDE_VGW_NAME = f"gcp-beem-{GCP_PROJECT_ID}-dev-app-network"      # hardcode to reuse existing
AWS_SIDE_CGW_NAME = f"gcp-beem-{GCP_PROJECT_ID}-dev-app-network"    # hardcode to reuse existing
AWS_SIDE_VPN_CONNECTION_NAME = f"gcp-beem-{GCP_PROJECT_ID}-{GCP_NETWORK}"

IKE_VERSION = "2"

AWS_VPC_ID = "vpc-xxxxxxxxxxxx"

# Used for mocks/default network
AWS_TUNNEL_INSIDE_CIDRS = [
    "169.254.16.0/30",
    "169.254.17.0/30",
    "169.254.18.0/30",
    "169.254.19.0/30",
]

# gcloud config set project $GCP_PROJECT_ID
# export PROJECT_ID=`gcloud config list --format="value(core.project)"`
PROJECT_ID = "xxxxxxxx-yyyyyy"


def get_google_managed_services_cidr():
    cmd = [
            "gcloud",
            "compute",
            "addresses",
            "list",
            "--filter",
            "name ~ google-managed-services",
            "--format=json",
    ]
    res = json.loads(subprocess.run(cmd, capture_output=True).stdout)[0]
    cidr = f"{res['address']}/{res['prefixLength']}"
    return cidr


def create_gcp_tunnel(
        external_vpn_gateway_interface_id, internal_vpn_gateway_interface_id, aws_peer_inside_address, tunnel_inside_address, shared_secret, google_managed_services_cidr
):
    # Create 4 VPN Tunnels
    tunnel_name = f"{GCP_SIDE_TUNNEL_NAMES}-{external_vpn_gateway_interface_id}"

    cmd = [
        "gcloud",
        "compute",
        "vpn-tunnels",
        "create",
        f"{tunnel_name}",
        "--peer-external-gateway",
        GCP_PEER_GATEWAY_NAME,
        "--peer-external-gateway-interface",
        str(external_vpn_gateway_interface_id),
        "--region",
        GCP_REGION,
        "--ike-version",
        IKE_VERSION,
        "--shared-secret",
        shared_secret,
        "--router",
        GCP_CLOUD_ROUTER_NAME,
        "--vpn-gateway",
        GCP_VPN_GW_NAME,
        "--interface",
        str(internal_vpn_gateway_interface_id),
    ]
    print(' '.join(cmd))
    err = subprocess.run(cmd, capture_output=True)
    print(err)

    cmd = [
        "gcloud",
        "compute",
        "routers",
        "add-interface",
        GCP_CLOUD_ROUTER_NAME,
        "--interface-name",
        f"{GCP_CLOUD_ROUTER_INTERFACE_NAMES}-{external_vpn_gateway_interface_id}",
        "--vpn-tunnel",
        tunnel_name,
        "--ip-address",
        str(tunnel_inside_address),
        "--mask-length",
        "30",
        "--region",
        GCP_REGION,
    ]
    print(' '.join(cmd))
    err = subprocess.run(cmd, capture_output=True)
    print(err)

    # Add BGP peers for each tunnel
    cmd = [
        "gcloud",
        "compute",
        "routers",
        "add-bgp-peer",
        GCP_CLOUD_ROUTER_NAME,
        "--peer-name",
        f"{GCP_SIDE_TUNNEL_NAMES}-conn{internal_vpn_gateway_interface_id}-tunn{external_vpn_gateway_interface_id}",
        "--peer-asn",
        str(AWS_SIDE_ASN),
        "--interface",
        f"{GCP_CLOUD_ROUTER_INTERFACE_NAMES}-{external_vpn_gateway_interface_id}",
        "--peer-ip-address",
        aws_peer_inside_address,
        "--region",
        GCP_REGION,
        "--advertisement-mode",
        "custom",
        "--set-advertisement-groups",
        "all_subnets",
        "--set-advertisement-ranges",
        f"{google_managed_services_cidr}=google-managed-services"
    ]
    print(' '.join(cmd))
    err = subprocess.run(cmd, capture_output=True).stderr
    if err:
        try:
            while(err.index(b'is not ready')):
                print('Google Cloud Router not yet ready, sleeping ...')
                time.sleep(2)
                err = subprocess.run(cmd, capture_output=True).stderr
                print(' '.join(cmd))
        except ValueError:
            if(b'' == err):
                print('... added BGP peer')
            elif(err.index(b'Duplicate BGP peer name')):
                cmd = [
                    "gcloud",
                    "compute",
                    "routers",
                    "update-bgp-peer",
                    GCP_CLOUD_ROUTER_NAME,
                    "--peer-name",
                    f"{GCP_SIDE_TUNNEL_NAMES}-conn{internal_vpn_gateway_interface_id}-tunn{external_vpn_gateway_interface_id}",
                    "--peer-asn",
                    str(AWS_SIDE_ASN),
                    "--interface",
                    f"{GCP_CLOUD_ROUTER_INTERFACE_NAMES}-{external_vpn_gateway_interface_id}",
                    "--peer-ip-address",
                    aws_peer_inside_address,
                    "--region",
                    GCP_REGION,
                    "--advertisement-mode",
                    "custom",
                    "--set-advertisement-groups",
                    "all_subnets",
                    "--set-advertisement-ranges",
                    f"{google_managed_services_cidr}=google-managed-services"
                ]
                print(' '.join(cmd))
                err = subprocess.run(cmd, capture_output=True).stderr
                print(err)
                if(b'' == err):
                    print('... added BGP peer')
                else:
                    print(err)
                    raise(err)
            else:
                print(err)
                raise Exception(err)


@click.command()
@click.option('--shared-secret-0', required=True)
@click.option('--shared-secret-1', required=True)
@click.option('--shared-secret-2', required=True)
@click.option('--shared-secret-3', required=True)
def main(shared_secret_0, shared_secret_1, shared_secret_2, shared_secret_3):
    ec2 = boto3.client('ec2')

    # Create AWS VPN Gateway, if needed
    res = ec2.describe_vpn_gateways(Filters=[
        {'Name': 'tag:Name', 'Values': [AWS_SIDE_VGW_NAME]},
        {'Name': 'state', 'Values': ['available']},
    ])
    if res['VpnGateways']:
        vpn_gateway_id = res['VpnGateways'][0]['VpnGatewayId']
        vpn_gateway_vpc_attachments = res['VpnGateways'][0]['VpcAttachments']
        print(f'Found AWS VPN Gateway ID {vpn_gateway_id} {AWS_SIDE_VGW_NAME}, skipping creation')
    else:
        print(f'Creating VPN Gateway {AWS_SIDE_VGW_NAME}')
        res = ec2.create_vpn_gateway(
            Type='ipsec.1',
            TagSpecifications=[
                {
                    'ResourceType': 'vpn-gateway',
                    'Tags': [
                        {
                            'Key': 'Name',
                            'Value': AWS_SIDE_VGW_NAME
                        },
                    ]
                },
            ],
            AmazonSideAsn=AWS_SIDE_ASN,
        )
        vpn_gateway_id = res['VpnGateway']['VpnGatewayId']
        vpn_gateway_vpc_attachments = res['VpnGateway']['VpcAttachments']

    if not(vpn_gateway_vpc_attachments):
        print(f'Attaching VPN Gateway {vpn_gateway_id} to VPC {AWS_VPC_ID}')
        ec2.attach_vpn_gateway(VpcId=AWS_VPC_ID, VpnGatewayId=vpn_gateway_id)
        time.sleep(3)       # Sleep to give some time to be attached before propagating

    # Propagate routes from VGWs to all subnets
    res = ec2.describe_route_tables(Filters=[{'Name': 'vpc-id', 'Values': [AWS_VPC_ID]}])
    futures = []
    with concurrent.futures.ThreadPoolExecutor(max_workers=int(os.getenv("MAX_WORKERS", THREADPOOL_MAX_WORKERS))) as executor:
        for route_table in res['RouteTables']:
            print(f'Enable route propgation from VGW {vpn_gateway_id} to Route Table {route_table["RouteTableId"]}')
            futures.append(executor.submit(
                ec2.enable_vgw_route_propagation,
                {'GatewayId': vpn_gateway_id, 'RouteTableId': route_table['RouteTableId']}
            ))
        concurrent.futures.wait(futures)

    if DELETE_GCP_GATEWAY:
        subprocess.run(["gcloud", "compute", "vpn-gateways",
                       "delete", GCP_VPN_GW_NAME], capture_output=True)
        subprocess.run(["gcloud", "compute", "routers",
                       "delete", GCP_CLOUD_ROUTER_NAME], capture_output=True)

    if DELETE_GCP_TUNNELS:
        for i in [1, 2, 3, 4]:
            print(f"Delete GCP tunnel {i}")
            subprocess.run(
                [
                    "gcloud",
                    "compute",
                    "vpn-tunnels",
                    "delete",
                    "--quiet",
                    f"{GCP_SIDE_TUNNEL_NAMES}-{i}",
                ]
            )

    if CREATE_GCP_GATEWAY:
        cmd = ['gcloud', 'compute', 'vpn-gateways',
               'describe', GCP_VPN_GW_NAME, '--format', 'json']
        res_gcp_vpn_gateway = subprocess.run(cmd, capture_output=True).stdout
        if res_gcp_vpn_gateway:
            res_gcp_vpn_gateway = json.loads(res_gcp_vpn_gateway)
            print(
                f'Found GCP VPN Gateway {res_gcp_vpn_gateway["name"]}, skipping creation')
        else:
            res_gcp_vpn_gateway = subprocess.run(
                [
                    "gcloud",
                    "compute",
                    "vpn-gateways",
                    "create",
                    GCP_VPN_GW_NAME,
                    "--network",
                    GCP_NETWORK,
                    "--region",
                    GCP_REGION,
                    "--format",
                    "json",
                ],
                capture_output=True).stdout
            res_gcp_vpn_gateway = json.loads(res_gcp_vpn_gateway)
            # Note appears to break on first run; possibly output from "create" different to "describe"

        # Create AWS Customer Gateways for the GCP VPN Gateway's (2) External IP Addresses if needed
        cgw_index = 0
        for interface in res_gcp_vpn_gateway['vpnInterfaces']:
            ip = interface['ipAddress']
            print(f'Setup Connectivity for GCP VPN Gateway Interface on {ip}')
            res = ec2.describe_customer_gateways(Filters=[
                {'Name': 'ip-address', 'Values': [ip]}]
            )
            if res['CustomerGateways']:
                customer_gateway_id = res['CustomerGateways'][0]['CustomerGatewayId']
                print(
                    f'\tFound AWS Customer Gateway ID {customer_gateway_id} for IP {ip}, skipping creation')
            else:
                # Create gateway
                cmd = [
                    'aws',
                    'ec2',
                    'create-customer-gateway',
                    '--device-name', f'{AWS_SIDE_CGW_NAME}-{cgw_index}-{ip}',
                    '--type', 'ipsec.1',
                    '--public-ip', ip,
                    '--bgp-asn', str(GCP_SIDE_ASN),
                    '--output', 'json'
                ]
                res_customer_gateway = subprocess.run(
                    cmd, capture_output=True).stdout
                res_customer_gateway = json.loads(res_customer_gateway)
                customer_gateway_id = res_customer_gateway['CustomerGateway']['CustomerGatewayId']

            # Attach Customer Gateway to VPN via 2 Tunnels, if needed
            res = ec2.describe_vpn_connections(Filters=[
                {
                    'Name': 'customer-gateway-id',
                    'Values': [customer_gateway_id]
                },
                {
                    'Name': 'vpn-gateway-id',
                    'Values': [vpn_gateway_id]
                },
            ])
            if res['VpnConnections']:
                print(
                    f'\tFound existing AWS VPN Connections for Customer Gateway ID {customer_gateway_id} to VPN Gateway ID {vpn_gateway_id}, skipping')
            else:
                # Create VPN Connection with 2 Tunnels
                this_shared_secret_0 = eval(f'shared_secret_{2 * cgw_index}')
                this_shared_secret_1 = eval(f'shared_secret_{2 * cgw_index + 1}')
                this_tunnel_inside_cidr_0 = AWS_TUNNEL_INSIDE_CIDRS[2 * cgw_index]
                this_tunnel_inside_cidr_1 = AWS_TUNNEL_INSIDE_CIDRS[2 * cgw_index + 1]
                try:
                    print(f'\tCreate AWS VPN Connection {cgw_index} to IP {ip} - Tunnel 1 {AWS_TUNNEL_INSIDE_CIDRS[2 * cgw_index]} Tunnel 2 {AWS_TUNNEL_INSIDE_CIDRS[2 * cgw_index + 1]}')
                    res = ec2.create_vpn_connection(
                        CustomerGatewayId=customer_gateway_id,
                        Type='ipsec.1',
                        VpnGatewayId=vpn_gateway_id,
                        Options={
                            'TunnelOptions': [
                                {
                                    'TunnelInsideCidr': this_tunnel_inside_cidr_0,
                                    'PreSharedKey': this_shared_secret_0,
                                },
                                {
                                    'TunnelInsideCidr': this_tunnel_inside_cidr_1,
                                    'PreSharedKey': this_shared_secret_1,
                                }
                            ]
                        }
                    )
                except botocore.exceptions.ClientError as error:
                    print('---------------------')
                    pprint(error)
                    print('---------------------')
                    raise error
            cgw_index += 1

        # Create GCP Cloud Router (will fail if existing) finally
        subprocess.run(
            [
                "gcloud",
                "compute",
                "routers",
                "create",
                GCP_CLOUD_ROUTER_NAME,
                "--region",
                GCP_REGION,
                "--network",
                GCP_NETWORK,
                "--asn",
                str(GCP_SIDE_ASN),
                "--advertisement-mode",
                "custom",
                "--set-advertisement-groups",
                "all_subnets",
            ],
            capture_output=True
        )

    if CREATE_GCP_TUNNELS:
        while vpn_connections := ec2.describe_vpn_connections(Filters=[{'Name': 'state', 'Values': ['pending']}])['VpnConnections']:
            print(f'VPN connections {[c["VpnConnectionId"] for c in vpn_connections]} still coming up/pending, waiting ....')
            time.sleep(2)
        aws_vpn_connections = ec2.describe_vpn_connections(Filters=[
            {'Name': 'state', 'Values': ['available']},
            {'Name': 'tag:Name', 'Values': [AWS_SIDE_VPN_CONNECTION_NAME]}
        ])['VpnConnections']

        print(f'Found existing {len(aws_vpn_connections)} AWS VPN Connections named {AWS_SIDE_VPN_CONNECTION_NAME}')
        # We expect 2 vpn connections
        assert 2 == len(aws_vpn_connections)

        # Create external VPN GW with 4 interfaces for the 4 AWS outside IPs
        aws_peer_ip_addresses = sum(
            [
                [x["OutsideIpAddress"]
                 for x in aws_vpn_connection["VgwTelemetry"]]
                for aws_vpn_connection in aws_vpn_connections
            ],
            [],
        )
        assert 4 == len(aws_peer_ip_addresses)

        res_vpn_gateway = subprocess.run(
            [
                "gcloud",
                "compute",
                "external-vpn-gateways",
                "list",
                "--filter",
                f"name={GCP_PEER_GATEWAY_NAME}",
                "--format",
                "json"
            ],
            capture_output=True
        ).stdout
        res_vpn_gateway = json.loads(res_vpn_gateway)
        if res_vpn_gateway:
            assert(1 == len(res_vpn_gateway))
            gcp_external_vpn_gateway_ips = [x['ipAddress'] for x in res_vpn_gateway[0]['interfaces']]
            print(f'Found existing GCP External VPN "{GCP_PEER_GATEWAY_NAME}" for IPs {gcp_external_vpn_gateway_ips}')
            if sorted(gcp_external_vpn_gateway_ips) != sorted(aws_peer_ip_addresses):
                raise Exception(f'Found GCP External VPN Gateway named "{GCP_PEER_GATEWAY_NAME}" but with IPs {gcp_external_vpn_gateway_ips}, not expected AWS ones {aws_peer_ip_addresses} - bailing')
        else:
            # Create the external VPN Gateway
            # This is equivalent to AWS-side customer gateway; it's a representation of the far side of the VPN connection
            # Hence it will have the 4 AWS IPs
            print(f"Create GCP VPN Gateway with External IPs {aws_peer_ip_addresses}")
            subprocess.run(
                [
                    "gcloud",
                    "compute",
                    "external-vpn-gateways",
                    "create",
                    GCP_PEER_GATEWAY_NAME,
                    "--interfaces",
                    f"0={aws_peer_ip_addresses[0]},1={aws_peer_ip_addresses[1]},2={aws_peer_ip_addresses[2]},3={aws_peer_ip_addresses[3]}",
                ],
                capture_output=True
            )

        # Stand up the tunnels - will just silently fail if existing
        google_managed_services_cidr = get_google_managed_services_cidr()
        print(f'google_managed_services_cidr (dbs) = {google_managed_services_cidr}')

        # Get mapping of AWS Peer IP Address -> interface id
        res = subprocess.run(
            [
                "gcloud",
                "compute",
                "external-vpn-gateways",
                "describe",
                GCP_PEER_GATEWAY_NAME,
                "--format",
                "json"
            ],
            capture_output=True
        ).stdout
        res = json.loads(res)
        aws_ip_to_interface = {}
        for interface in res['interfaces']:
            aws_ip_to_interface[interface['ipAddress']] = interface['id']

        # Get mapping of GCP Peer IP Address -> interface id
        res = subprocess.run(
            [
                "gcloud",
                "compute",
                "vpn-gateways",
                "describe",
                GCP_VPN_GW_NAME,
                "--format",
                "json"
            ],
            capture_output=True
        ).stdout
        res = json.loads(res)
        gcp_ip_to_interface = {}
        for interface in res['vpnInterfaces']:
            gcp_ip_to_interface[interface['ipAddress']] = interface['id']

        futures = []
        with concurrent.futures.ThreadPoolExecutor(max_workers=int(os.getenv("MAX_WORKERS", THREADPOOL_MAX_WORKERS))) as executor:
            for aws_vpn_connection in aws_vpn_connections:
                customer_gateway_configuration = BeautifulSoup(
                    aws_vpn_connection["CustomerGatewayConfiguration"],
                    features="xml",
                )
                tunnels = customer_gateway_configuration.find_all("ipsec_tunnel")
                assert 2 == len(tunnels)

                for tunnel in tunnels:
                    shared_secret = tunnel.find(
                        "ike").find("pre_shared_key").text
                    tunnel_inside_address = (
                        tunnel.find("customer_gateway")
                        .find("tunnel_inside_address")
                        .find("ip_address")
                        .text
                    )
                    aws_peer_inside_address = (
                        tunnel.find("vpn_gateway")
                        .find("tunnel_inside_address")
                        .find("ip_address")
                        .text
                    )
                    aws_peer_outside_address = (
                        tunnel.find("vpn_gateway")
                        .find("tunnel_outside_address")
                        .find("ip_address")
                        .text
                    )
                    gcp_peer_outside_address = (
                        tunnel.find("customer_gateway")
                        .find("tunnel_outside_address")
                        .find("ip_address")
                        .text
                    )

                    # Get the tunnel/interface number for this IP
                    print('-'*80)
                    external_vpn_gateway_interface_id = aws_ip_to_interface[aws_peer_outside_address]
                    pprint(aws_ip_to_interface)
                    print(f'aws ip {aws_peer_outside_address} id on interface id {external_vpn_gateway_interface_id}')

                    # Get the correct VPN Gateway Interface number for the GCP Peer Outside IP
                    internal_vpn_gateway_interface_id = gcp_ip_to_interface[gcp_peer_outside_address]
                    pprint(gcp_ip_to_interface)
                    print(f'gcp ip {gcp_peer_outside_address} id on internal interface id {internal_vpn_gateway_interface_id}')

                    print(f'Create GCP tunnel #{internal_vpn_gateway_interface_id} ({tunnel_inside_address}) - to {aws_peer_outside_address} ({aws_peer_inside_address})')
                    futures.append(executor.submit(
                        create_gcp_tunnel,
                        external_vpn_gateway_interface_id,
                        internal_vpn_gateway_interface_id,
                        aws_peer_inside_address,
                        tunnel_inside_address,
                        shared_secret,
                        google_managed_services_cidr,
                    ))
        concurrent.futures.wait(futures)


if __name__ == '__main__':
    main()
    print('Complete!')