Ground Signal Engineering - Service Discovery Example

discovery.py

import json
import boto3 # boto3-1.5.7
import os
import logging
import time
import sys

logger = logging.getLogger()
logger.setLevel(logging.INFO)

def discover(event, context):
	service_discovery_domain_name = os.environ['ServiceDiscoveryDomainName']
	zone_id = os.environ['Route53PrivateHostedZoneID']
	cluster = os.environ['ECSClusterName']
	region = os.environ['AWSRegion']

	ecs_client = boto3.client('ecs')
	ec2_client = boto3.client('ec2')
	route53_client = boto3.client('route53')

	last_status = event['detail']['lastStatus']

	if last_status == 'RUNNING' or last_status == 'STOPPED':
		desc = ecs_client.describe_container_instances(cluster=cluster, containerInstances=[event['detail']['containerInstanceArn']])
		logger.info("container instance description: %s", desc)
		container_instances = desc['containerInstances']
		for container_instance in container_instances:
			logger.info("container instance status: %s", container_instance['status'])
			ec2instance_id = container_instance['ec2InstanceId']
			logger.info("Host EC2 instance id is %s", ec2instance_id)
			ec2_instance_details = ec2_client.describe_instances(InstanceIds=[ec2instance_id])
			logger.info("EC2 instance details: %s", ec2_instance_details)
			ec2_instance_private_ip_address = ec2_instance_details['Reservations'][0]['Instances'][0]['NetworkInterfaces'][0]['PrivateIpAddress']
			logger.info("EC2 instance private IP address: %s", ec2_instance_private_ip_address)

			if last_status == 'RUNNING':
				try:
					existing_records = route53_client.list_resource_record_sets(
						HostedZoneId=zone_id,
						StartRecordName='_myservice._tcp.' + service_discovery_domain_name,
						StartRecordType='SRV'
					)
					resource_record_sets = existing_records['ResourceRecordSets']

					resource_record_sets_copy = list(resource_record_sets)
					srv_resource_record_sets = [r for r in resource_record_sets_copy if r['Type'] == 'SRV']
					if srv_resource_record_sets:
						srv_resource_records = srv_resource_record_sets[0]['ResourceRecords']
					else:
						srv_resource_records = []

					srv_resource_records_copy = list(srv_resource_records)
					new_resource_record = {'Value': '1 1 8080 ' + ec2instance_id + '.' + service_discovery_domain_name}
					if new_resource_record not in srv_resource_records_copy:
						logger.info("Registering myservice instance %s in DNS", event['detail']['containerInstanceArn'])
						srv_resource_records_copy.append(new_resource_record.copy())

						srv_response = route53_client.change_resource_record_sets(
							HostedZoneId=zone_id,
							ChangeBatch={
								'Comment': 'myservice instance',
								'Changes': [
									{
										'Action': 'UPSERT',
										'ResourceRecordSet': {
											'Name': '_myservice._tcp.' + service_discovery_domain_name,
											'Type': 'SRV',
											'TTL': 60,
											'ResourceRecords': srv_resource_records_copy
										}
									}
								]
							}
						)
						logger.info("upsert SRV record response: %s", srv_response)

						a_response = route53_client.change_resource_record_sets(
							HostedZoneId=zone_id,
							ChangeBatch={
								'Comment': 'myservice instance',
								'Changes': [
									{
										'Action': 'UPSERT',
										'ResourceRecordSet': {
											'Name': ec2instance_id + '.' + service_discovery_domain_name,
											'Type': 'A',
											'TTL': 60,
											'ResourceRecords': [
												{
													'Value': ec2_instance_private_ip_address
												}
											]
										}
									}
								]
							}
						)
						logger.info("upsert A record response: %s", a_response)
					else:
						logger.info("%s is already registered in DNS", ec2instance_id)
				except:
					logger.info("Unexpected error: %s", sys.exc_info())

			elif last_status == 'STOPPED':
				try:
					existing_records = route53_client.list_resource_record_sets(
						HostedZoneId=zone_id,
						StartRecordName='_myservice._tcp.' + service_discovery_domain_name,
						StartRecordType='SRV'
					)
					resource_record_sets = existing_records['ResourceRecordSets']

					resource_record_sets_copy = list(resource_record_sets)
					srv_resource_record_sets = [r for r in resource_record_sets_copy if r['Type'] == 'SRV']
					if srv_resource_record_sets:
						srv_resource_records = srv_resource_record_sets[0]['ResourceRecords']
					else:
						srv_resource_records = []

					srv_resource_records_copy = list(srv_resource_records)
					removed_resource_record = {'Value': '1 1 8080 ' + ec2instance_id + '.' + service_discovery_domain_name}
					if removed_resource_record in srv_resource_records_copy:
						logger.info("Unregistering myservice instance %s in DNS", event['detail']['containerInstanceArn'])
						srv_resource_records_copy.remove(removed_resource_record.copy())

						a_response = route53_client.change_resource_record_sets(
							HostedZoneId=zone_id,
							ChangeBatch={
								'Comment': 'myservice instance',
								'Changes': [
									{
										'Action': 'DELETE',
										'ResourceRecordSet': {
											'Name': ec2instance_id + '.' + service_discovery_domain_name,
											'Type': 'A',
											'TTL': 60,
											'ResourceRecords': [
												{
													'Value': ec2_instance_private_ip_address
												}
											]
										}
									}
								]
							}
						)
						logger.info("delete A record response: %s", a_response)

						# if we still have entries after removing this instance, update the SRV record to remove the instance
						if srv_resource_records_copy:
							srv_response = route53_client.change_resource_record_sets(
								HostedZoneId=zone_id,
								ChangeBatch={
									'Comment': 'myservice instance',
									'Changes': [
										{
											'Action': 'UPSERT',
											'ResourceRecordSet': {
												'Name': '_myservice._tcp.' + service_discovery_domain_name,
												'Type': 'SRV',
												'TTL': 60,
												'ResourceRecords': srv_resource_records_copy
											}
										}
									]
								}
							)
							logger.info("upsert SRV record response: %s", srv_response)
						else:
							# if we don't have entries left after removing this instance, delete the SRV record
							srv_response = route53_client.change_resource_record_sets(
								HostedZoneId=zone_id,
								ChangeBatch={
									'Comment': 'myservice instance',
									'Changes': [
										{
											'Action': 'DELETE',
											'ResourceRecordSet': {
												'Name': '_myservice._tcp.' + service_discovery_domain_name,
												'Type': 'SRV',
												'TTL': 60,
												'ResourceRecords': srv_resource_records_copy
											}
										}
									]
								}
							)
							logger.info("delete SRV record response: %s", srv_response)
				except:
					logger.info("Unexpected error: %s", sys.exc_info()[0])