Skip to content

Instantly share code, notes, and snippets.

@jtroberts83

jtroberts83/CloudWatch-To-Elasticsearch-Metrics-Ingester.py

Created September 6, 2019 10:22
Show Gist options
  • Save jtroberts83/851eb48bc811e6a56d4a9c3cd5d29b66 to your computer and use it in GitHub Desktop.
Save jtroberts83/851eb48bc811e6a56d4a9c3cd5d29b66 to your computer and use it in GitHub Desktop.
Download ZIP
Python3.7 Script Which Queries CloudWatch For Desired Metrics Across All Accounts And Regions And Pushes To ElasticSearch
Raw
CloudWatch-To-Elasticsearch-Metrics-Ingester.py
##
## CloudWatch-To-Elasticsearch-Metrics-Ingester.py Script written by Jamison Roberts
##
## Description: This script is written in Python 3 and uses the AWS boto3 python library to make bulk calls to AWS S3 and CloudWatch services.
## A CSV of all Federated AWS accounts and names is downloaded from S3 and then the script performs a for loop on each account.
## Within each account the script will perform a for loop on each region specified, create a cloudwatch boto3 client, and then
## query CloudWatch Metrics to pull metrics counts for each metric provided in the call. Then the total returned metrics count
## for each account are totalled and printed on the console.
## Import required libraries used in this script
from elasticsearch import Elasticsearch, RequestsHttpConnection
from elasticsearch import helpers
from requests_aws4auth import AWS4Auth
import boto3
import json
from botocore.client import Config
from datetime import datetime, timedelta, date
import time
import random
import string
## Create a variable called ScriptStartTime with the current date and time which we will use to calculate the total script runtime with
ScriptStartTime = datetime.now()
## Initial variables
RegionsArray = ['us-east-1','eu-west-1','us-west-1']
region = 'us-east-1'
thisaccount = '<AWS_ACCOUNT_NUMBER_THIS_IS_RUNNING_IN>'
RoleArn = 'arn:aws:iam::' + thisaccount + ':role/<ROLE_NAME_TO_ASSUME>'
docs = []
previoustimestamp = ""
## Create our AWS S3 boto3 resource (different than the S3 boto3 client)
## Download the csv of all AWS Federated Accounts and then read it into the variable accounts
## Convert the accounts variable from json into a list
s3resource = boto3.resource('s3', config=Config(signature_version='s3v4'), region_name="us-east-1")
randomstring = ''.join(random.choices(string.ascii_uppercase + string.digits, k = 10))
s3resource.meta.client.download_file('<S3_Bucket>', '<S3_OBJECT_PATH>/AccountNumbersAndNames.csv', 'C:\\Tempy\\'+ randomstring + 'accounts.csv')
f = open('C:\\Tempy\\'+ randomstring + 'accounts.csv', 'r')
accounts = f.read()
accounts = json.loads(accounts)
# COMMENT THE BELOW LINE OUT!!!!! Used For Testing so the script will only run against 1 account rather than all accounts
#accounts = ['<AWS_ACCOUNT_NUMBER>:<AWS_ACCOUNT_ALIAS>']
## Creates the AWS STS boto3 client which we call later in the script to obtain temp credentials for each account
stsClient = boto3.client('sts')
fifteenMinutesAgo = datetime.utcnow() - timedelta(hours=0, minutes=15)
fourtyFiveMinutesAgo = datetime.utcnow() - timedelta(hours=0, minutes=45)
## Set our initial policy invocation counters to 0 and then create the headers row for the csvFileData
##<FOREACH METRIC-LABEL Total<METRIC-LABEL>Values = 0>
TotaloldaccesskeysdisableValues = 0
TotalappelbupdateoldsslValues = 0
TotalunencrypteddynamodbtabledeletedValues = 0
Totalunencryptedec2terminatedbypolicyValues = 0
## Connect to Elasticsearch Service in AWS
host = '<AWS_ELASTICSEARCH_ARN>'
stsClient = boto3.client('sts')
Creds = stsClient.assume_role(
RoleArn=RoleArn,
RoleSessionName='CW-TO-ES-INGEST',
DurationSeconds=900
)
AccessKey = ""
AccessKey = Creds['Credentials']['AccessKeyId']
SecretAccessKey = ""
SecretAccessKey = Creds['Credentials']['SecretAccessKey']
SessionToken = ""
SessionToken = Creds['Credentials']['SessionToken']
awsauth = AWS4Auth(AccessKey,SecretAccessKey,'us-east-1', 'es',session_token=SessionToken)
es = Elasticsearch(
hosts=[{'host': host, 'port': 443}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection,
timeout=40
)
print(es.info())
###################################################################################
### START FOREACH ACCOUNT / REGION LOOP HERE
###################################################################################
## Loop through each account entry of the Accounts file we read in and converted from JSON earlier
for item in accounts:
## Extract the AWS Account Number and Name from the item passed in from Accounts list (In the format of "account_number:account_name")
Account = ""
AccountName = ""
item = item.split(":")
Account = item[0]
#print(Account)
AccountName = item[1]
AccountName = AccountName.lower()
Division = ""
if("<SOME_TERM>" in AccountName):
Division = "<DIVISION_NAME>"
elif("<SOME_OTHER_TERM>" in AccountName):
Division = "<ANOTHER_DIVISION_NAME>"
## Continue this for each division type
else:
Division = "UNKNOWN"
## Try to assume the <ROLE_NAME_TO_ASSUME> for the given account we are looping through using the AWS STS boto3 client
## These temp STS credentials will then be used for our CloudWatch client to pull metrics for the given account we are looping through
try:
RoleArn = 'arn:aws:iam::' + Account + ':role/<ROLE_NAME_TO_ASSUME>'
if Account == thisaccount:
print("%s ACCOUNT - NOT USING STS" % thisaccount)
else:
Creds = stsClient.assume_role(
RoleArn=RoleArn,
RoleSessionName='Gather-Metrics'
)
AccessKey = ""
AccessKey = Creds['Credentials']['AccessKeyId']
SecretAccessKey = ""
SecretAccessKey = Creds['Credentials']['SecretAccessKey']
SessionToken = ""
SessionToken = Creds['Credentials']['SessionToken']
except:
print("Error Getting Temp Creds From Account %s" % AccountName)
## Loops through each region in the RegionsArray list specified at the top of the script
for Region in RegionsArray:
try:
## If the account being compared is the thisaccount account DONT use STS as we already have credentials for this account,
## otherwise create a CloudWatch AWS boto3 client for the given account and region
if Account == thisaccount:
cloudwatch_client = boto3.client('cloudwatch', region_name='us-east-1')
else:
cloudwatch_client = boto3.client('cloudwatch', aws_access_key_id=AccessKey, aws_secret_access_key=SecretAccessKey, aws_session_token=SessionToken, region_name=Region)
except:
print("ERROR CREATING CLOUDWATCH BOTO3 CLIENT IN SOME REGION")
## Calls the CloudWatch Get Metric Data boto3 API call for the given account and region in the loop
response = cloudwatch_client.get_metric_data(
MetricDataQueries=[
## ALL Metrics To Retrieve (Each Line is a new Metric)
##FOREACH METRIC: {'Id':'metriclabel','MetricStat':{'Metric':{'Namespace':'<YOUR_CW_METRIC_NAMESPACE>','MetricName':'metricname','Dimensions':[]},'Period':1800,'Stat':'Sum','Unit':'Count'},'Label':'metriclabel','ReturnData':True},
{'Id':'oldaccesskeysdisable','MetricStat':{'Metric':{'Namespace':'<YOUR_CW_METRIC_NAMESPACE>','MetricName':'Old-Access-Keys-Disable','Dimensions':[]},'Period':1800,'Stat':'Sum','Unit':'Count'},'Label':'oldaccesskeysdisable','ReturnData':True},
{'Id':'appelbupdateoldssl','MetricStat':{'Metric':{'Namespace':'<YOUR_CW_METRIC_NAMESPACE>','MetricName':'App-ELB-Update-Old-SSL','Dimensions':[]},'Period':1800,'Stat':'Sum','Unit':'Count'},'Label':'appelbupdateoldssl','ReturnData':True},
{'Id':'unencrypteddynamodbtabledeleted','MetricStat':{'Metric':{'Namespace':'<YOUR_CW_METRIC_NAMESPACE>','MetricName':'Unencrypted-DynamoDB-Table-Deleted','Dimensions':[]},'Period':1800,'Stat':'Sum','Unit':'Count'},'Label':'unencrypteddynamodbtabledeleted','ReturnData':True},
{'Id':'unencryptedec2terminatedbypolicy','MetricStat':{'Metric':{'Namespace':'<YOUR_CW_METRIC_NAMESPACE>','MetricName':'Unencrypted-EC2-Terminated-By-Policy','Dimensions':[]},'Period':1800,'Stat':'Sum','Unit':'Count'},'Label':'unencryptedec2terminatedbypolicy','ReturnData':True}
],
StartTime=fourtyFiveMinutesAgo,
EndTime=fifteenMinutesAgo,
MaxDatapoints=100800
)
## For each metric in the returned CloudWatch get_metric_data command, loop through and extract the Label and Values and reset the counters
for metric in response['MetricDataResults']:
Label = ""
Label = metric['Label']
Values =""
Values = metric['Values']
Timestamp = datetime.now()
Timestamp = metric['Timestamps']
if not Timestamp:
Timestamp = datetime.now()
TotaloldaccesskeysdisableValuesByAccount = 0
TotalappelbupdateoldsslValuesByAccount = 0
TotalunencrypteddynamodbtabledeletedValuesByAccount = 0
Totalunencryptedec2terminatedbypolicyValuesByAccount = 0
##print(Label)
dt = datetime.now()
microseconds = dt.microsecond
epoch_time = int(time.time())
randominteger = random.randint(1000, 9999)
id=("%s%s%s" % (epoch_time,microseconds,randominteger))
## Sleep for a fraction of a second so that our microseconds timestamp is unique appending random int to ensure uniqueness
time.sleep(1/100000.0)
## Based on the Label of the Metric being returned, sum it, print it and add it as a row in the csv file data variable
#FOREACH IF..ELIF Label=MetricLabel
if Label == 'oldaccesskeysdisable':
for Value in Values:
dt = datetime.now()
microseconds = dt.microsecond
epoch_time = int(time.time())
randominteger = random.randint(1000, 9999)
id=("%s%s%s" % (epoch_time,microseconds,randominteger))
try:
TotaloldaccesskeysdisableValues = ( TotaloldaccesskeysdisableValues + Value )
TotaloldaccesskeysdisableValuesByAccount = ( TotaloldaccesskeysdisableValuesByAccount + Value )
except:
print("Error getting or adding Total oldaccesskeysdisable Values")
if len(Values) > 0:
print("Account: %s Region: %s Label: %s Total Invocations: %s" % (Account,Region,Label,int(TotaloldaccesskeysdisableValuesByAccount)))
oldaccesskeysdisableList = []
oldaccesskeysdisableList = [Account,AccountName,Region,Label,int(TotaloldaccesskeysdisableValuesByAccount)]
doc = {
'MetricName': 'Old-Access-Keys-Disable',
'id': id,
'Policy': '',
'Resource': '',
'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
'Sum': TotaloldaccesskeysdisableValuesByAccount,
'Account': Account,
'AccountName': AccountName,
'Division': Division,
'Region': Region,
'timestamp': Timestamp,
}
else:
doc = {
'MetricName': 'Old-Access-Keys-Disable',
'id': id,
'Policy': '',
'Resource': '',
'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
'Sum': 0,
'Account': Account,
'AccountName': AccountName,
'Division': Division,
'Region': Region,
'timestamp': Timestamp,
}
docs.append(doc)
elif Label == 'appelbupdateoldssl':
for Value in Values:
dt = datetime.now()
microseconds = dt.microsecond
epoch_time = int(time.time())
randominteger = random.randint(1000, 9999)
id=("%s%s%s" % (epoch_time,microseconds,randominteger))
try:
TotalappelbupdateoldsslValues = ( TotalappelbupdateoldsslValues + Value )
TotalappelbupdateoldsslValuesByAccount = ( TotalappelbupdateoldsslValuesByAccount + Value )
except:
print("Error getting or adding Total appelbupdateoldssl Values")
if len(Values) > 0:
print("Account: %s Region: %s Label: %s Total Invocations: %s" % (Account,Region,Label,int(TotalappelbupdateoldsslValuesByAccount)))
appelbupdateoldsslList = []
appelbupdateoldsslList = [Account,AccountName,Region,Label,int(TotalappelbupdateoldsslValuesByAccount)]
doc = {
'MetricName': 'App-ELB-Update-Old-SSL',
'id': id,
'Policy': '',
'Resource': '',
'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
'Sum': TotalappelbupdateoldsslValuesByAccount,
'Account': Account,
'AccountName': AccountName,
'Division': Division,
'Region': Region,
'timestamp': Timestamp,
}
else:
doc = {
'MetricName': 'App-ELB-Update-Old-SSL',
'id': id,
'Policy': '',
'Resource': '',
'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
'Sum': 0,
'Account': Account,
'AccountName': AccountName,
'Division': Division,
'Region': Region,
'timestamp': Timestamp,
}
docs.append(doc)
elif Label == 'unencrypteddynamodbtabledeleted':
for Value in Values:
dt = datetime.now()
microseconds = dt.microsecond
epoch_time = int(time.time())
randominteger = random.randint(1000, 9999)
id=("%s%s%s" % (epoch_time,microseconds,randominteger))
try:
TotalunencrypteddynamodbtabledeletedValues = ( TotalunencrypteddynamodbtabledeletedValues + Value )
TotalunencrypteddynamodbtabledeletedValuesByAccount = ( TotalunencrypteddynamodbtabledeletedValuesByAccount + Value )
except:
print("Error getting or adding Total unencrypteddynamodbtabledeleted Values")
if len(Values) > 0:
print("Account: %s Region: %s Label: %s Total Invocations: %s" % (Account,Region,Label,int(TotalunencrypteddynamodbtabledeletedValuesByAccount)))
unencrypteddynamodbtabledeletedList = []
unencrypteddynamodbtabledeletedList = [Account,AccountName,Region,Label,int(TotalunencrypteddynamodbtabledeletedValuesByAccount)]
doc = {
'MetricName': 'Unencrypted-DynamoDB-Table-Deleted',
'id': id,
'Policy': '',
'Resource': '',
'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
'Sum': TotalunencrypteddynamodbtabledeletedValuesByAccount,
'Account': Account,
'AccountName': AccountName,
'Division': Division,
'Region': Region,
'timestamp': Timestamp,
}
else:
doc = {
'MetricName': 'Unencrypted-DynamoDB-Table-Deleted',
'id': id,
'Policy': '',
'Resource': '',
'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
'Sum': 0,
'Account': Account,
'AccountName': AccountName,
'Division': Division,
'Region': Region,
'timestamp': Timestamp,
}
docs.append(doc)
elif Label == 'unencryptedec2terminatedbypolicy':
for Value in Values:
dt = datetime.now()
microseconds = dt.microsecond
epoch_time = int(time.time())
randominteger = random.randint(1000, 9999)
id=("%s%s%s" % (epoch_time,microseconds,randominteger))
try:
Totalunencryptedec2terminatedbypolicyValues = ( Totalunencryptedec2terminatedbypolicyValues + Value )
Totalunencryptedec2terminatedbypolicyValuesByAccount = ( Totalunencryptedec2terminatedbypolicyValuesByAccount + Value )
except:
print("Error getting or adding Total unencryptedec2terminatedbypolicy Values")
if len(Values) > 0:
print("Account: %s Region: %s Label: %s Total Invocations: %s" % (Account,Region,Label,int(Totalunencryptedec2terminatedbypolicyValuesByAccount)))
unencryptedec2terminatedbypolicyList = []
unencryptedec2terminatedbypolicyList = [Account,AccountName,Region,Label,int(Totalunencryptedec2terminatedbypolicyValuesByAccount)]
doc = {
'MetricName': 'Unencrypted-EC2-Terminated-By-Policy',
'id': id,
'Policy': '',
'Resource': '',
'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
'Sum': Totalunencryptedec2terminatedbypolicyValuesByAccount,
'Account': Account,
'AccountName': AccountName,
'Division': Division,
'Region': Region,
'timestamp': Timestamp,
}
else:
doc = {
'MetricName': 'Unencrypted-EC2-Terminated-By-Policy',
'id': id,
'Policy': '',
'Resource': '',
'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
'Sum': 0,
'Account': Account,
'AccountName': AccountName,
'Division': Division,
'Region': Region,
'timestamp': Timestamp,
}
docs.append(doc)
## Print Totals across all accounts and specified regions for each policy metric
print(" ")
print(" ")
print(" ")
##FOREACH METRIC DO: print(" Total metriclabel Policy Invocations: %s" % TotalmetriclabelValues)
print(" Total oldaccesskeysdisable Policy Invocations: %s" % TotaloldaccesskeysdisableValues)
print(" Total appelbupdateoldssl Policy Invocations: %s" % TotalappelbupdateoldsslValues)
print(" Total unencrypteddynamodbtabledeleted Policy Invocations: %s" % TotalunencrypteddynamodbtabledeletedValues)
print(" Total unencryptedec2terminatedbypolicy Policy Invocations: %s" % Totalunencryptedec2terminatedbypolicyValues)
bulk_action = []
print("THERE ARE %s DOCS IN THE DOCS ARRAY" % (len(docs)))
for doc in docs:
#print(doc)
bulk = {
"_index" : '<YOUR_ELASTICSEARCH_INDEX_NAME>',
"_type" : 'metrics',
"_id" : doc['id'],
"_source" : doc,
}
bulk_action.append(bulk)
try:
helpers.bulk(es, bulk_action)
print('Imported metrics data successfully!!!')
except Exception as ex:
print('Error:', ex)
## Calculates Total Script Run Time
TotalTime = (datetime.now() - ScriptStartTime)
## Print Script runtime
print(" ")
print(" ")
print("Script Finished. Elaspsed Time: %s" % TotalTime)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment