Skip to content

Instantly share code, notes, and snippets.

@sudharsans

sudharsans/cfn.yaml

Last active October 19, 2023 21:04
Show Gist options
  • Save sudharsans/af23ee7e8919947af83ceb269a40d8db to your computer and use it in GitHub Desktop.
Save sudharsans/af23ee7e8919947af83ceb269a40d8db to your computer and use it in GitHub Desktop.
Download ZIP
Lambda Script to Query Trust Advisor and Find Idle Resources
Raw
cfn.yaml
---
AWSTemplateFormatVersion: "2010-09-09"
Description:
Create Event rule and lambda functions to report for following criteria
Idle EC2 instances if CPU < 10% and Network < 5MB
RDS Instance with no connections for 7 days
ELB with No Active backends and requests below 100
Parameters:
LambdaName:
Default: AWSReport
Type: String
Environment:
Default: DEV
Type: String
Sender:
Default: [email protected]
Type: String
Recipient:
Default: [email protected]
Type: String
Exceptions:
Default: "TESTPRODUCT,ANOTHERTEST,Anothetest123"
Type: String
CodeBucket:
Default: devopscfn
Type: String
S3Key:
Default: cleanup/cleanup.zip
Type: String
Resources:
Role:
Type: "AWS::IAM::Role"
Properties:
RoleName: !Sub '${LambdaName}-Role'
Path: "/"
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Principal:
Service: lambda.amazonaws.com
Action: sts:AssumeRole
RolePolicies:
Type: "AWS::IAM::ManagedPolicy"
Properties:
ManagedPolicyName: !Sub '${LambdaName}-RolePolicies'
Roles:
- Ref: "Role"
PolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Action:
- ec2:describe*
- cloudtrail:LookupEvents
- autoscaling:Describe*
- elasticloadbalancing:DescribeTags
- config:PutEvaluations
- rds:DescribeDBLogFiles
- cloudwatch:GetMetricStatistics
- support:*
- trustedadvisor:*
- elasticloadbalancing:DescribeLoadBalancers
- rds:ListTagsForResource
- rds:DownloadDBLogFilePortion
- rds:DescribeDBInstances
- elasticloadbalancing:DescribeInstanceHealth
- redshift:DescribeClusters
Resource:
- '*'
- Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource:
- '*'
- Effect: Allow
Action:
- config:PutEvaluations
Resource:
- '*'
- Effect: Allow
Action:
- ses:SendEmail
Resource:
- '*'
Lambda:
Type: "AWS::Lambda::Function"
Properties:
Code:
S3Bucket: !Ref CodeBucket
S3Key: !Ref S3Key
Description: "AWS Cleanup"
FunctionName: !Sub '${LambdaName}Function'
Handler: "main.lambda_handler"
Timeout: 300
Environment:
Variables:
"env": !Ref Environment
"exceptions": !Ref Exceptions
"sender": !Ref Sender
"recipient": !Ref Recipient
Role:
Fn::GetAtt:
- Role
- Arn
Runtime: "python3.6"
Tags:
-
Key: "Name"
Value: "Ops"
ScheduledRule:
Type: "AWS::Events::Rule"
Properties:
Name: !Sub '${LambdaName}-EventRule'
Description: "ScheduledRule"
ScheduleExpression: "rate(1 day)"
State: "ENABLED"
Targets:
-
Arn:
Fn::GetAtt:
- "Lambda"
- "Arn"
Id: "TargetFunction1"
PermissionForEventsToInvokeLambda:
Type: "AWS::Lambda::Permission"
Properties:
FunctionName: !Ref Lambda
Action: "lambda:InvokeFunction"
Principal: "events.amazonaws.com"
SourceArn: !GetAtt ScheduledRule.Arn
Raw
email.html
<html>
<head></head>
<body>
<h1> EC2 Instances</h1>
<p><b>Criteria</b> :</br>
EC2 instances that have had 10% or less daily average CPU utilization<br />
and 5 MB or less network I/O on last 14 days </p>
{% if ec2 %}
<table>
<thead>
<tr>
<th>Instance-Id</th>
<th>Name</th>
<th>Instance Type</th>
<th>Cost</th>
<th>CPU</th>
<th>Network IO</th>
<th>Days</th>
<th>ProductName</th>
</tr>
</thead>
<tbody>
{% for instance in ec2 %}
<tr>
{% for details in instance %}
<td>{{ details }}</td>
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<b>No Instance matching the criteria </b>
{% endif %}
<h1>ELB</h1>
<p><b>Criteria:</b></br>
A load balancer has no active back-end instances.</br>
A load balancer has no healthy back-end instances.</br>
</br>
</p>
{% if elb %}
<table>
<thead>
<tr>
<th>ELB Name</th>
<th>Reason</th>
<th>Cost</th>
<th>ProductName</th>
</tr>
</thead>
<tbody>
{% for instance in elb %}
<tr>
{% for details in instance %}
<td>{{ details }}</td>
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<b>No Instance matching the criteria </b>
{% endif %}
<h1> RDS Instances</h1>
<p><b>Criteria:</b>
</br>An active DB instance has not had a connection in the last 7 days. </p>
{% if rds %}
<table>
<thead>
<tr>
<th>Intance ID</th>
<th>Instance Type</th>
<th>Size</th>
<th>Days</th>
<th>Cost</th>
<th>ProductName</th>
</tr>
</thead>
<tbody>
{% for instance in rds %}
<tr>
{% for details in instance %}
<td>{{ details }}</td>
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<b>No Instance matching the criteria </b>
{% endif %}
<h1> EBS Volumes</h1>
<p><b>Criteria:</b> </br>A volume is unattached </p>
{% if ebs %}
<table>
<thead>
<tr>
<th>Volume-Id</th>
<th>Size</th>
<th>Cost</th>
<th>ProductName</th>
</tr>
</thead>
<tbody>
{% for instance in ebs %}
<tr>
{% for details in instance %}
<td>{{ details }}</td>
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<b>No Volumes matching the criteria </b>
{% endif %}
<h1> Red Shift</h1>
<p><b>Criteria:</b></br>A running cluster has not had a connection in the last 7 days.</br>
A running cluster had less than 5% cluster-wide average CPU utilization for 99% of the last 7 days</p>
{% if redshift %}
<table>
<thead>
<tr>
<th>Name</th>
<th>Instance Type</th>
<th>Reason</th>
<th>Cost</th>
<th>ProductName</th>
</tr>
</thead>
<tbody>
{% for instance in redshift %}
<tr>
{% for details in instance %}
<td>{{ details }}</td>
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<b>No Instance matching the criteria </b>
{% endif %}
</body>
</html>
Raw
mail.py
import boto3
from botocore.exceptions import ClientError
client = boto3.client('ses')
def send_email(details):
SENDER = details['sender']
RECIPIENT = details['recipient']
SUBJECT = details['subject']
BODY_TEXT = details['body']
CHARSET = "UTF-8"
# Try to send the email.
try:
# Provide the contents of the email.
response = client.send_email(
Destination={
'ToAddresses': [
RECIPIENT,
],
},
Message={
'Body': {
'Html': {
'Charset': CHARSET,
'Data': BODY_TEXT,
},
},
'Subject': {
'Charset': CHARSET,
'Data': SUBJECT,
},
},
Source=SENDER,
)
# Display an error if something goes wrong.
except ClientError as e:
print(e.response['Error']['Message'])
else:
print("Email sent! Message ID:" + response['ResponseMetadata']['RequestId'])
Raw
main.py
import boto3
from botocore.exceptions import ClientError
from jinja2 import Environment, FileSystemLoader
from mail import send_email
import os
import time
import re
ENV = os.environ['env']
SENDER = os.environ['sender']
RECIPIENT = os.environ['recipient']
exceptions = os.environ['exceptions']
ec2_client = boto3.client('ec2')
elb_client = boto3.client('elb')
def get_ec2_tags( instanceid):
try:
ec2_tags = ec2_client.describe_tags (
Filters=[
{
'Name': 'resource-id',
'Values': [instanceid]
}])
except ClientError as e:
print (e.response['Error']['Message'])
else:
if len (ec2_tags.get ("Tags")) > 0:
for tag in ec2_tags.get("Tags"):
if tag["Key"] == "Name":
return tag["Value"]
def get_elb_tags(elbname):
try:
elb_tags = elb_client.describe_tags (
LoadBalancerNames=[elbname])
except ClientError as e:
print (e.response['Error']['Message'])
else:
for i in elb_tags['TagDescriptions']:
if len (i.get ("Tags")) > 0:
for tag in i.get ("Tags"):
if tag["Key"] == "Name":
return tag["Value"]
def get_ebs_tags(vol):
try:
response = ec2_client.describe_volumes (
VolumeIds=[
vol,
],)
except ClientError as e:
print (e.response['Error']['Message'])
else:
if "Tags" in response['Volumes'][0]:
tags = response['Volumes'][0]["Tags"]
if len(tags) > 0:
for tag in tags:
if tag["Key"] == "Name":
return tag["Value"]
def get_rds_arn(rdsname):
db_instances = boto3.client("rds").describe_db_instances (DBInstanceIdentifier=rdsname)['DBInstances']
for instance in db_instances:
if instance['DBInstanceStatus'] == 'available':
return instance['DBInstanceArn']
def get_rds_tags(rdsname):
try:
rds_tags = boto3.client ("rds").list_tags_for_resource (
ResourceName=get_rds_arn(rdsname),
)
except ClientError as e:
print (e.response['Error']['Message'])
else:
if len (rds_tags.get ("TagList")) > 0:
for tag in rds_tags['TagList']:
if tag["Key"] == "Name":
return tag["Value"]
def get_redshift_tags(redshift):
client = boto3.client("redshift")
response = client.describe_clusters(
ClusterIdentifier=redshift
)
if response["Clusters"][0]["Tags"]:
for tag in response["Clusters"][0]["Tags"]:
if tag["Key"] == "Name":
return tag["Value"]
def sendmail(subject, body):
message = {
"subject": "[{0}] {1} ".format (ENV, subject),
"body": body,
"sender": SENDER,
"recipient": RECIPIENT
}
send_email (message)
def get_ebs_status(id):
ec2 = boto3.resource ('ec2')
volume = ec2.Volume (id)
return volume.state
def main():
client = boto3.client('support')
response = client.describe_trusted_advisor_checks(
language='en'
)
ec2 = []
elb = []
rds = []
ebs = []
redshift =[]
checks = ["Low Utilization Amazon EC2 Instances", "Idle Load Balancers", "Amazon RDS Idle DB Instances",
"Underutilized Amazon EBS Volumes",
"Underutilized Amazon Redshift Clusters"]
for i in response["checks"]:
if i['category'] == "cost_optimizing":
#print(i)
if i["name"] in checks:
client.refresh_trusted_advisor_check (checkId=i['id'])
refresh = None
while refresh != "success":
refresh_status = client.describe_trusted_advisor_check_refresh_statuses (
checkIds=[i['id']]
)
#print(refresh_status)
refresh = refresh_status["statuses"][0]["status"]
print(i['name'],refresh)
if refresh != "success":
print("Sleeping for 10 secs")
time.sleep (10)
results = client.describe_trusted_advisor_check_result (
checkId=i["id"],
language='en'
)
# print(results["result"].get("categorySpecificSummary",None).get("costOptimizing",None))
# monthly_savings = results["result"].get("categorySpecificSummary",None).get("costOptimizing",None).get("estimatedMonthlySavings",None)
# if monthly_savings:
# print (i["name"])
for data in results["result"]["flaggedResources"]:
details = data["metadata"]
if i["name"] == "Low Utilization Amazon EC2 Instances":
skip = False
#print(details[1],details[2],details[3],details[4],details[19],details[20],details[21])
for name in exceptions:
if details[2].startswith(name):
print(name,details[2].startswith(name),details[2])
skip = True
if not skip:
ec2.append([details[1],details[2],details[3],details[4],details[19],details[20],details[21],get_ec2_tags(details[1])])
elif i["name"] == "Idle Load Balancers":
#print(details[1],details[2],details[3])
if details[2] != "Low request count":
elb.append([details[1],details[2],details[3],get_elb_tags(details[1])])
elif i["name"] == "Amazon RDS Idle DB Instances":
#print(details[1],details[3],details[4],details[5],details[6])
rds.append([details[1],details[3],details[4],details[5],details[6],get_rds_tags(details[1])])
elif i["name"] == "Underutilized Amazon EBS Volumes":
#print(details[1],details[4],details[5])
if get_ebs_status(details[1]) == "available":
ebs.append([details[1],details[4],details[5],get_ebs_tags(details[1])])
elif i["name"] == "Underutilized Amazon Redshift Clusters":
#print (details[0], details[4], details[5])
if details[0] == "Yellow":
redshift.append ([details[2], details[3], details[4],details[5],get_redshift_tags(details[2])])
context = {
'ec2': ec2,
'elb': elb,
'ebs': ebs,
'rds': rds,
'redshift': redshift
}
# jinja template
THIS_DIR = os.path.dirname (os.path.abspath (__file__))
j2_env = Environment (loader=FileSystemLoader (THIS_DIR),
trim_blocks=True)
email_body = j2_env.get_template ('email.html').render (context)
print(email_body)
subject = "AWS Cleanup Report - DRYRUN"
sendmail (subject, email_body)
def lambda_handler(event, context):
main()
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment