Query S3 bucket size by cloudwatch get-metric-data

cloudwatch_s3_size.py

#!/usr/bin/env python3
# inspire by https://www.slsmk.com/getting-the-size-of-an-s3-bucket-using-boto3-for-aws/
import boto3
import datetime

def main():
    # Get all regions
    ec2 = boto3.setup_default_session(region_name='us-east-1')
    ec2 = boto3.client('ec2')
    desc_regions = ec2.describe_regions()

    # Header Line for the output going to standard out
    print('Region'.ljust(25)+'Bucket'.ljust(45)+ 'Storage Type'.ljust(30) + 'Size in Bytes'.rjust(25))

    # Get S3 bucket by region
    for region in desc_regions['Regions']:
        get_s3_bucket_size(region['RegionName'])

def get_s3_bucket_size(region='not_set'):
    now = datetime.datetime.now()
    cw = boto3.setup_default_session(region_name=region)
    cw = boto3.client('cloudwatch')

    # Get a list of all buckets
    allbuckets = cw.list_metrics(Namespace='AWS/S3',MetricName='BucketSizeBytes')

    # Iterate through each bucket
    for bucket in allbuckets['Metrics']:
        # For each bucket item, look up the cooresponding metrics from CloudWatch
        response = cw.get_metric_statistics(Namespace='AWS/S3',
                                            MetricName='BucketSizeBytes',
                                            Dimensions=bucket['Dimensions'],
                                            Statistics=['Average'],
                                            Period=3600,
                                            StartTime=(now-datetime.timedelta(days=1)).isoformat(),
                                            EndTime=now.isoformat()
                                            )
        bucket_name = bucket['Dimensions'][1]['Value']
        bucket_type = bucket['Dimensions'][0]['Value']
        # The cloudwatch metrics will have the single datapoint, so we just report on it.
        for item in response["Datapoints"]:
            print(region.ljust(25) + bucket_name.ljust(45) + bucket_type.ljust(30) + str("{:,}".format(int(item["Average"]))).rjust(25))
            # Note the use of "{:,}".format.
            # This is a new shorthand method to format output.
            # I just discovered it recently.


if __name__ == '__main__':
    main()

s3_query_size.json

[
  {
    "Id": "s3",
    "MetricStat": {
      "Metric": {
        "Namespace": "AWS/S3",
        "MetricName": "BucketSizeBytes",
        "Dimensions": [
          {
            "Name": "StorageType",
            "Value": "YOUR_STORAGE_TYPE"
          },
          {
            "Name": "BucketName",
            "Value": "YOUR_S3_BUCKET_NAME"
          }
        ]
      },
      "Period": 3600,
      "Stat": "Average",
      "Unit": "Bytes"
    },
    "ReturnData": true
  }
]