Query historic usage metrics and export to CSV

historic_usage_to_csv.py

import datetime
import os
import time
import csv
import requests
import simplejson

from datadog import initialize, api

"""
This script is meant to pull historical usage metrics and export them to CSV.  Set your global variables below.
"""

# BEGIN GLOBAL VARIABLES

DD_API_KEY = os.environ.get('DD_API_KEY') or '<YOUR_API_KEY>'
DD_APP_KEY = os.environ.get('DD_APP_KEY') or '<YOUR_APP_KEY>'

START_HOUR = '2018-11-28T00'
END_HOUR = '2018-11-30T23'

# END GLOBAL VARIABLES

USAGE_URL = 'https://app.datadoghq.com/api/v1/usage/'
DD_KEYS = '?api_key=' + DD_API_KEY + '&application_key=' + DD_APP_KEY

options = {
    'api_key': DD_API_KEY,
    'app_key': DD_APP_KEY
}

initialize(**options)

def get_usage_metrics(url):
    usage_metrics = []
    error_messages = []
    try:
        metrics = requests.get(url).json()
        usage_metrics = metrics.get('usage', None)
        error_messages = metrics.get('errors', [])
        for m in error_messages:
            print('Error when retrieving metrics: {}'.format(m))
    except requests.exceptions.MissingSchema:
        print('Invalid URL format: {}'.format(url))
    except requests.exceptions.ConnectionError:
        print('Could not connect to url: {}'.format(url))
    except simplejson.scanner.JSONDecodeError:
        print('The response did not contain JSON data')
    return usage_metrics

def update_standard_metrics(endpoint):
    # Build url
    filename = 'datadog_usage_' + endpoint + '.csv'
    url = USAGE_URL + endpoint + DD_KEYS + '&start_hr=' + START_HOUR + '&end_hr=' + END_HOUR
    # Get usage metrics from Datadog
    metrics = get_usage_metrics(url)
    print(metrics)
    with open(filename, mode='a+') as output_file:
        metric_writer = csv.writer(output_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        metric_writer.writerow(['hour', 'count'])
        for metric in metrics:
            usage_hour = metric.get('hour', False)
            if endpoint == 'hosts':
                usage_number = metric.get('host_count', 0)
            else:
                usage_number = metric.get('num_custom_timeseries', 0)
            if usage_hour:
                metric_writer.writerow([usage_hour, usage_number])

class MainClass(object):

    def __init__(self, *args, **kwargs):
        pass

    @classmethod
    def main(self):
        # host data
        update_standard_metrics('hosts')
        update_standard_metrics('timeseries')

MainClass().main()