gistfile1.py

#!/usr/bin/python

import requests, urllib2, simplejson, datetime, time

# Constants
MAX_RETRIES = 15       #Max number of retries when requesting data
TIMEOUT_RETURN = 100   #Timeout in calling API to retrieve results
DELAY        = 0.5     #Delay per retry in seconds (increases * no_retries)
MAX_FIELDS = 300       #Max fields to retrieve from field request via API

# Configuration parameters
# Need to change these to your case, as well as remove decorator from logQueryData() once tested
T = 1000 # ms
pivot_field = 'json.response_time'
my_token = '3dc53bxe-4d3d-4a76-w515-c277dic4uc6c'
username = 'johndoe'
password = '123456'
subdomain = 'acme'
time_interval = '15m'

# Decorator function to print parameter names sent to function
# From: http://stackoverflow.com/questions/6200270/
# Used to test before sending data to Loggly
def dump_args(func):
    "This decorator dumps out the arguments passed to a function before calling it"
    argnames = func.func_code.co_varnames[:func.func_code.co_argcount]
    fname = func.func_name
    def echo_func(*args,**kwargs):
        print fname, "(", ', '.join(
            '%s=%r' % entry
            for entry in zip(argnames,args[:len(argnames)])+
                             [("args",list(args[len(argnames):]))]+[("kwargs",kwargs)]) +")"
    return echo_func


def getApdex(id, query, pivot_field, percentages, searchFrom, searchTo, timestamp,
              subdomain, deployment, username, password, output_key):
    """ Calculates Apdex score and stores result back into Loggly
        id: a label for the log, for instance, indicating how often the data is fetched
        query: dict with queries to generate numerator, denominator, etc. for alculation
        percentages: dict with definitions of how metrics are calculated
        -- all other terms are self explanatory --
    """

    results = {}

    #Iterate over queries and get results
    for key in query.iterkeys():

        search_url = "http://%s.%s/apiv2/fields/%s?q=%s&from=%s&until=%s&facet_size=%d" % (
                      subdomain, deployment,pivot_field, query[key],searchFrom,searchTo, MAX_FIELDS)

        print search_url

        status, results[key] =  getAPI(search_url, username, password)

    print results

    # Get last term in pivot field to use as key in log event
    pivot = pivot_field.split('.')[-1] + '_perf'

    Rs = getApdexPercent(results, percentages, 'Rs')
    Rt = getApdexPercent(results, percentages, 'Rt') - Rs

    apdex = Rs + Rt / 2.0

    metrics_dict = dict( zip( [ 'Rs','Rt','apdex'],
                              [ round(Rs,1), round(Rt,1), round(apdex,1)] ))

    #Send data to your data store 
    # (Put your own code here)
    
    #Send values back to Loggly
    logQueryData(output_key,dict([('timestamp',str(timestamp.isoformat()) ), ('action','analytics'),
                                    ('bin',id), (pivot,metrics_dict) ] ))

    return 0

def getApdexPercent(results, percentages, key):
        percentage = 0.
        numerator = results[percentages[key]['numerator']]
        denominator = results[percentages[key]['denominator']]

        if denominator > 0:
            percentage = round( float(numerator) / float(denominator) * 100., 2)

        return percentage

# Remove this decorator after testing in order to send data to your Loggly account
@dump_args
def logQueryData(loggly_key, data):

    """ Logs a JSON object to Loggly """
    log_data = "PLAINTEXT=" + urllib2.quote(simplejson.dumps(data))

    # Send log data to Loggly
    urllib2.urlopen("https://logs-01.loggly.com/inputs/" + loggly_key + "/tag/queryAPI/", log_data)

def getAPI(search_url, username, password):
    """ Fetches data from Loggly via the API """
    
    retries = 0
    while retries < MAX_RETRIES:
 
        try:
            # We launch the search
            r = requests.get(search_url, auth=(username, password), timeout=TIMEOUT_RETURN)
            res = r.json()
            try:
                if res['total_events']  >= 0:
                    return r.status_code, res['total_events']
                else:
                    return r.status_code, 0
 
            except ValueError as error:
                print '%s Error: %s - response: %s\n' % ( datetime.datetime.utcnow(), error, r )
 
            break
 
        except ValueError:
            print '%s Error status code: %d\n' % ( datetime.datetime.utcnow(), r.status_code )
            retries = retries + 1
 
        time.sleep(DELAY * (retries+1) )
 
    return 0

########## Main Code #############


# The query terms have the T and Y values
query_terms = {
               'A': '%s:*' % pivot_field,
               'B': '%s:<%d' % (pivot_field, T),
               'C': '%s:<%d' % (pivot_field, 4*T)
              }

# This dictionary structure can be used for any type of relational metric based on counts
percentages = {'Rs':{
                   'numerator':'B',
                   'denominator':'A'
                  },
               'Rt':{
                   'numerator':'C',
                   'denominator':'A'
                  }
              }
now = datetime.datetime.utcnow()

getApdex(time_interval, query_terms, pivot_field, percentages, '-'+time_interval, 'now', now,
              subdomain, 'loggly.com', username, password, my_token)