statslogger.py

#!/usr/bin/env python
 
import httplib
import json
from datetime import datetime, date, time
from time import sleep
import os.path
from multiprocessing import Process
 
 
def write_stats_to_file(directory, stats, fields):
    data = format_line(stats, fields)
    statsfile = open_file(directory, fields)
    statsfile.write(format_line(stats, fields))
    statsfile.close()
 
def open_file(directory, fields):
    file_path = directory + "/" + "stats_" + datetime.today().strftime("%Y%m%d") + ".csv"
    if os.path.isfile(file_path):
        return open(file_path, "a")
    else:
        outfile = open(file_path, "a")
        outfile.write(format_header(fields))
        return outfile
 
def format_header(fields):
    return '"timestamp","' + '","'.join(fields) + '"\n'
 
def format_line(stats, fields):
    stats_list = [get_statistic(stats, statistic) for statistic in fields]
    return '"' + datetime.today().strftime("%Y-%m-%d %H:%M:%S") + '",' + ','.join(stats_list) + '\n'
 
def get_statistic(stats, name):
    try:
        return str(stats[name])
    except:
    	return '"-"'
 
fields = ["index_fsm_active",
          "index_fsm_create",
          "list_fsm_active",
          "list_fsm_create",
          "node_get_fsm_active",
          "node_get_fsm_counter_objsize_100",
          "node_get_fsm_counter_objsize_mean",
          "node_get_fsm_counter_objsize_median",
          "node_get_fsm_counter_time_100",
          "node_get_fsm_counter_time_mean",
          "node_get_fsm_counter_time_median",
          "node_get_fsm_map_objsize_100",
          "node_get_fsm_map_objsize_mean",
          "node_get_fsm_map_objsize_median",
          "node_get_fsm_map_time_100",
          "node_get_fsm_map_time_mean",
          "node_get_fsm_map_time_median",
          "node_get_fsm_objsize_100",
          "node_get_fsm_objsize_mean",
          "node_get_fsm_objsize_median",
          "node_get_fsm_rejected",
          "node_get_fsm_set_objsize_100",
          "node_get_fsm_set_objsize_mean",
          "node_get_fsm_set_objsize_median",
          "node_get_fsm_set_time_100",
          "node_get_fsm_set_time_mean",
          "node_get_fsm_set_time_median",
          "node_get_fsm_time_100",
          "node_get_fsm_time_mean",
          "node_get_fsm_time_median",
          "node_gets",
          "node_gets_counter",
          "node_gets_map",
          "node_gets_set",
          "node_put_fsm_active",
          "node_put_fsm_counter_time_100",
          "node_put_fsm_counter_time_mean",
          "node_put_fsm_counter_time_median",
          "node_put_fsm_map_time_100",
          "node_put_fsm_map_time_mean",
          "node_put_fsm_map_time_median",
          "node_put_fsm_rejected",
          "node_put_fsm_set_time_100",
          "node_put_fsm_set_time_mean",
          "node_put_fsm_set_time_median",
          "node_put_fsm_time_100",
          "node_put_fsm_time_mean",
          "node_put_fsm_time_median",
          "node_puts",
          "node_puts_counter",
          "node_puts_map",
          "node_puts_set",
          "pbc_active",
          "pbc_connects",
          "pipeline_active",
          "pipeline_create_one",
          "search_index_fail_one",
          "search_index_latency_max",
          "search_index_latency_median",
          "search_index_throughtput_one",
          "search_query_fail_one",
          "search_query_latency_max",
          "search_query_latency_median",
          "search_query_throughput_one",
          "vnode_counter_update",
          "vnode_gets",
          "vnode_map_update",
          "vnode_puts",
          "vnode_set_update"]
 
def run():
    #signal.signal(signal.SIGINT, signal_handler)
    while(True):
         try:
             conn = httplib.HTTPConnection('localhost', 8098)
             conn.connect()
             conn.request('GET', '/stats')
             response = conn.getresponse()
             data = response.read()
             conn.close()
             stats = json.loads(data)
             write_stats_to_file(".", stats, fields)
         except:
             stats = {}
         sleep(60)
 
p = Process(target=run,args=())
 
import signal
import sys
def signal_handler(signal, frame):
    try:
        p.terminate()
        p.join()
    except:
        pass
    sys.exit(0)
 
signal.signal(signal.SIGINT, signal_handler)
p.daemon = False
p.start()