SonicTriton metrics analysis

getTimes.py

import sys,json,uuid
from collections import defaultdict

# keep lists on single line when json dumps w/ pretty settings
# from https://stackoverflow.com/questions/58736826/format-some-json-object-with-certain-fields-on-one-line
def replace(o):
    if isinstance(o, dict):
        replacements = []
        result = {}
        for key,val in o.iteritems():
            new_val, val_replacement = replace(val)
            result[key] = new_val
            replacements.extend(val_replacement)
        return result, replacements
    elif isinstance(o, list):
        replacement = uuid.uuid4().hex
        return replacement, [('"{}"'.format(replacement), str(o))]
    else:
        return o, []

inname = sys.argv[1]

client = defaultdict(list)
server = defaultdict(list)

ctoken = "() time: "
stoken = " usec"
with open(inname,'r') as infile:
    for line in infile:
        if ctoken in line:
            linesplit = line.rstrip().split(ctoken)
            client[linesplit[0]].append(int(linesplit[1]))
        elif stoken+" " in line:
            for char in ['(',')','+','  ']:
                line = line.replace(char,"")
            linesplit = line.rstrip().split(stoken)
            for item in linesplit:
                itemsplit = item.rsplit(' ',1)
                if len(itemsplit)==1: continue
                server[itemsplit[0]].append(int(itemsplit[1]))

output = {
    "client": client,
    "server": server,
}

# write out
output, replacements = replace(output)
result = json.dumps(output,sort_keys=True,indent=4)
for old, new in replacements:
    result = result.replace(old, new)
oname = inname.replace("log_","data_").replace(".log",".json")
with open(oname,'w') as ofile:
    ofile.write(result)

plotTimes.py

import sys,json,uuid
from collections import OrderedDict
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import numpy as np
from scipy.stats import sem
from cycler import cycler
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt

# hax (bypass "AttributeError: Unknown property fillstyle" when using cycler w/ errorbar)
def update_unknown(self, props):
    def _update_property(self, k, v):
        k = k.lower()
        if k in {'axes'}:
            return setattr(self, k, v)
        else:
            func = getattr(self, 'set_' + k, None)
            if not callable(func):
                return None
            return func(v)

    store = self.eventson
    self.eventson = False
    try:
        ret = [_update_property(self, k, v)
               for k, v in props.items()]
    finally:
        self.eventson = store

    if len(ret):
        self.pchanged()
        self.stale = True
    return ret
plt.Artist.update = update_unknown

parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-n", "--numer", dest="numer", type=str, required=True, help="numerator (JSON file)")
parser.add_argument("-d", "--denom", dest="denom", type=str, required=True, help="denominator (JSON file)")
parser.add_argument("-t", "--text", dest="text", type=str, default="", help="text info for plots")
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action="store_true", help="print verbose output")
parser.add_argument("-D", "--dir", dest="dir", type=str, default=".", help="directory for i/o")
parser.add_argument("-x", "--exclude", dest="exclude", type=int, default=0, help="exclude first N data points")
args = parser.parse_args()

qtys = {
    "client": ["acquire","dispatch","produce","total"],
    "server": ["overhead","queue","compute input","compute infer","compute output","total"],
}

def get_output(inname, verbose):
    with open(inname,'r') as infile:
        input = json.load(infile)
    
    # values and errors
    output = {
        "client": [[],[]],
        "server": [[],[]],
    }
    
    # basic analysis
    def print_avg(qty, vals, verbose):
        avg = np.average(vals)
        err = sem(vals)
        if verbose: print "\t{}: {:.1f} +/- {:.1f}".format(qty, avg, err)
        return avg,err
    if verbose: print inname
    for cat,val in input.iteritems():
        if verbose: print "{}:".format(cat)
        total = None
        for qty in qtys[cat]: # access in order
            if qty=="total":
                vals = total
            else:
                vals = val[qty][args.exclude:]
                if total is None: total = np.asarray(vals)
                else: total = np.add(total,vals)
            avg, err = print_avg(qty,vals,verbose)
            output[cat][0].append(avg)
            output[cat][1].append(err)
    
    return output

numer = args.numer.replace("data_test_","").replace(".json","")
denom = args.denom.replace("data_test_","").replace(".json","")
outputs = OrderedDict([
    (numer, get_output(args.dir+"/"+args.numer, args.verbose)),
    (denom, get_output(args.dir+"/"+args.denom, args.verbose)),
])
if args.verbose: print "ratio"

for cat,labels in qtys.iteritems():
    fig, ax = plt.subplots()
    ax.set_prop_cycle(
        cycler('color',['b','m']) +
        cycler('fillstyle',['full','none'])
    )
    for name, output in outputs.iteritems():
        ax.errorbar(range(len(labels)), output[cat][0], yerr=output[cat][1], marker='o', linestyle='none', label=name)
    ax.set_yscale('log')
    ax.set_ylabel(r'time [$\mu s$]')
    ax.set_xticks(range(len(labels)))
    ax.set_xticklabels(qtys[cat], rotation=20, ha='right')
    ax.set_title(args.text)
    ax.legend(loc='best')
    fig.savefig(args.dir+"/plotTimes_{}__{}_vs_{}.png".format(cat,numer,denom),**{"dpi":100})

    # make ratio
    ratio = np.divide(outputs.items()[0][1][cat][0], outputs.items()[1][1][cat][0])
    rfig, rax = plt.subplots()
    rax.plot(range(len(labels)), ratio, marker='o', color='k')
    rax.plot(range(len(labels)), [1]*len(labels), linestyle='--')
    rax.set_ylim([0,2])
    rax.set_ylabel('ratio ({} / {})'.format(numer,denom))
    rax.set_xticks(range(len(labels)))
    rax.set_xticklabels(labels, rotation=20, ha='right')
    rax.set_title(args.text)
    rfig.savefig(args.dir+"/plotTimes_{}__{}_vs_{}_ratio.png".format(cat,numer,denom),**{"dpi":100})

    if args.verbose:
        print "{}:".format(cat)
        for i in range(len(ratio)):
            print "\t{}: {:.3f}".format(labels[i],ratio[i])