perf_trials.py

#!/usr/bin/env python

# https://gist.github.com/4438441
import os

from coverage_model import *
import numpy as np
import time
import objgraph
import gevent
from clint.textui import progress
from guppy import hpy
import gc
import time

def _build_coverage(num_params=1, value_encoding='float32', brick_size=1000, in_memory=False, verbose=False):
    if verbose:
        print '>> Build Coverage: num_params=%s  value_encoding=%s  brick_size=%s  in_memory=%s' % (num_params, value_encoding, brick_size, in_memory)
    # Instantiate a ParameterDictionary
    pdict = ParameterDictionary()

    # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
    t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=value_encoding))
    t_ctxt.axis = AxisTypeEnum.TIME
    t_ctxt.uom = 'seconds since 01-01-1970'
    pdict.add_context(t_ctxt)

    # Construct temporal and spatial Coordinate Reference System objects
    tcrs = CRS([AxisTypeEnum.TIME])
    scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])

    # Construct temporal and spatial Domain objects
    tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
    sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)

    if num_params > 1:
        for x in range(num_params - 1):
            pdict.add_context(ParameterContext(str(x), param_type=QuantityType(value_encoding=value_encoding)))

    # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
    bricking_scheme = {'brick_size':brick_size,'chunk_size':500}

    return SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, bricking_scheme=bricking_scheme, in_memory_storage=in_memory)

def insert_values(cov=None, num_iters=100, insert_size=10, verbose=False):
    if cov is None:
        cov = _build_coverage()

    ori = cov.num_timesteps
    iover = make_iterator(xrange(num_iters), verbose)
    if verbose: print ">>>>> Insert Loop: insert_size=%s <<<<<" % insert_size

    for x in iover:
        cov.insert_timesteps(insert_size)
        for p in cov.list_parameters():
            cov.set_parameter_values(p, np.arange(ori,ori+insert_size), slice(ori,ori+insert_size))
        ori += insert_size
#        time.sleep(0.1)

    return cov

def _print_heap(heap, msg='', verbose=False):
    print ">>>> %s: %s" % (msg, heap.size)
    if verbose:
        print heap

def make_iterator(rng, use_bar=False):
    if use_bar:
        return progress.bar(rng)
    else:
        return rng

def create_looped_insert():
    t_hp = hpy()
    p_hp = hpy()

    verbose = False

    if verbose:
        print ">> Waiting 5 seconds..."
    gevent.sleep(5)

    if verbose:
        print ">> Reset Total Heap"
    t_hp.setrelheap()
    _print_heap(t_hp.heap(), 'Initial Total Heap')

    # Coverage Creation
    if verbose:
        print ">> Reset Partial Heap"
    p_hp.setrelheap()
    if verbose:
        print ">> Build Coverage"
    cov = _build_coverage()
    ph=p_hp.heap()
    _print_heap(p_hp.heap(), 'Partial Heap after coverage creation')
    th = t_hp.heap()
    _print_heap(t_hp.heap(), 'Total Heap after coverage creation')

    # Value Insertion
    if verbose:
        print ">> Reset Partial Heap"
    p_hp.setrelheap()
    if verbose:
        print ">> Insert Values"
    insert_values(cov=cov)
    _print_heap(p_hp.heap(), 'Partial Heap after value insertion')
    _print_heap(t_hp.heap(), 'Total Heap after value insertion')

    ppath = cov.persistence_dir
    for i in xrange(10):
        if verbose:
            print ">> Close coverage \'%s\'" % ppath
        cov.close()

        if verbose:
            print ">> Load coverage \'%s\'" % ppath
            print ">> Reset Partial Heap"
        p_hp.setrelheap()
        cov = SimplexCoverage.load(ppath)
        _print_heap(p_hp.heap(), 'Partial Heap after load')

        # Value Insertion
        if verbose:
            print ">> Reset Partial Heap"
        p_hp.setrelheap()
        if verbose:
            print ">> Insert Values"
        insert_values(cov=cov)
        _print_heap(p_hp.heap(), 'Partial Heap after value insertion')
        _print_heap(t_hp.heap(), 'Total Heap after value insertion')

    #        gevent.sleep(2)

    if verbose:
        print ">> Close coverage \'%s\'" % ppath
    cov.close()

    _print_heap(t_hp.heap(), 'Total Heap')

def insertion_stats(verbose=False):
    hp = hpy()
    cov=_build_coverage() # Throwaway - removes 1st initialization overhead
    insert_values(cov=cov)
    cov.close()
    cov=None
    gc.collect()

    headers = ['Insert Size','Create Heap (bytes)','Insertion Heap (bytes)','Create Time (s)','Insertion Time (s)','Close Time (s)']
    stats = StatsWriter('perf_stats/insertion_stats.csv', headers)
    insert_sizes = [1,5,10,50,100,500,1000]
    for i in make_iterator(insert_sizes, not verbose): # 1-20 parameters
        ni = 10000 / i
        hp.setrelheap()
        bts=time.time()
        cov = _build_coverage(num_params=1, brick_size=1000)
        bt=time.time()-bts
        bld_size = hp.heap().size
        hp.setrelheap()
        its=time.time()
        insert_values(cov=cov, num_iters=ni, insert_size=i)
        it=time.time()-its

        cts=time.time()
        cov.close()
        ct=time.time()-cts
        stats.append((i, bld_size, hp.heap().size, bt, it, ct))

    return headers, stats

def brick_stats(verbose=False):
    hp = hpy()
    cov=_build_coverage() # Throwaway - removes 1st initialization overhead
    insert_values(cov=cov)
    cov.close()
    cov=None
    gc.collect()

    headers = ['Brick Size','Create Heap (bytes)','Insertion Heap (bytes)','Create Time (s)','Insertion Time (s)','Close Time (s)']
    stats = StatsWriter('perf_stats/brick_stats.csv', headers)
    brick_sizes = [100, 500, 1000, 5000, 10000, 50000, 100000]
    for p in progress.bar(brick_sizes, not verbose): # 1-20 parameters
        hp.setrelheap()
        bts=time.time()
        cov = _build_coverage(num_params=10, brick_size=p)
        bt=time.time()-bts
        bld_size = hp.heap().size
        hp.setrelheap()
        its=time.time()
        insert_values(cov=cov, num_iters=10, insert_size=1000)
        it=time.time()-its

        cts=time.time()
        cov.close()
        ct=time.time()-cts
        stats.append((p, bld_size, hp.heap().size, bt, it, ct))

    return headers, stats

def param_stats(verbose=False):
    hp = hpy()
    cov=_build_coverage() # Throwaway - removes 1st initialization overhead
    insert_values(cov=cov)
    cov.close()
    cov=None
    gc.collect()

    headers = ['# Params','Create Heap (bytes)','Insertion Heap (bytes)','Create Time (s)','Insertion Time (s)','Close Time (s)']
    stats = StatsWriter('perf_stats/param_stats.csv', headers)
    for p in progress.bar(xrange(1,21), not verbose): # 1-20 parameters
        hp.setrelheap()
        bts=time.time()
        cov = _build_coverage(num_params=p,brick_size=1000)
        bt=time.time()-bts
        bld_size = hp.heap().size
        hp.setrelheap()
        its=time.time()
        insert_values(cov=cov, num_iters=10, insert_size=100)
        it=time.time()-its

        cts=time.time()
        cov.close()
        ct=time.time()-cts
        stats.append((p, bld_size, hp.heap().size, bt, it, ct))

    return headers, stats

def creation_stats(verbose=False):
    hp = hpy()
    cov=_build_coverage() # Throwaway - removes 1st initialization overhead
    insert_values(cov=cov)
    cov.close()
    cov=None
    gc.collect()

    headers = ['# Params','Create Heap (bytes)','Create Time (s)','Close Time (s)']
    stats = StatsWriter('perf_stats/creation_stats.csv', headers)
    for p in make_iterator(xrange(1,21), not verbose): # 1-20 parameters
        hp.setrelheap()
        bts=time.time()
        cov = _build_coverage(num_params=p,brick_size=1000)
        bt=time.time()-bts
        bld_size = hp.heap().size

        cts=time.time()
        cov.close()
        ct=time.time()-cts
        stats.append((p, bld_size, bt, ct))

    return headers, stats

def growth_stats(verbose=False):
    total_heap = hpy()
    insert_heap = hpy()
    total_insert_heap = hpy()
    cov=_build_coverage()
    insert_values(cov=cov)
    cov.close()
    cov=None
    gc.collect()

    headers=['# Params', 'Brick Size', 'Total Heap (bytes)', 'Total Insert Heap (bytes)', 'Average Insert Heap (bytes)']
    stats=StatsWriter('perf_stats/growth_stats.csv', headers)
    for p in make_iterator(xrange(1,21,2), not verbose):
        for b in make_iterator(xrange(1000, 11000, 1000), not verbose):
            isize=0
            total_heap.setrelheap()
            cov=_build_coverage(num_params=p, brick_size=b)
            total_insert_heap.setrelheap()
            for x in xrange(0, 10000, 5000):
                insert_heap.setrelheap()
                insert_values(cov=cov, num_iters=1, insert_size=5000)
                isize += insert_heap.heap().size
            isize /= 2
            stats.append((p, b, total_heap.heap().size, total_insert_heap.heap().size, isize))
            cov.close()

    return headers, stats

def multi_insert_stats(verbose=False, do_close=False):
    total_heap = hpy()
    per_coverage_heap = hpy()
    cov=_build_coverage()
    insert_values(cov=cov)
    cov.close()
    cov=None
    gc.collect()

    headers=['# Params', 'Coverage #', 'Coverage Heap (bytes)', 'Total Heap (bytes)']
    stats = StatsWriter('perf_stats/multi_insert_stats_5k_bs5k{0}.csv'.format('_close' if do_close else ''), headers)
    total_heap.setrelheap()
    covs = []
    ci=-1
    for p in make_iterator(xrange(1, 21, 2), not verbose):
        if verbose: print '# Params: %s' % p
        for c in make_iterator(xrange(10), not verbose):
            if verbose: print 'Coverage %s' % c
            per_coverage_heap.setrelheap()
            if verbose: print 'make new coverage'
            covs.append(_build_coverage(num_params=p, brick_size=5000))
            if verbose: print 'insert values'
            insert_values(cov=covs[-1], num_iters=1, insert_size=5000)
            if do_close:
                if verbose: print 'close coverage'

                covs[-1].close()
                covs[-1] = None
                covs.pop(-1)

            stats.append((p, c, per_coverage_heap.heap().size, total_heap.heap().size))

            if verbose: print

        if verbose: print

    return headers, stats

def single_insert_stats(verbose=False, do_close=False):
    total_heap = hpy()
    per_insertion_heap = hpy()
    cov=_build_coverage()
    insert_values(cov=cov)
    cov.close()
    cov=None
    gc.collect()

    headers=['# Params', 'Repetition #', 'Insertion Heap (bytes)', 'Total Heap (bytes)', 'Total Real Memory (bytes)']
    stats = StatsWriter('perf_stats/single_insert_stats_5k_bs5k{0}.csv'.format('_close' if do_close else ''), headers)
    total_heap.setrelheap()
    for p in make_iterator(xrange(1, 21, 2), not verbose):
        if verbose: print '# Params: %s' % p
        if cov is not None:
            cov.close()
            cov = None
        perdir = None
        for i in make_iterator(xrange(10), not verbose):
            if verbose: print 'Insertion %s' % i
            if perdir is None and cov is None:
                if verbose: print 'make new coverage'
                cov = _build_coverage(num_params=p, brick_size=5000)
                perdir = cov.persistence_dir
            elif cov is None:
                if verbose: print 'load coverage'
                cov = SimplexCoverage.load(perdir)
            else:
                if verbose: print 'coverage already open'

            per_insertion_heap.setrelheap()

            if verbose: print 'insert values'
            insert_values(cov=cov, num_iters=1, insert_size=5000)

            if do_close:
                if verbose: print 'close coverage'

                cov.close()
                cov = None
                gc.collect()

            stats.append((p, i, per_insertion_heap.heap().size, total_heap.heap().size))

            if verbose: print

        if verbose: print

    return headers, stats

def ruse_single_insert_stats(verbose=False, do_close=False):
    time.sleep(5)
    import resource
#    total_heap = hpy()
#    per_insertion_heap = hpy()
    cov=_build_coverage()
    insert_values(cov=cov)
    cov.close()
    cov=None
    gc.collect()

#    headers=['# Params', 'Repetition #', 'Insertion Heap (bytes)', 'Total Heap (bytes)', 'Total Real Memory (bytes)']
    headers=['# Params', 'Repetition #', 'time', 'ru_utime', 'ru_stime', 'ru_maxrss', 'ru_ixrss', 'ru_idrss', 'ru_isrss', 'ru_minflt', 'ru_majflt', 'ru_nswap', 'ru_inblock', 'ru_oublock', 'ru_msgsnd', 'ru_msgrcv', 'ru_nsignals', 'ru_nvcsw', 'ru_nivcsw']
    stats = StatsWriter('perf_stats/ruse_single_insert_stats_5k_bs5k{0}.csv'.format('_close' if do_close else ''), headers)
#    total_heap.setrelheap()
    for p in make_iterator(xrange(1, 21, 2), not verbose):
        if verbose: print '# Params: %s' % p
        if cov is not None:
            cov.close()
            cov = None
        perdir = None
        for i in make_iterator(xrange(10), not verbose):
            if verbose: print 'Insertion %s' % i
            if perdir is None and cov is None:
                if verbose: print 'make new coverage'
                cov = _build_coverage(num_params=p, brick_size=5000)
                perdir = cov.persistence_dir
            elif cov is None:
                if verbose: print 'load coverage'
                cov = SimplexCoverage.load(perdir)
            else:
                if verbose: print 'coverage already open'

#            per_insertion_heap.setrelheap()

            if verbose: print 'insert values'
            insert_values(cov=cov, num_iters=1, insert_size=5000)

            if do_close:
                if verbose: print 'close coverage'

                cov.close()
                cov = None
                gc.collect()

            stats.append([p, i, time.time()]+list(resource.getrusage(resource.RUSAGE_SELF)))
#            stats.append((p, i, per_insertion_heap.heap().size, total_heap.heap().size))

            if verbose: print

        if verbose: print

    return headers, stats

import csv
class StatsWriter(object):

    def __init__(self, fout, headers):
        self.fout = fout
        self.headers = headers

        self._write_headers()

    def _write_headers(self):
        with open(self.fout, 'a') as f:
            writer = csv.writer(f)
            writer.writerow(self.headers)

    def append(self, stats):
        with open(self.fout, 'a') as f:
            writer = csv.writer(f)
            writer.writerow(stats)


def _print_stats(headers, stat_table):
    if isinstance(stat_table, StatsWriter):
        print 'Statistics written to \'%s\'' % stat_table.fout
        return

    print ''.join(['%s\t' % x for x in headers])
    for r in stat_table:
        print ''.join(['%s\t' % x for x in r])

if __name__ == '__main__':
#    create_looped_insert()

    print 'Process ID:  %s' % os.getpid()

    from argparse import ArgumentParser
    parser = ArgumentParser(description="Memory statistics for various aspects of the Coverage Model")
    parser.add_argument('-v', '--verbose', help="Use verbose output", action="store_true")
    parser.add_argument('-c', '--create', help="Creation stats", action="store_true")
    parser.add_argument('-p', '--param',  help="Parameter stats", action="store_true")
    parser.add_argument('-b', '--brick', help="Bricking stats", action="store_true")
    parser.add_argument('-i', '--insert', help="Insertion stats", action="store_true")
    parser.add_argument('-g', '--growth', help="Growth Stats", action="store_true")
    parser.add_argument('-mi', '--multiinsert', help="Open & insert multiple coverages", action="store_true")
    parser.add_argument('-mic', '--multiinsertclose', help="Open, insert & close multiple coverages", action="store_true")
    parser.add_argument('-si', '--singleinsert', help="Open & insert one coverage multiple times", action="store_true")
    parser.add_argument('-sic', '--singleinsertclose', help="Open, insert & close one coverage multiple times", action="store_true")

    parser.add_argument('-rsi', '--rusesingleinsert', help="Open & insert one coverage multiple times", action="store_true")
    parser.add_argument('-rsic', '--rusesingleinsertclose', help="Open, insert & close one coverage multiple times", action="store_true")

    args = parser.parse_args()

    if args.create:
        _print_stats(*creation_stats(args.verbose))

    if args.param:
        _print_stats(*param_stats(args.verbose))

    if args.brick:
        _print_stats(*brick_stats(args.verbose))

    if args.insert:
        _print_stats(*insertion_stats(args.verbose))

    if args.growth:
        _print_stats(*growth_stats(args.verbose))

    if args.multiinsert:
        _print_stats(*multi_insert_stats(args.verbose))

    if args.multiinsertclose:
        _print_stats(*multi_insert_stats(args.verbose, True))

    if args.singleinsert:
        _print_stats(*single_insert_stats(args.verbose))

    if args.singleinsertclose:
        _print_stats(*single_insert_stats(args.verbose, True))

    if args.rusesingleinsert:
        _print_stats(*ruse_single_insert_stats(args.verbose))

    if args.rusesingleinsertclose:
        _print_stats(*ruse_single_insert_stats(args.verbose, True))