Skip to content

Instantly share code, notes, and snippets.

@blazetopher

blazetopher/memory_trials.py

Last active December 17, 2015 08:08
Show Gist options
  • Save blazetopher/5577554 to your computer and use it in GitHub Desktop.
Save blazetopher/5577554 to your computer and use it in GitHub Desktop.
Download ZIP
Trying to track down the apparent memory leak in the coverage model
Raw
memory_trials.py
#!/usr/bin/env python
"""
@package
@file memory_trials.py
@author Christopher Mueller
@brief
"""
from coverage_model import *
import numpy as np
import os
import resource
def make_cov(root_dir, params, nt=10, data_dict=None, make_temporal=True, in_memory=False, val_cache=True):
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
if isinstance(params, ParameterDictionary):
pdict = params
else:
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
if make_temporal:
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt, is_temporal=True)
for p in params:
if isinstance(p, ParameterContext):
pdict.add_context(p)
elif isinstance(params, tuple):
pdict.add_context(ParameterContext(p[0], param_type=QuantityType(value_encoding=np.dtype(p[1]))))
else:
pdict.add_context(ParameterContext(p, param_type=QuantityType(value_encoding=np.dtype('float32'))))
scov = SimplexCoverage(root_dir, create_guid(), 'sample coverage_model', parameter_dictionary=pdict,
temporal_domain=tdom, spatial_domain=sdom,
in_memory_storage=in_memory, value_caching=val_cache)
if nt != 0:
scov.insert_timesteps(nt)
for p in scov.list_parameters():
if data_dict is not None and p in data_dict:
dat = data_dict[p]
else:
dat = range(nt)
try:
scov.set_parameter_values(p, dat)
except Exception as ex:
import sys
raise Exception('Error setting values for {0}: {1}'.format(p, data_dict[p])), None, sys.exc_traceback
scov.close()
return os.path.realpath(scov.persistence_dir) if scov._in_memory_storage is False else 'in-memory'
def load_cov(cov_path):
return AbstractCoverage.load(cov_path)
def load_and_read(cov_path, read_reps=1):
cov = load_cov(cov_path)
for r in range(read_reps):
[cov.get_parameter_values(p) for p in cov.list_parameters()]
def get_mem():
return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024.0**2)
def run_many_params(cov_pth=None, root='test_data'):
import gc
import gevent
print 'Many Params Trial:'
params = ['p%i'% i for i in xrange(100)]
ntimes = 3600
vals = range(ntimes)
if cov_pth is None:
cov_pth = make_cov(root, params, nt=ntimes, data_dict={p: vals for p in params})
print 'Cov path:', cov_pth
cov = AbstractCoverage.load(cov_pth, mode='r')
print 'Cov has %i parameters and %i timesteps' % (len(cov.list_parameters()), cov.num_timesteps)
cov.close()
for x in xrange(10):
s = get_mem()
cov = AbstractCoverage.load(cov_pth, mode='r')
e = get_mem()
print e-s, 'mb'
cov.close()
gc.collect()
gevent.sleep(0.2)
return cov_pth
def run_creation_trial(params=None, ntimes=None, ncovs=None, inmem=False, valcache=True):
print 'Creation Trial:'
params = params or ['a', 'b']
ntimes = ntimes or [10, 100, 1000, 10000, 100000]
ncovs = ncovs or [10, 100, 200]
data_dicts = {t: {k: range(t) for k in params} for t in ntimes}
# Throwaway cov - sucks up 'first use' memory
make_cov('test_data', params, nt=data_dicts.keys()[0], data_dict=data_dicts[data_dicts.keys()[0]])
print 'num_times\tnum_covs\tmem_inc_mb'
for nt in ntimes:
for nc in ncovs:
smem = get_mem()
covs = []
for x in range(nc):
make_cov('test_data', params, nt=nt, data_dict=data_dicts[nt], in_memory=inmem, val_cache=valcache)
emem = get_mem()
print '%s\t%s\t%s' % (nt, nc, emem - smem)
def run_load_trial(cov_path=None, params=None, ntimes=None, ncovs=None, inmem=False, valcache=True):
print 'Load Trial:'
params = params or ['a', 'b']
ntimes = ntimes or [10, 100, 1000, 10000, 100000]
ncovs = ncovs or [10, 100, 200]
data_dicts = {t: {k: range(t) for k in params} for t in ntimes}
# Throwaway cov - sucks up 'first use' memory
make_cov('test_data', params, nt=data_dicts.keys()[0], data_dict=data_dicts[data_dicts.keys()[0]])
domake = True
if cov_path is None:
domake = True
print 'num_times\tnum_covs\tmem_inc_mb'
for nt in ntimes:
if domake:
cov_path = make_cov('test_data', params, nt=nt, data_dict=data_dicts[nt], in_memory=inmem, val_cache=valcache)
for nc in ncovs:
smem = get_mem()
for x in range(nc):
load_cov(cov_path)
emem = get_mem()
print '%s\t%s\t%s' % (nt, nc, emem - smem)
def run_read_trial(cov_path=None, params=None, ntimes=None, ncovs=None, inmem=False, valcache=True):
print 'Read Trial:'
params = params or ['a', 'b']
ntimes = ntimes or [10, 100, 1000, 10000, 100000]
ncovs = ncovs or [10, 100, 200]
data_dicts = {t: {k: range(t) for k in params} for t in ntimes}
# Throwaway cov - sucks up 'first use' memory
make_cov('test_data', params, nt=data_dicts.keys()[0], data_dict=data_dicts[data_dicts.keys()[0]])
domake = True
if cov_path is None:
domake = True
print 'num_times\tnum_covs\tmem_inc_mb'
for nt in ntimes:
if domake:
cov_path = make_cov('test_data', params, nt=nt, data_dict=data_dicts[nt], in_memory=inmem, val_cache=valcache)
cov = load_cov(cov_path)
for nc in ncovs:
smem = get_mem()
for x in range(nc):
for p in cov.list_parameters():
cov.get_parameter_values(p)
emem = get_mem()
print '%s\t%s\t%s' % (nt, nc, emem - smem)
if __name__ == '__main__':
from argparse import ArgumentParser
parser = ArgumentParser(description="Description of script")
parser.add_argument('-a', '--arg', help="Help text", action="store_true")
# Add additional arguments
args = parser.parse_args()
run_creation_trial()
Raw
sample_output
><> from scratch import memory_trials
><> memory_trials.run_creation_trial();memory_trials.run_load_trial();memory_trials.run_read_trial()
Creation Trial:
num_times num_covs mem_inc_mb
10 10 0.58984375
10 100 6.55859375
10 200 13.19921875
100 10 0.69140625
100 100 6.73828125
100 200 13.37890625
1000 10 0.76171875
1000 100 6.77734375
1000 200 13.515625
10000 10 1.55078125
10000 100 6.8828125
10000 200 13.5
100000 10 4.09375
100000 100 6.58203125
100000 200 14.5
Load Trial:
num_times num_covs mem_inc_mb
10 10 0.30078125
10 100 0.1015625
10 200 0.0234375
100 10 0.0078125
100 100 0.02734375
100 200 0.04296875
1000 10 0.01171875
1000 100 0.01953125
1000 200 0.01171875
10000 10 0.01171875
10000 100 0.00390625
10000 200 0.015625
100000 10 0.03125
100000 100 0.01171875
100000 200 0.0
Read Trial:
num_times num_covs mem_inc_mb
10 10 0.04296875
10 100 0.0
10 200 0.0
100 10 0.0
100 100 0.0
100 200 0.0
1000 10 0.01171875
1000 100 0.0
1000 200 0.0
10000 10 0.0
10000 100 0.0
10000 200 0.0
100000 10 1.1484375
100000 100 0.0
100000 200 0.0
><> memory_trials.run_creation_trial();memory_trials.run_load_trial();memory_trials.run_read_trial()
Creation Trial:
num_times num_covs mem_inc_mb
10 10 0.0
10 100 0.0
10 200 1.765625
100 10 0.0
100 100 0.203125
100 200 0.7265625
1000 10 0.0
1000 100 0.0
1000 200 0.0
10000 10 0.0
10000 100 0.0
10000 200 0.0
100000 10 0.0
100000 100 0.0
100000 200 0.09375
Load Trial:
num_times num_covs mem_inc_mb
10 10 0.0
10 100 0.0
10 200 0.0
100 10 0.0
100 100 0.0
100 200 0.0
1000 10 0.0
1000 100 0.0
1000 200 0.0
10000 10 0.0
10000 100 0.0
10000 200 0.0
100000 10 0.0
100000 100 0.0
100000 200 0.0
Read Trial:
num_times num_covs mem_inc_mb
10 10 0.0
10 100 0.0
10 200 0.0
100 10 0.0
100 100 0.0
100 200 0.0
1000 10 0.0
1000 100 0.0
1000 200 0.0
10000 10 0.0
10000 100 0.0
10000 200 0.0
100000 10 0.3828125
100000 100 0.0
100000 200 0.0
><>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment