large_gibbs_kernel_run.py

import argparse
#
from experiment_runner.general_utils import Timer
from experiment_runner.ExperimentRunner import ExperimentRunner
import experiment_runner.ExperimentRunner


parser = argparse.ArgumentParser()
parser.add_argument('--dirname', default='geweke_on_schemas', type=str)
parser.add_argument('--dirname_prefix', default=None, type=str)
parser.add_argument('--storage_type', default='fs', type=str)
parser.add_argument('--pull_s3', action='store_true')
parser.add_argument('--bucket_str', default='experiment_runner', type=str)
parser.add_argument('--do_plots', action='store_true')
args = parser.parse_args()
dirname = args.dirname
dirname_prefix = args.dirname_prefix
storage_type = args.storage_type
bucket_str = args.bucket_str
pull_s3 = args.pull_s3
do_plots = args.do_plots
#
if dirname_prefix is None:
    dirname_prefix = dirname
    pass


if pull_s3:
    er_s3 = ExperimentRunner(None, dirname, dirname_prefix=dirname_prefix,
            bucket_str=bucket_str, storage_type='s3')
    _frame = er_s3.frame[er_s3.frame.num_iters==10000]
    experiment_runner.ExperimentRunner.copy_from_s3(er_s3, _frame)


er = ExperimentRunner(None, dirname, dirname_prefix=dirname_prefix,
        bucket_str=bucket_str, storage_type=storage_type)
print er.frame


if do_plots:
    from crosscat.utils.geweke_utils import plot_result
    import crosscat.utils.geweke_utils as geweke_utils
    geweke_utils.dirname_prefix = dirname_prefix
    for _id in er.frame.index:
        result = er._get_result(_id)
        plot_result(result, dirname)
        pass
    pass

read_geweke.py

import argparse
#
from experiment_runner.general_utils import Timer
from experiment_runner.ExperimentRunner import ExperimentRunner
import experiment_runner.ExperimentRunner


parser = argparse.ArgumentParser()
parser.add_argument('--dirname', default='geweke_on_schemas', type=str)
parser.add_argument('--dirname_prefix', default=None, type=str)
parser.add_argument('--storage_type', default='fs', type=str)
parser.add_argument('--pull_s3', action='store_true')
parser.add_argument('--bucket_str', default='experiment_runner', type=str)
parser.add_argument('--do_plots', action='store_true')
args = parser.parse_args()
dirname = args.dirname
dirname_prefix = args.dirname_prefix
storage_type = args.storage_type
bucket_str = args.bucket_str
pull_s3 = args.pull_s3
do_plots = args.do_plots
#
if dirname_prefix is None:
    dirname_prefix = dirname
    pass


if pull_s3:
    er_s3 = ExperimentRunner(None, dirname, dirname_prefix=dirname_prefix,
            bucket_str=bucket_str, storage_type='s3')
    experiment_runner.ExperimentRunner.copy_from_s3(er_s3)


er = ExperimentRunner(None, dirname, dirname_prefix=dirname_prefix,
        bucket_str=bucket_str, storage_type=storage_type)
print er.frame


if do_plots:
    from crosscat.utils.geweke_utils import plot_result
    import crosscat.utils.geweke_utils as geweke_utils
    geweke_utils.dirname_prefix = dirname_prefix
    for _id in er.frame.index:
        result = er._get_result(_id)
        plot_result(result, dirname)
        pass
    pass

update_geweke_results.py

import argparse
#
from experiment_runner.ExperimentRunner import ExperimentRunner


parser = argparse.ArgumentParser()
# parser.add_argument('dirname', type=str)
parser.add_argument('--dirname', default='geweke_on_schemas.post', type=str)
parser.add_argument('--dirname_prefix', default=None, type=str)
args = parser.parse_args()
dirname = args.dirname
dirname_prefix = args.dirname_prefix
if dirname_prefix is None:
    dirname_prefix = dirname
    pass


def update_geweke_result(result):
    result = result.copy()
    result['summary'] = result['processed_data']['summary_kls']
    return result


storage_type = 's3'
bucket_str = 'experiment_runner'


er1 = ExperimentRunner(None, dirname, dirname_prefix=dirname_prefix,
        summary_filename='result.pkl',
        bucket_str=bucket_str, storage_type=storage_type)
er2 = ExperimentRunner(None, dirname, dirname_prefix=dirname_prefix,
        bucket_str=bucket_str, storage_type=storage_type)


for _id in er1.frame.index:
    print 'getting result for dirname=%s, id=%s' % (dirname, _id)
    result = er1._get_result(_id)
    result = update_geweke_result(result)
    print 'writing updated result for dirname=%s, id=%s' % (dirname, _id)
    er2.storage_accessor.writer(result, dirname)
    pass