A just script to summarize chainer log and output as csv

summarize_chainer_log.py

import argparse
import csv
import json
import glob
import numpy as np

parser = argparse.ArgumentParser()
parser.add_argument('--input', type=str, required=True)
parser.add_argument('--output', default='result.csv')
args = parser.parse_args()


json_log_path_list = glob.glob(args.input)
csv_writer = csv.writer(open(args.output, 'w'))
csv_writer.writerow(['minibatch_size', 'mean_elapsed_time', 'sd_elapsed_time', 'n'])

for log_path in json_log_path_list:
    minibatch_size = int(log_path.split('_')[1])
    elapsed_times = []
    print(log_path)
    with open(log_path) as f:
        raw_records = json.load(f)[0:-1]
        previous_elapsed_time = 0
        for raw_record in raw_records:
            elapsed_time = raw_record['elapsed_time']
            if previous_elapsed_time == 0:
                previous_elapsed_time = elapsed_time
                continue

            elapsed_times.append(elapsed_time - previous_elapsed_time)
            previous_elapsed_time = elapsed_time

    elapsed_times = np.array(elapsed_times)
    n = elapsed_times.size
    mean_elapsed_time = elapsed_times.mean()
    sd_elapsed_time = elapsed_times.std()

    csv_writer.writerow([minibatch_size, mean_elapsed_time, sd_elapsed_time, n])