Extract all tensorboard events files to pandas dataframe

tensorboard_events_to_csv.py

import tensorflow as tf
from pathlib import Path
import pandas as pd
from tqdm import tqdm
from collections import defaultdict

# Get all event* runs from logging_dir subdirectories
logging_dir = Path('.', 'logs')
event_paths = logging_dir.glob('**/event*')

# Extraction function
def sum_log(path):
    try:
        tags = []
        values = []
        steps = []
        curr_step = defaultdict(int)
        for e in tf.train.summary_iterator(str(path)):
            for v in e.summary.value:
                tags.append(v.tag)
                values.append(v.simple_value)
                steps.append(curr_step[v.tag])

                curr_step[v.tag] = curr_step[v.tag]+1
    
    # Dirty catch of DataLossError
    except:
        print('Event file possibly corrupt: {}'.format(path))
        return None

    runlog = pd.DataFrame({'metric': tags, 'value': values, 'step': steps})

    return runlog


# Call & append
all_log = pd.DataFrame()
for path in event_paths:
    log = sum_log(path)
    if log is not None:
        log['run'] = path.parent.stem
        if all_log.shape[0] == 0:
            all_log = log
        else:
            all_log = all_log.append(log)
    


# Inspect
print(all_log.shape)
all_log.head()    
            
# Store
all_log.to_csv('all_training_logs_in_one_file.csv', index=None)