Convert dictionary based JSON data structures to CSV files

json_to_csv_with_plots.py

import os, json
import matplotlib.pyplot as plt
import pandas as pd


if __name__ == "__main__":
    base_dir = os.path.dirname(os.path.abspath(__file__))

    all_dirs = os.listdir(base_dir)

    for dir in all_dirs:
        current_dir = os.path.join(base_dir, dir)
        if os.path.isdir(current_dir):
            print(f"Processing {current_dir}")
            json_files = [f for f in os.listdir(current_dir) if f.endswith(".json")]
            for file in json_files:
                print(f"Processing {file}")
                with open(f"{current_dir}/{file}") as f:
                    data = json.load(f)
                    df = pd.DataFrame()
                    if isinstance(data, list):
                        df = pd.json_normalize(data)
                    elif isinstance(data, dict):
                        df = pd.DataFrame(data).transpose()

                    if not df.empty:
                        path_to_write_csv = os.path.join(
                            current_dir, file.replace(".json", ".csv")
                        )
                        path_to_write_fig = os.path.join(
                            current_dir, file.replace(".json", ".png")
                        )
                        for path_to_check in [path_to_write_csv, path_to_write_fig]:
                            if os.path.exists(path_to_check):
                                os.remove(path_to_check)
                        print(f"Writing to {path_to_write_csv}")
                        df.rename_axis("epoch", inplace=True)
                        df.to_csv(path_to_write_csv, index=True)
                        df = df.astype(float)
                        df.plot()
                        plt.title(f"Training plots for {dir}")
                        plt.xlabel("Index")
                        plt.ylabel("Losses")
                        plt.grid()
                        plt.grid(which="minor", alpha=0.25)
                        # plt.show()
                        plt.tight_layout()
                        plt.savefig(path_to_write_fig)