caching_pattern_for_long_running.py

caching_pattern_for_long_running.py

from pathlib import Path
import pandas as pd
from tqdm.auto import tqdm

experiments = {
  'exp1': dict(lr=1e-4),
 ]
cache_dir = Path('../data/30_processed/results_cache1')
cache_dir.mkdir(exists_ok=True)
  
results = {}
for experiment_name, experiment_kwargs in tqdm(experiments.items()):
    f_cache = cache_dir / f"{experiment_name}.parquet"
    if Path(f_cache).exists():
        print(f"Skipping {experiment_name}")
        results = pd.read_parquet(f_cache)
        results[experiment_name] = results
        continue
    
    ... do experiment
  
    results[experiment_name] = results
    # cache
    pd.to_parquet(results, f_cache)
  
 df_res = pd.DataFrame(results)
 df_res