persist data locally

data_persistance.py

def persist_local(data, args, folder, 
                    id_keys=['experiment_id'], 
                    as_type='.parquet.gz',
                    save_path='persistance'):

    save_path = build_path(args, folder, id_keys, as_type, save_path)

    # Check if path exists, add path if not
    if not save_path.parent.exists():
        save_path.parent.mkdir(mode=777, parents=True, exist_ok=True)

        subprocess.run(['chmod', '-R', '777', f'{save_path.parent}'])

    if as_type == '.parquet.gz':
        df = pa.Table.from_pandas(data)
        pq.write_table(df, save_path, compression='gzip')

    elif as_type == '.p':
        pickle.dump(data, open(save_path, 'wb'))
    elif as_type == '.dill':
        dill.dump(data, open(save_path, 'wb'))
    elif as_type == '.png':
        data.savefig(save_path)
    else:
        raise 'This file type cannot be saved '

    subprocess.run(['chmod', '777', f'{save_path}'])


def build_path(args, folder, id_keys=['experiment_id'], as_type='.parquet.gz', save_path='persistance'):

    if save_path == 'persistance':

        save_path = load_environment['persistance_path']

    path_id = '_'.join([str(args[key]) for key in id_keys])

    # If folder == Nolsne, it does not use it to build path
    return Path(os.path.join(*filter(lambda x: x is not None, 
                        [save_path, folder, path_id + as_type])))