HDf5 test

read_hdf5.py

import h5py
import numpy as np
from tqdm import tqdm

indices = list(range(100000))
rand_index = np.random.permutation(len(indices)).astype(str)
BATCH_SIZE = 128


def batch_gen():
    batch = []
    for ind in rand_index:
        batch.append(ind)
        if len(batch) == BATCH_SIZE:
            yield batch
            batch = []
    if len(batch) > 0:
        yield batch


with h5py.File('data.h5', 'r') as store, tqdm(total=len(indices)) as pbar:
    for batch_index in batch_gen():
        for ind in sorted(batch_index):
        # for ind in batch_index:
            f = store[ind][()]
            pbar.set_postfix(ind=ind)
            pbar.update()

write_hdf5.py

import h5py
import numpy as np
from tqdm import tqdm

with h5py.File('data.h5','w') as store:
    for idx in tqdm(range(100000)):
        x = np.random.normal(size=10000)
        store[str(idx)] = x