random_access_is_slow.py

# -*- coding: utf-8 -*-
"""
Created on Thu Sep 15 14:00:48 2016

@author: sakurai
"""

import os
import contexttimer

import numpy as np
from skimage import io


def make_minibatch(indexes):
    images = []
    paths_batch = paths[indexes]
    for path in paths_batch:
        image = io.imread(path).transpose((2, 0, 1))
        images.append(image)
    x_batch = np.stack(images)
    return x_batch

if __name__ == '__main__':
    dir_name = "resized_dataset_56"
    batch_size = 300
    num_batches = 10

    # dir_name 配下の全てのファイルパスを paths に取得する
    paths = []
    for root, dirs, filenames in os.walk(dir_name):
        for filename in filenames:
            paths.append(os.path.join(root, filename))
    paths = np.array(paths)
    num_examples = len(paths)

    # 先頭から連続したbatch_size個の画像を読み込む場合
    sequential_indexes = np.arange(num_examples)
    with contexttimer.Timer() as timer:
        for i in range(num_batches):
            indexes = sequential_indexes[i*batch_size:(i+1)*batch_size]
            x_data = make_minibatch(indexes)
    print "time:", timer.elapsed, "(sequential read)"

    # batch_size個のランダムな位置の画像を読み込む
    random_indexes = np.random.permutation(num_examples)
    with contexttimer.Timer() as timer:
        for i in range(num_batches):
            indexes = random_indexes[i*batch_size:(i+1)*batch_size]
            x_data = make_minibatch(indexes)
    print "time:", timer.elapsed, "(random read)"