test_read.py

from contextlib import contextmanager
import io
import os
import shutil
import sys
import tempfile
import time


class Timer(object):

    def __init__(self, verbose=False):
        self.verbose = verbose

    def __enter__(self):
        self.start = time.time()
        return self

    def __exit__(self, *args):
        self.end = time.time()
        self.secs = self.end - self.start
        self.msecs = self.secs * 1000  # millisecs
        if self.verbose:
            print('elapsed time: %f ms' % self.msecs)


@contextmanager
def TemporaryDirectory(directory=None):
    name = tempfile.mkdtemp(dir=directory)
    try:
        yield name
    finally:
        shutil.rmtree(name)


def readfiles(topdir, buffering=-1):
    n = 0
    secs = 0
    for subdir, dirs, files in os.walk(topdir):
        for f in files:
            filepath = os.path.join(subdir, f)

            if os.path.isfile(filepath):
                # print (filepath)
                n += 1
                with Timer() as t:
                    with open(filepath, "rb", buffering=buffering) as fileobj:
                        header = fileobj.read(128)
                secs += t.secs

    if n > 0:
        print("buf=%6d files=%6d secs=%8.3f usecsperfile=%8.3f" %
              (buffering, n, secs, 1000000 * secs / float(n)))


rootdir = sys.argv[1]
if not os.path.isdir(rootdir):
    sys.exit(1)

print("Default buffer size: %d" % io.DEFAULT_BUFFER_SIZE)

with TemporaryDirectory(rootdir) as tmpdir:
    numfiles = 5000
    datasize = 16384
    print("datasize: %d" % datasize)
    data = bytes(b'\x07' * datasize)

    for i in range(0, numfiles):
        fname = os.path.join(tmpdir, "tmpfile_%06d" % i)
        with open(fname, "wb") as wf:
            wf.write(data)
            wf.flush()
            os.fsync(wf.fileno())
    for buf in [-1, 0, 128, 16384, 128, 0, -1]:
        readfiles(tmpdir, buf)