Skip to content

Instantly share code, notes, and snippets.

@gdunstone
Created May 18, 2017 07:11
Show Gist options
  • Select an option

  • Save gdunstone/8eb936d7dfb1ace45bd758617b500bc0 to your computer and use it in GitHub Desktop.

Select an option

Save gdunstone/8eb936d7dfb1ace45bd758617b500bc0 to your computer and use it in GitHub Desktop.
A generator based file line reader, optimised for reading in order, with a list like interface.
def get_generator(fh):
while True:
data = fh.readline()
if not data:
break
yield data
class GenFileReader(object):
def __init__(self, fn):
self._fn = fn
self._fh = open(self._fn)
self._rewind()
def _rewind(self):
self._fh.seek(0)
self._generator = get_generator(self._fh)
self._index = 0
def _getitem_slice(self, slice):
start, stop, step = slice.start, slice.stop, slice.step
start = 0 if start is None else start
stop = float("inf") if stop is None else stop
step = 1 if step is None else step
if stop is not None and stop < start:
start, stop = stop, start
step *= -1
r = []
ii = start
while ii <= stop:
try:
r.append(self[ii])
ii += step
except IndexError:
break
return r
def _getitem_int(self, index):
if index < 0:
raise IndexError("Negative index used")
if index == 0:
self._rewind()
if index < self._index:
self._rewind()
data = ""
while self._index <= index:
try:
data = next(self._generator)
self._index += 1
except StopIteration:
raise IndexError
return data.strip()
def __getitem__(self, index):
if isinstance(index, slice):
return self._getitem_slice(index)
elif isinstance(index, int):
return self._getitem_int(index)
def __del__(self):
self._fh.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment