Apply function to file in chunks.

chunkapply.py

import fileinput
import numpy as np
import cPickle as pickle

# (String -> a) -> Filename -> Int -> SideEffect[FileSystem] # List a
def chunkapply(f, filename, chunksize, savefile):
  """ For each chunk, apply f to each line, and save the list of results to 
      file starting with savefilename, and followed by chunk number. """
      
  data = []
  line_counter = 0
  chunk_index = 0
  
  for line in fileinput.input(filepath):
    data.append(f(line)) 
      
    if line_counter > chunk_size:
      savefilename = '{}_{}.pkl'.format(savefile, str(chunk_index))
      pickle.dump(np.array(data), open(savefilename, 'wb'))
        
      line_counter = 0  # reset line counter        
      chunk_index += 1 # increment chunk index
      del data[:] # free memory