File.each_lines_batch

gistfile1.rb

class IO
  def each_lines_batch(batch_size=1000, buffer_size=4096)
    lines = []
    chunk = ""

    while data = file.read(buffer_size)
      chunk << data
      lines += chunk.lines.to_a

      # last line might not be read until it's end
      chunk = lines.pop

      while lines.size > batch_size
        batch = lines.shift(batch_size)
        yield batch
      end
    end

    lines += chunk.lines.to_a
    yield lines if lines.any?
  end
end