Created
February 10, 2011 12:59
-
-
Save miku/820472 to your computer and use it in GitHub Desktop.
How do you split a csv file into evenly sized chunks in Python?
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# import csv | |
# reader = csv.reader(open('4956984.csv', 'rb')) | |
def gen_chunks(reader, chunksize=100): | |
""" | |
Chunk generator. Take a CSV `reader` and yield | |
`chunksize` sized slices. | |
""" | |
chunk = [] | |
for index, line in enumerate(reader): | |
if (index % chunksize == 0 and index > 0): | |
yield chunk | |
del chunk[:] | |
chunk.append(line) | |
yield chunk | |
# test gen_chunk with some dummy range object ... | |
for chunk in gen_chunks(range(10), chunksize=3): | |
print chunk # process chuck | |
# $ python 4956984.py | |
# [0, 1, 2] | |
# [3, 4, 5] | |
# [6, 7, 8] | |
# [9] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment