Created
May 25, 2012 21:05
-
-
Save mdwhatcott/2790544 to your computer and use it in GitHub Desktop.
Read any number of files in parallel.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
FIRST = 'path to first file' | |
SECOND = 'path to second file' | |
def main(): | |
with open(FIRST) as first, open(SECOND) as second: | |
for line1, line2 in read_parallel(first, second): | |
pass # process lines | |
def read_parallel(*files): | |
lines = list(gather_lines(*files)) | |
while any(lines): | |
yield lines | |
lines = list(gather_lines(*files)) | |
def gather_lines(*files): | |
for f in files: | |
yield f.readline() | |
if __name__ == '__main__': | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment