Skip to content

Instantly share code, notes, and snippets.

@kimbo
Last active May 2, 2019 16:40
Show Gist options
  • Save kimbo/4ac1eb28ca119f2851b2965d31131810 to your computer and use it in GitHub Desktop.
Save kimbo/4ac1eb28ca119f2851b2965d31131810 to your computer and use it in GitHub Desktop.
Use tqdm while reading a file (without reading the entire file into memory)
import subprocess
import tqdm
import os
import argparse
def get_line_count(file):
abs_path = os.path.abspath(file)
completed_process = subprocess.run(args=['wc', '-l', abs_path], check=True, encoding='utf-8',
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return int(completed_process.stdout.split()[0])
def main():
parser = argparse.ArgumentParser()
parser.add_argument('file')
args = parser.parse_args()
line_count = get_line_count(args.file)
t = tqdm.tqdm(total=line_count, ncols=100)
with open(args.file, 'r') as file:
for line in file:
# do stuff with this line
# at some point, call:
t.update()
t.close()
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment