Skip to content

Instantly share code, notes, and snippets.

@yassu
Created July 3, 2019 18:09
Show Gist options
  • Save yassu/3330a1e47d9b7f8a9def36a79eb41900 to your computer and use it in GitHub Desktop.
Save yassu/3330a1e47d9b7f8a9def36a79eb41900 to your computer and use it in GitHub Desktop.
Use feature with tqdm
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from tqdm import tqdm
import time
from concurrent.futures import ThreadPoolExecutor
import random
class TaskManager:
def __init__(self, task, args, number_of_max_workers=4):
self.task = task # Dict[task_id, task_function]
self.args = args
self.number_of_max_workers = number_of_max_workers
def run(self):
results = {}
with ThreadPoolExecutor(max_workers=self.number_of_max_workers) as executor:
futures = {}
for arg in self.args:
futures[arg] = executor.submit(self.task, *arg)
for arg, f in tqdm(futures.items()):
results[arg] = f.result()
return results
def task(*args):
time.sleep(random.uniform(0.5, 1.0))
return 3 * args[0]
manager = TaskManager(
task,
[ (i, 2 * i) for i in range(100)],
number_of_max_workers=10)
res = manager.run()
print(res)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment