Created
July 3, 2019 18:09
-
-
Save yassu/3330a1e47d9b7f8a9def36a79eb41900 to your computer and use it in GitHub Desktop.
Use feature with tqdm
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# -*- coding: utf-8 -*- | |
from tqdm import tqdm | |
import time | |
from concurrent.futures import ThreadPoolExecutor | |
import random | |
class TaskManager: | |
def __init__(self, task, args, number_of_max_workers=4): | |
self.task = task # Dict[task_id, task_function] | |
self.args = args | |
self.number_of_max_workers = number_of_max_workers | |
def run(self): | |
results = {} | |
with ThreadPoolExecutor(max_workers=self.number_of_max_workers) as executor: | |
futures = {} | |
for arg in self.args: | |
futures[arg] = executor.submit(self.task, *arg) | |
for arg, f in tqdm(futures.items()): | |
results[arg] = f.result() | |
return results | |
def task(*args): | |
time.sleep(random.uniform(0.5, 1.0)) | |
return 3 * args[0] | |
manager = TaskManager( | |
task, | |
[ (i, 2 * i) for i in range(100)], | |
number_of_max_workers=10) | |
res = manager.run() | |
print(res) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment