Last active
September 18, 2023 12:19
-
-
Save morrisalp/65dfdadd756ec1e9fa7566e539eda87a to your computer and use it in GitHub Desktop.
send async HTTP requests using grequests with tqdm progress bar
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from tqdm import tqdm | |
import requests, grequests | |
class ProgressSession(): | |
def __init__(self, urls): | |
self.pbar = tqdm(total = len(urls), desc = 'Making async requests') | |
self.urls = urls | |
def update(self, r, *args, **kwargs): | |
if not r.is_redirect: | |
self.pbar.update() | |
def __enter__(self): | |
sess = requests.Session() | |
sess.hooks['response'].append(self.update) | |
return sess | |
def __exit__(self, *args): | |
self.pbar.close() | |
def get_urls_async(urls): | |
with ProgressSession(urls) as sess: | |
rs = (grequests.get(url, session = sess, timeout = 5) for url in urls) | |
return grequests.map(rs) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
fire