Skip to content

Instantly share code, notes, and snippets.

@palindrom615
Created April 26, 2022 13:40
Show Gist options
  • Save palindrom615/250edfcc82564ee256982b85ac3d6115 to your computer and use it in GitHub Desktop.
Save palindrom615/250edfcc82564ee256982b85ac3d6115 to your computer and use it in GitHub Desktop.
Concurrent dowload in python
import asyncio
from aiohttp import ClientSession
CONCURRENT_DOWNLOADS = 16
semaphore = asyncio.Semaphore(CONCURRENT_DOWNLOADS)
def limit_concurrent(f):
async def wrapper(*args, **kwargs):
async with semaphore:
return await f(*args, **kwargs)
return wrapper
@limit_concurrent
async def download(session: ClientSession, url: str, filename: str = None):
if not filename:
filename = url.split('/')[-1]
async with session.get(url, ) as res:
with open(filename, "wb") as f:
async for chunk, _ in res.content.iter_chunks():
f.write(chunk)
async def main(download_list):
async with ClientSession() as session:
await asyncio.gather(*[download(session, i) for i in download_list])
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment