main.py

#!/usr/bin/env python3
import asyncio


class AsyncThrottle():
	DEFAULT_THROTTLE_MAIN = 100
	DEFAULT_THROTTLE_KEY = 3

	def __init__(self, main_throttle=None):
		if main_throttle is None: main_throttle = self.DEFAULT_THROTTLE_MAIN
		self.throttle = {}
		self.main = asyncio.Semaphore(main_throttle)

	def __call__(self, key=None, key_throttle=None):
		if key is None:
			return self.main
		else:			
			if key not in self.throttle:
				if key_throttle is None: key_throttle = self.DEFAULT_THROTTLE_KEY
				self.throttle[key] = asyncio.Semaphore(key_throttle)
			return self.throttle[key]

	def set(self, key, key_throttle=None):
		if key not in self.throttle:
			if key_throttle is None: key_throttle = self.DEFAULT_THROTTLE_KEY
			self.throttle[key] = asyncio.Semaphore(key_throttle)
			return True
		return False

	async def acquire(self, key):
		await self.throttle[key].acquire()

	def release(self, key):
		self.throttle[key].release()


async def get_url(url, throttle):
	async with throttle():
		async with throttle(url):
			print(f"GET URL: {url} ...")
			await asyncio.sleep(3)
			print(f"GOT IT! {url}.")
	return 1


async def wait_for_urls(urls, throttle):
	requests = []
	for url in urls:
		requests.append(get_url(url, throttle))
	results = await asyncio.gather(*requests, return_exceptions=True)
	return results


async def my_data_grabber1(throttle):
	result = await wait_for_urls(("abc", "xxx", "abc", "abc", "abc", "ooo", "ooo","ooo","ooo","ooo","ooo","ooo","ooo","ooo","ooo","ooo","ooo","ooo",), throttle)
	return result


async def my_data_grabber2(throttle):
	result = await wait_for_urls(("xxx","xxx","xxx","xxx","xxx"), throttle)
	return result


async def main():
	throttle = AsyncThrottle()
	requests = (wait_for_urls(("abc", "eee", "abc", "abc", "eee", "abc"), throttle), 
		my_data_grabber1(throttle),
		my_data_grabber2(throttle),my_data_grabber2(throttle),my_data_grabber2(throttle),my_data_grabber2(throttle),my_data_grabber2(throttle))
	results = await asyncio.gather(*requests, return_exceptions=True)
	print(results)


if __name__ == "__main__":
	asyncio.run(main())