scrapping.py

async def scrap_exchanges():
    coins = pd.DataFrame.from_csv('crypto/data/coins.csv')

    data = defaultdict(lambda:[])
    chunks = [coins[i: i + 5] for i in range(0, len(coins), 5)]
    
    for chunk in chunks[0:10]:
        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
            futures = [loop.run_in_executor(executor, get_exchanges_for_coin, coin.split(":")[0]) for coin in chunk["Coins"]]

            for future in futures:
                coin, exchanges = await future
                for exchange in exchanges:
                    data[exchange].append(coin)
        time.sleep(3)

    print("done")
    # json.dump(data, open('crypto/data/dict.json', 'w'))


if __name__ == "__main__":
    loop = asyncio.get_event_loop()
    loop.run_until_complete(scrap_exchanges())