monitor_scraping.py

import os
import ast
import time
import string
from tqdm import tqdm_notebook
tqdm_notebook()

def files():
    return [t for t in os.listdir(os.curdir) if t.endswith('.txt')]
    
def open_dic(path):
    with open(path, 'r') as f:
        return ast.literal_eval(f.read())
        
def get_all_artists():
    final_dic = {}
    total_scraped = 0
    for path in files():
        new_dic = open_dic(path)
        total_scraped += len(new_dic)
        final_dic = {**final_dic, **new_dic}
    return final_dic, len(final_dic), total_scraped
    
def update_status_all_bots(pbars):
    f = files()
    for l in string.ascii_lowercase:
        pbars[l].update(-pbars[l].last_print_n)
        pbars[l].update(len([x for x in f if x.replace('artists_','').startswith(l)]))
        
pbars = {l:tqdm_notebook(total=26, desc=l) for l in string.ascii_lowercase}

while True:
    final_dic, total_artists, total_scraped = get_all_artists()
    print('Number of unique artists:', total_artists, ' --- Duplicates:', total_scraped - total_artists, end='\r')
    update_status_all_bots(pbars)
    time.sleep(5)