twitch smiles downloader

twitch-grabber.py

from argparse import ArgumentParser
from multiprocessing import cpu_count
from os import path, makedirs
from threading import Thread

from progressbar import ProgressBar
from requests import get

args = ArgumentParser()
args.add_argument_group('General')
args.add_argument('-f', '--start', metavar='url', type=str, help='Start index', default=1)
args.add_argument('-t', '--stop', metavar='url', type=str, help='Stop index', default=2000000)
args.add_argument('-m', '--max-threads', metavar='x', type=str, help='Max threads', default=(cpu_count() * 2))
args.add_argument('-s', '--allow-subdir', const=True, action='store_const', default=False)
args.add_argument('-d', '--destination', metavar='x', type=str, help='Destination', default='')
_args = args.parse_args()

max_threads = _args.max_threads
threads = []
url = 'https://static-cdn.jtvnw.net/emoticons/v1/{}/3.0'
progress = ProgressBar(max_value=_args.stop)


def download(n):
    if n > 0 and n % max_threads == 0:
        progress.update(n)
        # print('[{} / {}]'.format(n, max_n))
    subdir = ''
    if _args.allow_subdir:
        _n = '{:0>7}'.format(n)
        subdir = _n[::-1][4:7:][::-1]
    file = path.join(_args.destination, subdir, 'icon_{}.png'.format(n))
    if not path.isdir(path.dirname(file)):
        print(subdir)
        makedirs(path.dirname(file))
    if not path.isfile(file):
        with open(file, 'wb') as w:
            w.write(get(url.format(n)).content)


for i in range(int(_args.start), int(_args.stop)):
    if i > 0 and i % max_threads == 0:
        for thread in threads:
            thread.join()
        threads = []
    t = Thread(target=download, args=(i,))
    t.start()
    threads.append(t)
for thread in threads:
    thread.join()

progress.finish()