scrape github gists

scrape_gists.py

"""scrape github gists"""
#!/usr/bin/env python
from sys import argv
from json import load
from urllib import request
from subprocess import call

GITHUB = "https://api.github.com/users/"
USER = argv[1]
call(['mkdir', '-p', USER])

COUNT = 0
PERPAGE = 100
GISTCOUNT = load(request.urlopen(f"{GITHUB}{USER}"))['public_gists']
PAGES = (GISTCOUNT//PERPAGE)+2

print(f"Found {GISTCOUNT} gists in {PAGES-1} pages")

for page in range(1, PAGES):
    for gist in load(request.urlopen(f"{GITHUB}{USER}/gists?page={page}&per_page={PERPAGE}")):
        COUNT += 1
        gistd = list(gist['files'].keys())[0]
        gistUrl = f"git://gist.github.com/{gist['id']}.git"
        path = f"{USER}/{gist['id']}"
        print(f"[*] cloning {gistUrl} from page {page} [{COUNT}/{GISTCOUNT}]")
        call(['git', 'clone', '--quiet', gistUrl, path])
        call(['mv', path, f"{USER}/{gistd}"])