Download all gists of a user

get_gists.py

import sys
from subprocess import call
import json
import os
import requests


def download_gists(gists: list):
    for gist in gists:
        call(["git", "clone", gist["git_pull_url"]])

        #  name of the first file in the gist
        new_folder_name = sorted(list(gist.get("files", {}).keys()))[0].split(".")[0]

        os.rename(gist["id"], new_folder_name)

        description_file = os.path.join(new_folder_name, "description.txt")
        with open(description_file, "w") as f:
            f.write(f"{gist['description']}\n")


def visit_pages(user: str):
    next_page = True
    page = 1
    while next_page:
        url = f"https://api.github.com/users/{user}/gists?page={page}"
        r = requests.get(url)

        if not len(r.json()):
            next_page = False
        else:
            page += 1

        download_gists(r.json())


user = sys.argv[1]
visit_pages(user)

Sorry to be dense but I don't really know python. I tried python3 get_gists.py but got this message:

Traceback (most recent call last):
  File "/Users/tjluoma/Downloads/get_gists.py", line 5, in <module>
    import requests
ModuleNotFoundError: No module named 'requests'

Can you tell me what I'm doing wrong?

you need to install requests module. please run pip install requests

Just added some codestyle, shebang, one useless comment and parallelism.

🙃

#!/usr/bin/env python3

import os
import sys
import json
import hashlib
import requests

from subprocess import call
from concurrent.futures import ThreadPoolExecutor as PoolExecutor

def download_all_from_user(user: str):
    
    next_page = True
    page = 1
    
    while next_page:
        
        url = f"https://api.github.com/users/{user}/gists?page={page}"
        
        response = requests.get(url)

        if not len(response.json()):
            next_page = False
        else:
            page += 1

        download_all(response.json())

def download_all(gists: list):
    with PoolExecutor(max_workers=10) as executor:
        for _ in executor.map(download, gists):
            pass

def download(gist):
    
    target = gist["id"] + hashlib.md5(gist["updated_at"].encode('utf-8')).hexdigest()
    
    call(["git", "clone", gist["git_pull_url"], target])

    description_file = os.path.join(target, "description.txt")
    
    with open(description_file, "w") as f:
        f.write(f"{gist['description']}\n")

# Run

user = sys.argv[1]

download_all_from_user(user)

@antonydevanchi thank you :)

I forked this as well! https://gist.github.com/varenc/c65e6af52e4dc6f2d5f1aac9ea9a349e

Fixed a bug where a user might have some gists with the same name, resulting in errors. Also added handling for the when the github API rate limits you (which happens often for unauthenticated requests!)

Just added some codestyle, shebang, one useless comment and parallelism.

🙃

#!/usr/bin/env python3

import os
import sys
import json
import hashlib
import requests

from subprocess import call
from concurrent.futures import ThreadPoolExecutor as PoolExecutor

def download_all_from_user(user: str):
    
    next_page = True
    page = 1
    
    while next_page:
        
        url = f"https://api.github.com/users/{user}/gists?page={page}"
        
        response = requests.get(url)

        if not len(response.json()):
            next_page = False
        else:
            page += 1

        download_all(response.json())

def download_all(gists: list):
    with PoolExecutor(max_workers=10) as executor:
        for _ in executor.map(download, gists):
            pass

def download(gist):
    
    target = gist["id"] + hashlib.md5(gist["updated_at"].encode('utf-8')).hexdigest()
    
    call(["git", "clone", gist["git_pull_url"], target])

    description_file = os.path.join(target, "description.txt")
    
    with open(description_file, "w") as f:
        f.write(f"{gist['description']}\n")

# Run

user = sys.argv[1]

download_all_from_user(user)

This is amazing :) thank you