googlecurl - Wrapper around oauth2curl providing automatic pagination of items in google API responses. Supports gdata json, youtube jsonc and new-style google API responses.

googlecurl.py

#! /usr/bin/python3 -u
"""A wrapper around oauth2curl. Makes multiple
calls to oauth2curl. Returns items one item
per line."""
from argparse import ArgumentParser
from subprocess import Popen, PIPE
import sys
from urllib.parse import urlsplit, parse_qs, urlencode, urlunsplit
import json
import signal

signal.signal(signal.SIGPIPE, signal.SIG_DFL)

def main():
    p = ArgumentParser()
    args, rest = p.parse_known_args()
    if not rest:
        p.error("URL not specified")
    url = urlsplit(rest[-1])
    del rest[-1]
    query = dict((k, v[0]) for k, v in parse_qs(url.query).items())
    while True:
        page_url = urlunsplit(url._replace(query = urlencode(query)))
        x = Popen(["oauth2curl"] + rest + [page_url], stdout = PIPE)
        stdout, _ = x.communicate()
        if x.returncode != 0:
            sys.exit(x.returncode)
        y = json.loads(stdout.decode().strip())
        if "feed" in y: 
            #Gdata 
            for x in y["feed"]["entry"]:
                print(json.dumps(x))
            total_results = y["feed"]["openSearch$totalResults"]["$t"]
            start_index = y["feed"]["openSearch$startIndex"]["$t"]
            items_per_page = y["feed"]["openSearch$itemsPerPage"]["$t"]
            if start_index + items_per_page > total_results:
                break
            query["start-index"] = start_index + items_per_page
        elif "data" in y:
            #Youtube jsonc:
            for x in y["data"]["items"]:
                print(json.dumps(x))
            total_results = y["data"]["totalItems"]
            start_index = y["data"]["startIndex"]
            items_per_page = y["data"]["itemsPerPage"]
            if start_index + items_per_page > total_results:
                break
            query["start-index"] = start_index + items_per_page
        elif "items" in y:
            #New Google API pagination
            for x in y["items"]:
                print(json.dumps(x))
            if "nextPageToken" not in y:
                break
            query["pageToken"] = y["nextPageToken"]

if __name__ == "__main__":
    main()

youtube-dl-favorites.sh

#Download youtube favourites using youtube-dl since youtube seems to love deleting my favorited videos.
googlecurl 'https://gdata.youtube.com/feeds/api/users/default/favorites?v=2&alt=jsonc' | while read -r; do jshon -e 'video' -e 'player' -e 'default' -u <<< "$REPLY"; done | youtube-dl -t -a -