Notebook version of the CLI at https://github.com/dogsheep/github-to-sqlite/

github2sqlite.ipynb

{
  "cells": [
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "from github_to_sqlite import utils\nfrom github_to_sqlite.cli import *\nfrom fastcore.all import *\nimport sqlite_utils",
      "execution_count": 7,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "db = sqlite_utils.Database(\"github.db\")\ntoken = load_token(\"auth.json\")",
      "execution_count": 171,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "repos = ['jph00','fastai']",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "for user in repos:\n    for repo in utils.fetch_all_repos(user, token):\n        utils.save_repo(db, repo)\n        time.sleep(1)",
      "execution_count": 4,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "def get(repos, fetch, save, **kwargs):\n    for repo in L(repos):\n        repo_full = utils.fetch_repo(repo, token)\n        utils.save_repo(db, repo_full)\n        res = fetch(repo, token, **kwargs)\n        save(db, list(res), repo_full if \"issue\" in kwargs else repo_full[\"id\"])\n        time.sleep(1)\n    utils.ensure_db_shape(db)",
      "execution_count": 59,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "def releases(repos): get(repos, utils.fetch_releases, utils.save_releases)",
      "execution_count": 60,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "def tags(repos): get(repos, utils.fetch_tags, utils.save_tags)",
      "execution_count": 62,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "def stop_when(commit):\n    try:\n        db[\"commits\"].get(commit[\"sha\"])\n        return True\n    except sqlite_utils.db.NotFoundError: return False\n\ndef commits(repos): get(repos, utils.fetch_commits, utils.save_commits, stop_when=stop_when)",
      "execution_count": 64,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "def issues(repos, issue=None): get(repos, utils.fetch_issues, utils.save_issues, issue=issue)",
      "execution_count": 66,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "repo_name = \"fastai/fastai\"",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "commits(repo_name)",
      "execution_count": 65,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "releases(repo_name)",
      "execution_count": 61,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "tags(repo_name)",
      "execution_count": 63,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "issues(repo_name)",
      "execution_count": 67,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.7.7",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "toc": {
      "nav_menu": {},
      "number_sections": false,
      "sideBar": true,
      "skip_h1_title": true,
      "base_numbering": 1,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "gist": {
      "id": "",
      "data": {
        "description": "github2sqlite.ipynb",
        "public": true
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 4
}

Install modules first with:

pip install github-to-sqlite fastcore