fetch-most-popular-python-packages.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fetch most popular PyPI packages\n",
    "\n",
    "Top package list: https://hugovk.github.io/top-pypi-packages/\n",
    "\n",
    "PyPI JSON API: https://pypi.org/pypi/Django/json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "data = requests.get(\n",
    "    \"https://hugovk.github.io/top-pypi-packages/top-pypi-packages-365-days.min.json\"\n",
    ").json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_100 = data[\"rows\"][:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'download_count': 243085535, 'project': 'pip'},\n",
       " {'download_count': 187328653, 'project': 'six'},\n",
       " {'download_count': 181632012, 'project': 'botocore'},\n",
       " {'download_count': 164559678, 'project': 'python-dateutil'},\n",
       " {'download_count': 161660352, 'project': 's3transfer'},\n",
       " {'download_count': 147438614, 'project': 'setuptools'},\n",
       " {'download_count': 140312969, 'project': 'simplejson'},\n",
       " {'download_count': 140203328, 'project': 'pyyaml'},\n",
       " {'download_count': 139997780, 'project': 'pyasn1'},\n",
       " {'download_count': 137538812, 'project': 'docutils'},\n",
       " {'download_count': 136857259, 'project': 'futures'},\n",
       " {'download_count': 128690492, 'project': 'jmespath'},\n",
       " {'download_count': 122920938, 'project': 'requests'},\n",
       " {'download_count': 117938354, 'project': 'awscli'},\n",
       " {'download_count': 115212080, 'project': 'colorama'},\n",
       " {'download_count': 114989489, 'project': 'rsa'},\n",
       " {'download_count': 109370400, 'project': 'wheel'},\n",
       " {'download_count': 103335183, 'project': 'idna'},\n",
       " {'download_count': 100990041, 'project': 'certifi'},\n",
       " {'download_count': 95651916, 'project': 'urllib3'},\n",
       " {'download_count': 94197822, 'project': 'chardet'},\n",
       " {'download_count': 68982441, 'project': 'awscli-cwlogs'},\n",
       " {'download_count': 62954500, 'project': 'boto3'},\n",
       " {'download_count': 60445933, 'project': 'pytz'},\n",
       " {'download_count': 53603384, 'project': 'pbr'},\n",
       " {'download_count': 53601562, 'project': 'cffi'},\n",
       " {'download_count': 52888490, 'project': 'cryptography'},\n",
       " {'download_count': 48535925, 'project': 'jinja2'},\n",
       " {'download_count': 47460115, 'project': 'markupsafe'},\n",
       " {'download_count': 45821525, 'project': 'pycparser'},\n",
       " {'download_count': 43561870, 'project': 'enum34'},\n",
       " {'download_count': 42207495, 'project': 'numpy'},\n",
       " {'download_count': 38016222, 'project': 'ipaddress'},\n",
       " {'download_count': 37077981, 'project': 'setuptools-scm'},\n",
       " {'download_count': 35175460, 'project': 'asn1crypto'},\n",
       " {'download_count': 35119607, 'project': 'virtualenv'},\n",
       " {'download_count': 30889784, 'project': 'click'},\n",
       " {'download_count': 30099895, 'project': 'pytest-runner'},\n",
       " {'download_count': 30099212, 'project': 'pytest'},\n",
       " {'download_count': 29472427, 'project': 'decorator'},\n",
       " {'download_count': 28884820, 'project': 'future'},\n",
       " {'download_count': 28777581, 'project': 'pyparsing'},\n",
       " {'download_count': 28326453, 'project': 'boto'},\n",
       " {'download_count': 27558186, 'project': 'werkzeug'},\n",
       " {'download_count': 26999428, 'project': 'protobuf'},\n",
       " {'download_count': 26221078, 'project': 'psutil'},\n",
       " {'download_count': 25400379, 'project': 'lxml'},\n",
       " {'download_count': 25268462, 'project': 'argparse'},\n",
       " {'download_count': 24603982, 'project': 'pyopenssl'},\n",
       " {'download_count': 23937013, 'project': 'pandas'},\n",
       " {'download_count': 23686271, 'project': 'flask'},\n",
       " {'download_count': 23132867, 'project': 'mock'},\n",
       " {'download_count': 22569365, 'project': 'pyasn1-modules'},\n",
       " {'download_count': 22488580, 'project': 'py'},\n",
       " {'download_count': 21483410, 'project': 'pexpect'},\n",
       " {'download_count': 21415870, 'project': 'psycopg2'},\n",
       " {'download_count': 21287133, 'project': 'funcsigs'},\n",
       " {'download_count': 21279551, 'project': 'ptyprocess'},\n",
       " {'download_count': 21025707, 'project': 'coverage'},\n",
       " {'download_count': 20170432, 'project': 'sqlalchemy'},\n",
       " {'download_count': 19792602, 'project': 'jsonschema'},\n",
       " {'download_count': 19242226, 'project': 'httplib2'},\n",
       " {'download_count': 19048969, 'project': 'grpcio'},\n",
       " {'download_count': 19000174, 'project': 'pymysql'},\n",
       " {'download_count': 18970613, 'project': 'pygments'},\n",
       " {'download_count': 18773788, 'project': 'tornado'},\n",
       " {'download_count': 18496343, 'project': 'attrs'},\n",
       " {'download_count': 18409063, 'project': 'paramiko'},\n",
       " {'download_count': 18263457, 'project': 'itsdangerous'},\n",
       " {'download_count': 17761742, 'project': 'oauth2client'},\n",
       " {'download_count': 17612776, 'project': 'pillow'},\n",
       " {'download_count': 17248687, 'project': 'nose'},\n",
       " {'download_count': 17166689, 'project': 'cython'},\n",
       " {'download_count': 16756040, 'project': 'docopt'},\n",
       " {'download_count': 16653714, 'project': 'mccabe'},\n",
       " {'download_count': 16418721, 'project': 'selenium'},\n",
       " {'download_count': 16084550, 'project': 'django'},\n",
       " {'download_count': 15987566, 'project': 'singledispatch'},\n",
       " {'download_count': 15194487, 'project': 'scipy'},\n",
       " {'download_count': 15072760, 'project': 'pluggy'},\n",
       " {'download_count': 14590371, 'project': 'wrapt'},\n",
       " {'download_count': 14449059, 'project': 'bcrypt'},\n",
       " {'download_count': 14430072, 'project': 'uritemplate'},\n",
       " {'download_count': 14288465, 'project': 'google-cloud-core'},\n",
       " {'download_count': 14008795, 'project': 'babel'},\n",
       " {'download_count': 13898080, 'project': 'html5lib'},\n",
       " {'download_count': 13757328, 'project': 'google-api-python-client'},\n",
       " {'download_count': 13375194, 'project': 'pynacl'},\n",
       " {'download_count': 13272964, 'project': 'greenlet'},\n",
       " {'download_count': 13234813, 'project': 'functools32'},\n",
       " {'download_count': 13166819, 'project': 'tzlocal'},\n",
       " {'download_count': 13115403, 'project': 'websocket-client'},\n",
       " {'download_count': 12994334, 'project': 'backports-abc'},\n",
       " {'download_count': 12922980, 'project': 'pyflakes'},\n",
       " {'download_count': 12759680, 'project': 'redis'},\n",
       " {'download_count': 12572706, 'project': 'pycodestyle'},\n",
       " {'download_count': 12550053, 'project': 'amqp'},\n",
       " {'download_count': 12452089, 'project': 'arrow'},\n",
       " {'download_count': 12412179, 'project': 'markdown'},\n",
       " {'download_count': 12301282, 'project': 'matplotlib'}]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pip\n",
      "six\n",
      "botocore\n",
      "python-dateutil\n",
      "s3transfer\n",
      "setuptools\n",
      "simplejson\n",
      "pyyaml\n",
      "pyasn1\n",
      "docutils\n",
      "futures\n",
      "jmespath\n",
      "requests\n",
      "awscli\n",
      "colorama\n",
      "rsa\n",
      "wheel\n",
      "idna\n",
      "certifi\n",
      "urllib3\n"
     ]
    }
   ],
   "source": [
    "records = []\n",
    "for row in top_100[:20]:\n",
    "    info = requests.get(\"https://pypi.org/pypi/{}/json\".format(row[\"project\"])).json()[\"info\"]\n",
    "    info[\"download_count\"] = row[\"download_count\"]\n",
    "    print(row[\"project\"])\n",
    "    records.append(info)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'author': 'The pip developers',\n",
       " 'author_email': '[email protected]',\n",
       " 'bugtrack_url': None,\n",
       " 'classifiers': ['Development Status :: 5 - Production/Stable',\n",
       "  'Intended Audience :: Developers',\n",
       "  'License :: OSI Approved :: MIT License',\n",
       "  'Programming Language :: Python',\n",
       "  'Programming Language :: Python :: 2',\n",
       "  'Programming Language :: Python :: 2.7',\n",
       "  'Programming Language :: Python :: 3',\n",
       "  'Programming Language :: Python :: 3.4',\n",
       "  'Programming Language :: Python :: 3.5',\n",
       "  'Programming Language :: Python :: 3.6',\n",
       "  'Programming Language :: Python :: Implementation :: CPython',\n",
       "  'Programming Language :: Python :: Implementation :: PyPy',\n",
       "  'Topic :: Software Development :: Build Tools'],\n",
       " 'description': \"pip\\n===\\n\\nThe `PyPA recommended`_ tool for installing Python packages.\\n\\n.. image:: https://img.shields.io/pypi/v/pip.svg\\n   :target: https://pypi.org/project/pip/\\n\\n.. image:: https://img.shields.io/travis/pypa/pip/master.svg?label=travis-ci\\n   :target: https://travis-ci.org/pypa/pip\\n\\n.. image:: https://img.shields.io/appveyor/ci/pypa/pip.svg?label=appveyor-ci\\n   :target: https://ci.appveyor.com/project/pypa/pip/history\\n\\n.. image:: https://readthedocs.org/projects/pip/badge/?version=latest\\n   :target: https://pip.pypa.io/en/latest\\n\\n* `Installation`_\\n* `Documentation`_\\n* `Changelog`_\\n* `GitHub Page`_\\n* `Issue Tracking`_\\n* `User mailing list`_\\n* `Dev mailing list`_\\n* User IRC: #pypa on Freenode.\\n* Dev IRC: #pypa-dev on Freenode.\\n\\nCode of Conduct\\n---------------\\n\\nEveryone interacting in the pip project's codebases, issue trackers, chat\\nrooms and mailing lists is expected to follow the `PyPA Code of Conduct`_.\\n\\n.. _PyPA recommended: https://packaging.python.org/en/latest/current/\\n.. _Installation: https://pip.pypa.io/en/stable/installing.html\\n.. _Documentation: https://pip.pypa.io/en/stable/\\n.. _Changelog: https://pip.pypa.io/en/stable/news.html\\n.. _GitHub Page: https://github.com/pypa/pip\\n.. _Issue Tracking: https://github.com/pypa/pip/issues\\n.. _User mailing list: https://groups.google.com/forum/#!forum/python-virtualenv\\n.. _Dev mailing list: https://groups.google.com/forum/#!forum/pypa-dev\\n.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/\\n\\n\\n\",\n",
       " 'description_content_type': '',\n",
       " 'docs_url': None,\n",
       " 'download_count': 243085535,\n",
       " 'download_url': '',\n",
       " 'downloads': {'last_day': -1, 'last_month': -1, 'last_week': -1},\n",
       " 'home_page': 'https://pip.pypa.io/',\n",
       " 'keywords': 'distutils easy_install egg setuptools wheel virtualenv',\n",
       " 'license': 'MIT',\n",
       " 'maintainer': '',\n",
       " 'maintainer_email': '',\n",
       " 'name': 'pip',\n",
       " 'package_url': 'https://pypi.org/project/pip/',\n",
       " 'platform': '',\n",
       " 'project_url': 'https://pypi.org/project/pip/',\n",
       " 'project_urls': {'Homepage': 'https://pip.pypa.io/'},\n",
       " 'release_url': 'https://pypi.org/project/pip/18.0/',\n",
       " 'requires_dist': None,\n",
       " 'requires_python': '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*',\n",
       " 'summary': 'The PyPA recommended tool for installing Python packages.',\n",
       " 'version': '18.0'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "db = sqlite_utils.Database(\"/tmp/pypi.db\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Table packages>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db[\"packages\"].insert_all(\n",
    "    records, pk=\"name\",\n",
    "    column_order=[\"name\", \"summary\", \"download_count\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Table packages>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db[\"packages\"].enable_fts([\"name\", \"summary\", \"keywords\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I then ran `datasette /tmp/pypi.db` to demonstrate the newly created database.\n",
    "\n",
    "Documentation on sqlite-utils is available at https://sqlite-utils.readthedocs.io/en/latest/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}