Skip to content

Instantly share code, notes, and snippets.

@abitrolly

abitrolly/gql-pagination.ipynb

Last active June 21, 2021 09:43
Show Gist options
  • Save abitrolly/2b5351962b3b6ed57ce7141599fcde7e to your computer and use it in GitHub Desktop.
Save abitrolly/2b5351962b3b6ed57ce7141599fcde7e to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
gql-pagination.ipynb
{"cells":[{"metadata":{},"id":"verified-statement","cell_type":"markdown","source":"[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gist/abitrolly/2b5351962b3b6ed57ce7141599fcde7e/HEAD?filepath=gql-pagination.ipynb)"},{"metadata":{"trusted":true},"id":"challenging-material","cell_type":"code","source":"!pip install -U --pre gql[aiohttp]\n!pip install dpath","execution_count":1,"outputs":[{"output_type":"stream","text":"Collecting gql[aiohttp]\n Downloading gql-3.0.0a6.tar.gz (86 kB)\n\u001b[K |████████████████████████████████| 86 kB 2.9 MB/s eta 0:00:011\n\u001b[?25hCollecting graphql-core<3.2,>=3.1.5\n Downloading graphql_core-3.1.5-py3-none-any.whl (188 kB)\n\u001b[K |████████████████████████████████| 188 kB 6.3 MB/s eta 0:00:01\n\u001b[?25hCollecting yarl<2.0,>=1.6\n Downloading yarl-1.6.3-cp37-cp37m-manylinux2014_x86_64.whl (294 kB)\n\u001b[K |████████████████████████████████| 294 kB 8.3 MB/s eta 0:00:01\n\u001b[?25hCollecting aiohttp<3.8.0,>=3.7.1\n Downloading aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_x86_64.whl (1.3 MB)\n\u001b[K |████████████████████████████████| 1.3 MB 10.5 MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /srv/conda/envs/notebook/lib/python3.7/site-packages (from aiohttp<3.8.0,>=3.7.1->gql[aiohttp]) (20.3.0)\nRequirement already satisfied: typing-extensions>=3.6.5 in /srv/conda/envs/notebook/lib/python3.7/site-packages (from aiohttp<3.8.0,>=3.7.1->gql[aiohttp]) (3.7.4.3)\nCollecting async-timeout<4.0,>=3.0\n Downloading async_timeout-3.0.1-py3-none-any.whl (8.2 kB)\nCollecting multidict<7.0,>=4.5\n Downloading multidict-5.1.0-cp37-cp37m-manylinux2014_x86_64.whl (142 kB)\n\u001b[K |████████████████████████████████| 142 kB 22.0 MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied: chardet<5.0,>=2.0 in /srv/conda/envs/notebook/lib/python3.7/site-packages (from aiohttp<3.8.0,>=3.7.1->gql[aiohttp]) (4.0.0)\nRequirement already satisfied: idna>=2.0 in /srv/conda/envs/notebook/lib/python3.7/site-packages (from yarl<2.0,>=1.6->gql[aiohttp]) (2.10)\nBuilding wheels for collected packages: gql\n Building wheel for gql (setup.py) ... \u001b[?25ldone\n\u001b[?25h Created wheel for gql: filename=gql-3.0.0a6-py2.py3-none-any.whl size=34853 sha256=729e900aa4fc7d63e09b46a1710e7fde1ac4409ada412f7d5f46fd1e1a0ea355\n Stored in directory: /home/jovyan/.cache/pip/wheels/0f/60/75/8de694b1018d6b15ae7053ca8c0e423be3e02f2b56ef73a98a\nSuccessfully built gql\nInstalling collected packages: multidict, yarl, graphql-core, async-timeout, gql, aiohttp\nSuccessfully installed aiohttp-3.7.4.post0 async-timeout-3.0.1 gql-3.0.0a6 graphql-core-3.1.5 multidict-5.1.0 yarl-1.6.3\nCollecting dpath\n Downloading dpath-2.0.1.tar.gz (21 kB)\nBuilding wheels for collected packages: dpath\n Building wheel for dpath (setup.py) ... \u001b[?25ldone\n\u001b[?25h Created wheel for dpath: filename=dpath-2.0.1-py3-none-any.whl size=15138 sha256=33d9aa2753cc6789a3e4870441734be2df37ee377120b4d5b6fa2d15023b839a\n Stored in directory: /home/jovyan/.cache/pip/wheels/bc/f8/ae/bc69cb5f61393ebf9ade4cde41d1a813d35bfe78263a26f99e\nSuccessfully built dpath\nInstalling collected packages: dpath\nSuccessfully installed dpath-2.0.1\n","name":"stdout"}]},{"metadata":{"trusted":true},"id":"reflected-preference","cell_type":"code","source":"from gql import gql, Client\nfrom gql.transport.aiohttp import AIOHTTPTransport","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"id":"duplicate-developer","cell_type":"code","source":"# code by @RolandColored with modifications\n# https://github.com/graphql-python/gql/issues/206#issuecomment-850247771\n\nimport os\n\nfrom functools import reduce\n\nclass GitLabGraphQL:\n def __init__(self, auth=False):\n if auth:\n transport = AIOHTTPTransport(url=\"https://gitlab.com/api/graphql\", headers={'Authorization': 'Bearer ' + os.environ['GITLAB_TOKEN']})\n else:\n transport = AIOHTTPTransport(url=\"https://gitlab.com/api/graphql\")\n self.client = Client(transport=transport, fetch_schema_from_transport=True)\n\n def _query_with_pagination(self, query, page_info_path):\n page_info = {\n 'hasNextPage': True,\n 'endCursor': None\n }\n\n while page_info['hasNextPage']:\n result = self.client.execute(query, variable_values={'cursor': page_info['endCursor']})\n page_info = reduce(dict.get, page_info_path, result)\n yield result\n\n def query_projects_jobs(self) -> (int, int):\n query = gql(\"\"\"\n query getGroupJobs($cursor: String) {\n group(fullPath: \"abitrolly\") {\n projects(includeSubgroups: true, first: 1, after: $cursor) {\n nodes {\n jobs {\n count\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n }\n }\n \"\"\")\n\n projects = self._query_with_pagination(query, ('group', 'projects', 'pageInfo'))\n return projects","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"id":"controlled-decrease","cell_type":"code","source":"for i, e in enumerate(GitLabGraphQL().query_projects_jobs()):\n print(i, e)","execution_count":4,"outputs":[{"output_type":"error","ename":"TypeError","evalue":"descriptor 'get' requires a 'dict' object but received a 'NoneType'","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)","File \u001b[0;34m/srv/conda/envs/notebook/lib/python3.7/site-packages/IPython/core/interactiveshell.py\u001b[0m, in \u001b[0;32mrun_code\u001b[0m:\nLine \u001b[0;34m3427\u001b[0m: exec(code_obj, \u001b[36mself\u001b[39;49;00m.user_global_ns, \u001b[36mself\u001b[39;49;00m.user_ns)\n","File \u001b[0;34m<ipython-input-4-77c819e5d368>\u001b[0m, in \u001b[0;32m<module>\u001b[0m:\nLine \u001b[0;34m1\u001b[0m: \u001b[34mfor\u001b[39;49;00m i, e \u001b[35min\u001b[39;49;00m \u001b[36menumerate\u001b[39;49;00m(GitLabGraphQL().query_projects_jobs()):\n","File \u001b[0;34m<ipython-input-3-5793cb3f9360>\u001b[0m, in \u001b[0;32m_query_with_pagination\u001b[0m:\nLine \u001b[0;34m24\u001b[0m: page_info = reduce(\u001b[36mdict\u001b[39;49;00m.get, page_info_path, result)\n","\u001b[0;31mTypeError\u001b[0m: descriptor 'get' requires a 'dict' object but received a 'NoneType'\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m"]}]},{"metadata":{"trusted":true},"id":"western-adapter","cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"file_extension":".py","mimetype":"text/x-python","name":"python","version":"3.7.10"},"kernelspec":{"name":"xpython","display_name":"Python 3.7 (XPython)","language":"python"}},"nbformat":4,"nbformat_minor":5}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment