Skip to content

Instantly share code, notes, and snippets.

@Evan-Kim2028

Evan-Kim2028/query_size_bug

Created May 13, 2023 18:59
Show Gist options
  • Save Evan-Kim2028/688639c1d71928ae4299df38afe24033 to your computer and use it in GitHub Desktop.
Save Evan-Kim2028/688639c1d71928ae4299df38afe24033 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
query_size_bug
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from dotenv import load_dotenv\n",
"import os\n",
"from subgrounds import Subgrounds\n",
"\n",
"load_dotenv()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"sg = Subgrounds(headers={\"Playgrounds-Api-Key\": os.environ.get('PG_KEY')})\n",
"\n",
"# https://gateway.thegraph.com/api/[api-key]/subgraphs/id/ELUcwgpm14LKPLrBRuVvPvNKHQ9HvwmtKgKSH6123cr7\n",
"# look at header behavior with hosted vs decentralized queries\n",
"\n",
"# instantiate Dex class with subgraph key\n",
"eth = 'G3JZhmKKHC4mydRzD6kSz5fCWve5WDYYCyTFSJyv3SD5' # indexed on ethereum\n",
"\n",
"subgraph = sg.load_subgraph(\n",
" # f\"https://api.playgrounds.network/v1/proxy/subgraphs/id/{arb}\" \n",
"f\"https://api.playgrounds.network/v1/proxy/subgraphs/id/{eth}\" \n",
")\n",
"\n",
"# arbitrum doesn't have 'swaps',\n",
"# ethereum doesn't have marketHourlySnapshots"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"my_qp = subgraph.Query.swaps(first=100000)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"ename": "PaginationError",
"evalue": "502 Server Error: Bad Gateway for url: https://api.playgrounds.network/v1/proxy/subgraphs/id/G3JZhmKKHC4mydRzD6kSz5fCWve5WDYYCyTFSJyv3SD5",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/pagination/pagination.py:81\u001b[0m, in \u001b[0;36mpaginate\u001b[0;34m(schema, doc, pagination_strategy, headers)\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m---> 81\u001b[0m page_data \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39;49mquery(\n\u001b[1;32m 82\u001b[0m url\u001b[39m=\u001b[39;49mdoc\u001b[39m.\u001b[39;49murl,\n\u001b[1;32m 83\u001b[0m query_str\u001b[39m=\u001b[39;49mdoc\u001b[39m.\u001b[39;49mgraphql,\n\u001b[1;32m 84\u001b[0m variables\u001b[39m=\u001b[39;49mdoc\u001b[39m.\u001b[39;49mvariables \u001b[39m|\u001b[39;49m args,\n\u001b[1;32m 85\u001b[0m headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[1;32m 86\u001b[0m )\n\u001b[1;32m 87\u001b[0m data \u001b[39m=\u001b[39m merge(data, page_data)\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/client.py:185\u001b[0m, in \u001b[0;36mquery\u001b[0;34m(url, query_str, variables, headers)\u001b[0m\n\u001b[1;32m 175\u001b[0m resp \u001b[39m=\u001b[39m requests\u001b[39m.\u001b[39mpost(\n\u001b[1;32m 176\u001b[0m url,\n\u001b[1;32m 177\u001b[0m json\u001b[39m=\u001b[39m(\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 182\u001b[0m headers\u001b[39m=\u001b[39mdefault_header() \u001b[39m|\u001b[39m headers,\n\u001b[1;32m 183\u001b[0m )\n\u001b[0;32m--> 185\u001b[0m resp\u001b[39m.\u001b[39;49mraise_for_status()\n\u001b[1;32m 187\u001b[0m \u001b[39mtry\u001b[39;00m:\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/requests/models.py:1021\u001b[0m, in \u001b[0;36mResponse.raise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[39mif\u001b[39;00m http_error_msg:\n\u001b[0;32m-> 1021\u001b[0m \u001b[39mraise\u001b[39;00m HTTPError(http_error_msg, response\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m)\n",
"\u001b[0;31mHTTPError\u001b[0m: 502 Server Error: Bad Gateway for url: https://api.playgrounds.network/v1/proxy/subgraphs/id/G3JZhmKKHC4mydRzD6kSz5fCWve5WDYYCyTFSJyv3SD5",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mPaginationError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df1 \u001b[39m=\u001b[39m sg\u001b[39m.\u001b[39;49mquery_df(my_qp)\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/subgrounds.py:414\u001b[0m, in \u001b[0;36mSubgrounds.query_df\u001b[0;34m(self, fpaths, columns, concat, pagination_strategy)\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Same as :func:`Subgrounds.query` but formats the response data into a\u001b[39;00m\n\u001b[1;32m 348\u001b[0m \u001b[39mPandas DataFrame. If the response data cannot be flattened to a single query\u001b[39;00m\n\u001b[1;32m 349\u001b[0m \u001b[39m(e.g.: when querying multiple list fields that return different entities),\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 410\u001b[0m \u001b[39m 9 1643213196 2610.686563\u001b[39;00m\n\u001b[1;32m 411\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 413\u001b[0m fpaths \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m([fpaths] \u001b[39m|\u001b[39m traverse \u001b[39m|\u001b[39m \u001b[39mmap\u001b[39m(FieldPath\u001b[39m.\u001b[39m_auto_select) \u001b[39m|\u001b[39m traverse)\n\u001b[0;32m--> 414\u001b[0m json_data \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mquery_json(fpaths, pagination_strategy\u001b[39m=\u001b[39;49mpagination_strategy)\n\u001b[1;32m 415\u001b[0m \u001b[39mreturn\u001b[39;00m df_of_json(json_data, fpaths, columns, concat)\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/subgrounds.py:316\u001b[0m, in \u001b[0;36mSubgrounds.query_json\u001b[0;34m(self, fpaths, pagination_strategy)\u001b[0m\n\u001b[1;32m 314\u001b[0m fpaths \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m([fpaths] \u001b[39m|\u001b[39m traverse \u001b[39m|\u001b[39m \u001b[39mmap\u001b[39m(FieldPath\u001b[39m.\u001b[39m_auto_select) \u001b[39m|\u001b[39m traverse)\n\u001b[1;32m 315\u001b[0m req \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmk_request(fpaths)\n\u001b[0;32m--> 316\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mexecute(req, pagination_strategy\u001b[39m=\u001b[39;49mpagination_strategy)\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/subgrounds.py:229\u001b[0m, in \u001b[0;36mSubgrounds.execute\u001b[0;34m(self, req, pagination_strategy)\u001b[0m\n\u001b[1;32m 225\u001b[0m \u001b[39mreturn\u001b[39;00m transform\u001b[39m.\u001b[39mtransform_response(req, data)\n\u001b[1;32m 227\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mFalse\u001b[39;00m \u001b[39m# Suppress mypy missing return statement warning\u001b[39;00m\n\u001b[0;32m--> 229\u001b[0m \u001b[39mreturn\u001b[39;00m transform_req(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mglobal_transforms, req)\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/subgrounds.py:214\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.transform_req\u001b[0;34m(transforms, req)\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[39mmatch\u001b[39;00m transforms:\n\u001b[1;32m 213\u001b[0m \u001b[39mcase\u001b[39;00m []:\n\u001b[0;32m--> 214\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlist\u001b[39;49m(\n\u001b[1;32m 215\u001b[0m req\u001b[39m.\u001b[39;49mdocuments\n\u001b[1;32m 216\u001b[0m \u001b[39m|\u001b[39;49m \u001b[39mmap\u001b[39;49m(\n\u001b[1;32m 217\u001b[0m \u001b[39mlambda\u001b[39;49;00m doc: transform_doc(\n\u001b[1;32m 218\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msubgraphs[doc\u001b[39m.\u001b[39;49murl]\u001b[39m.\u001b[39;49m_transforms, doc\n\u001b[1;32m 219\u001b[0m )\n\u001b[1;32m 220\u001b[0m )\n\u001b[1;32m 221\u001b[0m )\n\u001b[1;32m 222\u001b[0m \u001b[39mcase\u001b[39;00m [transform, \u001b[39m*\u001b[39mrest]:\n\u001b[1;32m 223\u001b[0m new_req \u001b[39m=\u001b[39m transform\u001b[39m.\u001b[39mtransform_request(req)\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/subgrounds.py:217\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.transform_req.<locals>.<lambda>\u001b[0;34m(doc)\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[39mmatch\u001b[39;00m transforms:\n\u001b[1;32m 213\u001b[0m \u001b[39mcase\u001b[39;00m []:\n\u001b[1;32m 214\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlist\u001b[39m(\n\u001b[1;32m 215\u001b[0m req\u001b[39m.\u001b[39mdocuments\n\u001b[1;32m 216\u001b[0m \u001b[39m|\u001b[39m \u001b[39mmap\u001b[39m(\n\u001b[0;32m--> 217\u001b[0m \u001b[39mlambda\u001b[39;00m doc: transform_doc(\n\u001b[1;32m 218\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msubgraphs[doc\u001b[39m.\u001b[39;49murl]\u001b[39m.\u001b[39;49m_transforms, doc\n\u001b[1;32m 219\u001b[0m )\n\u001b[1;32m 220\u001b[0m )\n\u001b[1;32m 221\u001b[0m )\n\u001b[1;32m 222\u001b[0m \u001b[39mcase\u001b[39;00m [transform, \u001b[39m*\u001b[39mrest]:\n\u001b[1;32m 223\u001b[0m new_req \u001b[39m=\u001b[39m transform\u001b[39m.\u001b[39mtransform_request(req)\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/subgrounds.py:204\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.transform_doc\u001b[0;34m(transforms, doc)\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[39mcase\u001b[39;00m [transform, \u001b[39m*\u001b[39mrest]:\n\u001b[1;32m 203\u001b[0m new_doc \u001b[39m=\u001b[39m transform\u001b[39m.\u001b[39mtransform_document(doc)\n\u001b[0;32m--> 204\u001b[0m data \u001b[39m=\u001b[39m transform_doc(rest, new_doc)\n\u001b[1;32m 205\u001b[0m \u001b[39mreturn\u001b[39;00m transform\u001b[39m.\u001b[39mtransform_response(doc, data)\n\u001b[1;32m 207\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mFalse\u001b[39;00m\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/subgrounds.py:204\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.transform_doc\u001b[0;34m(transforms, doc)\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[39mcase\u001b[39;00m [transform, \u001b[39m*\u001b[39mrest]:\n\u001b[1;32m 203\u001b[0m new_doc \u001b[39m=\u001b[39m transform\u001b[39m.\u001b[39mtransform_document(doc)\n\u001b[0;32m--> 204\u001b[0m data \u001b[39m=\u001b[39m transform_doc(rest, new_doc)\n\u001b[1;32m 205\u001b[0m \u001b[39mreturn\u001b[39;00m transform\u001b[39m.\u001b[39mtransform_response(doc, data)\n\u001b[1;32m 207\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mFalse\u001b[39;00m\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/subgrounds.py:201\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.transform_doc\u001b[0;34m(transforms, doc)\u001b[0m\n\u001b[1;32m 199\u001b[0m \u001b[39mmatch\u001b[39;00m transforms:\n\u001b[1;32m 200\u001b[0m \u001b[39mcase\u001b[39;00m []:\n\u001b[0;32m--> 201\u001b[0m \u001b[39mreturn\u001b[39;00m execute_document(doc)\n\u001b[1;32m 202\u001b[0m \u001b[39mcase\u001b[39;00m [transform, \u001b[39m*\u001b[39mrest]:\n\u001b[1;32m 203\u001b[0m new_doc \u001b[39m=\u001b[39m transform\u001b[39m.\u001b[39mtransform_document(doc)\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/subgrounds.py:186\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.execute_document\u001b[0;34m(doc)\u001b[0m\n\u001b[1;32m 182\u001b[0m subgraph: Subgraph \u001b[39m=\u001b[39m \u001b[39mnext\u001b[39m(\n\u001b[1;32m 183\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msubgraphs\u001b[39m.\u001b[39mvalues() \u001b[39m|\u001b[39m where(\u001b[39mlambda\u001b[39;00m sg: sg\u001b[39m.\u001b[39m_url \u001b[39m==\u001b[39m doc\u001b[39m.\u001b[39murl)\n\u001b[1;32m 184\u001b[0m )\n\u001b[1;32m 185\u001b[0m \u001b[39mif\u001b[39;00m pagination_strategy \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m subgraph\u001b[39m.\u001b[39m_is_subgraph:\n\u001b[0;32m--> 186\u001b[0m \u001b[39mreturn\u001b[39;00m paginate(\n\u001b[1;32m 187\u001b[0m subgraph\u001b[39m.\u001b[39;49m_schema,\n\u001b[1;32m 188\u001b[0m doc,\n\u001b[1;32m 189\u001b[0m pagination_strategy\u001b[39m=\u001b[39;49mpagination_strategy,\n\u001b[1;32m 190\u001b[0m headers\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mheaders,\n\u001b[1;32m 191\u001b[0m )\n\u001b[1;32m 192\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 193\u001b[0m \u001b[39mreturn\u001b[39;00m client\u001b[39m.\u001b[39mquery(\n\u001b[1;32m 194\u001b[0m doc\u001b[39m.\u001b[39murl, doc\u001b[39m.\u001b[39mgraphql, variables\u001b[39m=\u001b[39mdoc\u001b[39m.\u001b[39mvariables, headers\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mheaders\n\u001b[1;32m 195\u001b[0m )\n",
"File \u001b[0;32m~/Documents/github/subgraph-query-portal/.venv/lib/python3.10/site-packages/subgrounds/pagination/pagination.py:92\u001b[0m, in \u001b[0;36mpaginate\u001b[0;34m(schema, doc, pagination_strategy, headers)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m 91\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m exn:\n\u001b[0;32m---> 92\u001b[0m \u001b[39mraise\u001b[39;00m PaginationError(exn\u001b[39m.\u001b[39margs[\u001b[39m0\u001b[39m], strategy)\n\u001b[1;32m 94\u001b[0m \u001b[39mreturn\u001b[39;00m data\n\u001b[1;32m 96\u001b[0m \u001b[39mexcept\u001b[39;00m SkipPagination:\n",
"\u001b[0;31mPaginationError\u001b[0m: 502 Server Error: Bad Gateway for url: https://api.playgrounds.network/v1/proxy/subgraphs/id/G3JZhmKKHC4mydRzD6kSz5fCWve5WDYYCyTFSJyv3SD5"
]
}
],
"source": [
"df1 = sg.query_df(my_qp)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment