Demonstrate accessing ESnet's SNMP GraphQL service

poke_esnet_graphql.sh

#!/usr/bin/env bash

CSRF_TOK=$(curl --verbose -D - https://my.es.net/ 2>/dev/null | grep -o 'csrftoken=[^;]*' | cut -d= -f2)
echo "CSRF token = [$CSRF_TOK]"

curl \
    --referer https://my.es.net/nersc-400g \
    -X POST \
    -b "csrftoken=$CSRF_TOK" \
    -H "X-CSRFToken: $CSRF_TOK" \
    -H "Content-Type: application/json" \
    -H "Accept: application/json" \
    --data '{"query":"{ mapTopology(name: \"nersc-400g\"){ paths { traffic(beginTime: \"2019-04-01T00:00:00.000Z\", endTime: \"2019-04-01T01:00:00.000Z\") } } }"}' \
    https://my.es.net/graphql

poke_esnet_graphql_400g.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demonstrate ESnet's GraphQL API\n",
    "\n",
    "This notebook demonstrates how to access SNMP data from ESnet's newer GraphQL query interface."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "import numpy\n",
    "import pandas\n",
    "import matplotlib\n",
    "matplotlib.rcParams['font.size'] = 16\n",
    "\n",
    "import requests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define input parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "START_TIME = datetime.datetime(2019, 4, 1, 0, 0, 0)\n",
    "END_TIME = datetime.datetime(2019, 4, 1, 1, 0, 0) - datetime.timedelta(seconds=1)\n",
    "\n",
    "URL = 'https://my.es.net/'\n",
    "GRAPHQL_URL = URL + \"graphql\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert datetime objects into strings to be inserted into the GraphQL query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_str = datetime.datetime.utcfromtimestamp(time.mktime(START_TIME.timetuple())).strftime(\"%Y-%m-%dT%H:%M:%S.000Z\")\n",
    "end_str = datetime.datetime.utcfromtimestamp(time.mktime(END_TIME.timetuple())).strftime(\"%Y-%m-%dT%H:%M:%S.000Z\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build the actual GraphQL query as a string; this string will be embedded into JSON before being sent over REST."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERY = \"\"\"\n",
    "{\n",
    "    mapTopology(name: \"nersc-400g\") {\n",
    "        paths {\n",
    "            traffic(beginTime: \"%s\",\n",
    "                    endTime: \"%s\")\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\"\"\" % (start_str, end_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Establish the HTTP session, get the required referral and CSRF token, and dispatch the GraphQL query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "session = requests.Session()\n",
    "\n",
    "# Get the CSRF token\n",
    "response = session.get(URL)\n",
    "response.raise_for_status()\n",
    "csrf = session.cookies['csrftoken']\n",
    "\n",
    "# Set the CSRF token and referer for all subsequent queries\n",
    "session.headers.update({\n",
    "    \"Referer\": URL,\n",
    "    \"X-CSRFToken\": csrf,\n",
    "    \"Accept\": \"application/json\",\n",
    "})\n",
    "\n",
    "# Issue the query\n",
    "ret = session.post(\n",
    "    GRAPHQL_URL,\n",
    "    json={\"query\": QUERY},\n",
    "    headers={\n",
    "        \"Content-Type\": \"application/json\",\n",
    "    })\n",
    "\n",
    "# Parse the resulting JSON into a Python dictionary\n",
    "raw_rest = ret.json()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert the raw output from GraphQL into a dictionary of Series, then convert those Series into a single DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = []\n",
    "\n",
    "for blob in raw_rest['data']['mapTopology']['paths']:\n",
    "    blob_data = json.loads(blob['traffic'])\n",
    "#   print(json.dumps(blob_data, indent=4, sort_keys=True))\n",
    "    x = [datetime.datetime.fromtimestamp(point[0] / 1000) for point in blob_data[\"points\"]]\n",
    "    for i in range(len(blob_data['columns']) - 1):\n",
    "        columns.append(pandas.Series([point[i+1] for point in blob_data[\"points\"]], index=x))\n",
    "        columns[-1].name = blob_data['name'] + \"_\" + blob_data['columns'][i+1]\n",
    "\n",
    "dataframe = pandas.concat(columns, axis=1, keys=[s.name for s in columns])\n",
    "dataframe.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the data to show aggregate read and write"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_dfs = {\n",
    "    \"atoz\": dataframe[[x for x in dataframe.columns if x.endswith('_atoz')]],\n",
    "    \"ztoa\": dataframe[[x for x in dataframe.columns if x.endswith('_ztoa')]],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = matplotlib.pyplot.subplots(figsize=(8, 6))\n",
    "\n",
    "UNIT = 2**30\n",
    "UNIT_LABEL = \"GiB\"\n",
    "\n",
    "x = next(iter(plot_dfs.values())).index.values\n",
    "y = {\n",
    "    'atoz': numpy.array([0.0 for y in range(len(x))]),\n",
    "    'ztoa': numpy.array([0.0 for y in range(len(x))]),\n",
    "}\n",
    "\n",
    "for rw, plot_df in plot_dfs.items():\n",
    "    for isys, system in enumerate(plot_df.columns):\n",
    "        this = y[rw] + plot_df[system].values / UNIT * (-1.0 if rw == 'atoz' else 1.0)\n",
    "        ax.fill_between(x, y[rw], this,\n",
    "                        label=\"%s\" % (system.replace(\"_\", \" \")) if rw == 'atoz' else None,\n",
    "                        color='C%d' % (isys % len(plot_df.columns))\n",
    "                       )\n",
    "        y[rw] = this\n",
    "\n",
    "# Make the y axis mirrored\n",
    "xmin, xmax = ax.get_xlim()\n",
    "ymin, ymax = ax.get_ylim()\n",
    "ymax = max(abs(ymin), ymax)\n",
    "ax.set_ylim(-ymax, ymax)\n",
    "\n",
    "# Draw the zero point\n",
    "ax.plot((xmin, xmax), (0, 0), ls='-', color='black')\n",
    "ax.set_xlim(min(x), max(x))\n",
    "\n",
    "# Make the tick marks more sensible\n",
    "ax.grid()\n",
    "ax.set_axisbelow(True)\n",
    "ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter(\"%b-%d %H:%M\"))\n",
    "fig.autofmt_xdate()\n",
    "\n",
    "legend = ax.legend(ncol=2, loc='lower left') # 'upper right' if writes are greater than reads\n",
    "#for ltxt in legend.get_texts():\n",
    "#    old_text = ltxt.get_text()\n",
    "#    new_text = SYSTEM_NAMES.get(old_text, old_text)\n",
    "#    ltxt.set_text(new_text)\n",
    "\n",
    "\n",
    "ydirections = [\n",
    "    {\n",
    "        'y': 0.25,\n",
    "        's': '%s atoz' % UNIT_LABEL,\n",
    "    },\n",
    "    {\n",
    "        'y': 0.75,\n",
    "        's': '%s ztoa' % UNIT_LABEL,\n",
    "    },\n",
    "]\n",
    "for kwargs in ydirections:\n",
    "    ax.text(x=-0.1,\n",
    "            ha='center',\n",
    "            va='center',\n",
    "            rotation=90,\n",
    "            transform=ax.transAxes,\n",
    "             **kwargs)\n",
    "    \n",
    "_ = ax.set_yticklabels([\"%d\" % abs(y) for y in ax.get_yticks()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

poke_esnet_graphql_nersc.ipynb