tf-benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "from time import time\n",
    "import numpy as np\n",
    "from tqdm import tqdm, trange\n",
    "from matplotlib import pyplot as plt\n",
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a simple tensorflow graph\n",
    "with tf.Graph().as_default() as graph:\n",
    "    placeholder = tf.placeholder(dtype=tf.float32)\n",
    "    result = tf.sin(placeholder)\n",
    "    \n",
    "session = tf.Session(graph=graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [05:39<00:00, 30.95s/it]\n"
     ]
    }
   ],
   "source": [
    "num_runs = 5\n",
    "k = 16  # Use a batch size of up to 2 ** k\n",
    "\n",
    "# Create data to feed to tensorflow that is sufficiently large to be problematic\n",
    "data = np.random.normal(0, 1, size=(2 ** k, 32, 32))\n",
    "num_batches = 2 ** np.arange(k)\n",
    "batch_sizes = 2 ** k / num_batches\n",
    "mean_times = []\n",
    "std_times = []\n",
    "\n",
    "# Iterate over the different batch sizes\n",
    "for num in tqdm(num_batches):\n",
    "    times = []\n",
    "    # Use multiple runs to estimate the mean and standard error of the mean\n",
    "    for run in range(num_runs):\n",
    "        start = time()\n",
    "        batches = np.split(data, num)\n",
    "        for batch in batches:\n",
    "            session.run(result, {placeholder: batch})\n",
    "        end = time()\n",
    "        times.append(end - start)\n",
    "        \n",
    "    # Store the results\n",
    "    mean_times.append(np.mean(times))\n",
    "    std_times.append(np.std(times) / np.sqrt(num_runs - 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f0a6c6ec160>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAETCAYAAAAyK6EVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu8lGW5//HPxUHUAOWgUmKgohIHRVNQzM0yD7jBzEqt\n1BJEM8u03LvY7q1BvSoPtbV2ltuUoPKs/UjFPGS7pSgHTURYgAgeEtKEFGRxEJF1//64Zpw1y3WY\nNcwz9zOzvu/Xa17rmZln5rnWzWKuuc8WQkBERCSrU+wAREQkXZQYREQkjxKDiIjkUWIQEZE8Sgwi\nIpJHiUFERPIkmhjMrL+Z/Z+ZLTGzxWZ2cebxXmb2iJktN7OHzWy3JOMQEZHCWZLzGMysH9AvhLDQ\nzLoDzwCfBiYCb4YQrjGzyUCvEMJ/JBaIiIgULNEaQwjhHyGEhZnjjcAyoD+eHH6TOe03wKlJxiEi\nIoVLtMaQdyGzgUAtMAxYFULo1ei5t0IIvcsSiIiItKpLOS6SaUa6B7gkhLDRzJpmo2azUzPniYhI\nAUIIVuxrEx+VZGZd8KTwuxDCvZmH3zCzvTLP9wPWtPT6EELJblOmTCnp+a0939xzbT3W9PnWnqu2\nsmjPfZWFykJl0fr9HVWO4aq/BpaGEH7W6LH7gAmZ43OAe5u+KAk1NTUlPb+155t7rq3Hmj7f3njb\nI21l0d77paSyKP69VRaFn19RZVHKDNv0BhwNbAcWAs8CC4CTgN7Ao8By4BFg9xZeH8RNmTIldgip\nobLIUVnkqCxyMp+dRX92J9rHEEJ4EujcwtPHJ3ntapPkN6NKo7LIUVnkqCxKp2yjkophZiHN8YmI\npJGZEdLc+SwiIpVFiUFERPIoMYiISB4lBhERyaPEICIieZQYREQkjxKDiIjkUWIQEZE8SgwiIpJH\niUFERPIoMYiISB4lBhERyaPEICIieZQYREQkT1n2fBYRkcLV1vote5zdaqKmJnecJO3HICKSYmbQ\n3o9B7ccgIiIlpcQgIiJ5lBhERCSPEoOIiORRYhARkTxKDCIikkeJQURE8igxiIhIHiUGERHJo8Qg\nIiJ5lBhERCSPEoOIiORRYhARkTxKDCIikkeJQURE8igxiIhIHiUGERHJo8QgIiJ5lBhERCSPEoOI\niORRYhARkTxKDCIikkeJQURE8igxiIhIHiUGERHJo8QgIiJ5lBhERCSPEoOIiORRYhARkTxKDCIi\nkifRxGBm08zsDTNb1OixKWa22swWZG4nJRmDiIi0T9I1hunA2GYevzaEcFjm9lDCMYiISDskmhhC\nCE8A65p5ypK8roiIFC9WH8NFZrbQzG42s90ixSAiIs3oEuGavwS+H0IIZvYD4FpgUksnT5069f3j\nmpoaampqko5PRKSi1NbWUltbW7L3sxBCyd6s2QuYDQDuDyEc3J7nMs+HpOMTEUkzM2jvx6CZEUIo\nusm+HE1JRqM+BTPr1+i5zwJ1ZYhBREQKlGhTkpndBtQAfczsVWAKcKyZjQAagFeAC5KMQURE2ifx\npqQdoaYkEenoqrUpSUREKogSg4iI5FFiEBGRPEoMIiKSR4lBRCSl6uvzf5aLEoOISArV18PRR/vx\nJz5R3uSgxCAikkJ1dbBsmR8vWwZLlpTv2koMIiIpNGwY7LmnHw8ZAkOHlu/aSgwiIinUowccc4wf\nz57t98tFiUFEJKWeecZ/ljMpgBKDiEgqvfkmrFkT59pKDCIiKTR/PhxxRJxrKzGIiKTQ/PkwalSc\naysxiIik0Lx5cOSRca6tZbdFRFKmoQH69IHly2GvvbTstohIh/fCC9CrV24eQ7kpMYiIpEzMZiRQ\nYhARSZ2YHc+gxCAikjqxawzqfBYRSZFNm7xv4a23oFs37fksItLhPfMMDB/uSSEWJQYRkRSZNy9u\n/wIoMYiIpMr8+XH7F0CJQUQkVVRjEBGR961eDdu2wb77xo1DiUFEJCWyw1St6PFEpaHEICKSErEn\ntmUpMYiIpETsiW1ZmuAmIpIC27b5wnmvvQY9e+Ye1wQ3EZEOavFiGDgwPynEosQgIpICaRimmqXE\nICKSAmmY2JalxCAikgJp6XgGdT6LiET31lvev7BuHXTunP9cajufzWwvM5tmZg9m7g8xs0nFXlRE\nRHKeegoOP/yDSSGWQpuSZgAPAx/J3H8B+GYSAYmIdDRpmdiW1aXA8/qGEO4ys8sAQgjvmdn2BOMq\nmdpav2WPa2r8uKYmdywiEtO8efDVr8aOIqegPgYzqwU+B/wphHCYmR0JXB1CGJNocCXuYyimrU5E\nJEkhQN++sGQJ9Ov3wedj9DEUWmO4FLgP2N/MngT2AE4r9qIiIuJWrIAePZpPCrEUlBhCCAvMbAxw\nEGDA8hDCtkQjK7H6+tzPHj3ixiIikpWmiW1ZBSUGM+sMjAMGZl5zYqaqcm2CsZVMfT0MG+bHxxwD\ns2crOYhIOqRpYltWoaOS7gcmAH2AHo1uFaGuzhemAli61NvyRETSIE0T27IK7XxeFEI4uAzxNL1u\nSTqf6+u9pvDcc9C/vycH1RhEJLbNm2GPPeDNN2HnnZs/J7UT3IAHzezEYi8SW48e3nwEsOeeSgoi\nkg4LFsDQoS0nhVgKTQzzgJlmtsXMNphZvZltSDKwUssmg7VrveYgIhJb2ia2ZRU6XPVa4ChgcaUt\nXtR4gtuYMV4l+8pX4OqrNcFNROKaNw9OPTV2FB9UaB/D40BNCKEh+ZDyrlvyPPTKK74myerV6au+\niUjHss8+8Je/wKBBLZ+T5j6Gl4BaM7vMzC7N3goIbpqZvWFmixo91svMHjGz5Wb2sJntVmzwxRg4\nEA47DGbOLOdVRUTy/f3vsGUL7L9/7Eg+qNDE8DLwZ2An2jdcdTowtslj/wE8GkI4CPg/4LICYyiZ\n886Dm28u91VFRHKy/QtW9Pf65CS+H4OZDQDuzw53NbPngTEhhDfMrB9QG0IY3MJrE+nS2LrVq3Dz\n5sF++5X87UVE2jR5MnTvDldc0fp5qWtKMrOfZn7eb2b3Nb0Vec09QwhvAIQQ/gHsWeT7FK1bNzjr\nLPj1r8t9ZRERl8aJbVltjUr6XebnTxKModVcOHXq1PePa2pqqCnRUKJJk2DsWJg6FboUOjZLRKQE\n3nvP5zAccURp3q+2tpba7PDLEih0VNIlIYSftfVYC69t2pS0DB/hlG1K+ksI4WMtvDbR0bFHHQWX\nXw7jxyd2CRGRD1i4EM4801dhaEual90+B2iaBCY081hzLHPLui/z2qsz73tvgTGU3KRJ3gmtxCAi\n5dTWxLam86+yDSfl2mCs1RqDmX0ROBP4BDC70VM9gIYQwnGtvrnZbUANvvjeG8AU4A/A3cA+wN+A\nM0II61t4faI1hvp6+OhHYdmydK2FLiLVbeJE71+44IJk3n9HawxtJYYBwL7Alfgw06x6YFEI4b1i\nL1xQcAknBvChqwceCN/5TqKXERF538c+BrffDiNGJPP+iSaG2MqRGObOhXPOgeXL0zmeWESqy/r1\nPlx+3brkBr6UZeazmX3WzFaY2duVuoheS448Erp2za2+KiKSpKee8tUX0jwastCZz9cAp4QQdgsh\n9Awh9Agh9EwysHIx8+akadNiRyIiHUEad2xrqtDE8EYIYVmikUT0pS/Bvfd6FU9EJElpntiWVeg8\nhp8B/fARRVuzj4cQ/l9yoZWnjyHrjDPg2GPhwgvLcjkR6YBC8B3bFi2Cj3wkueuUa3XVnsBm4ETg\nU5nbycVeNI3UnCQiSXvxRdh112STQikU1P0RQpiYdCCxHX88nH8+PPssHHpo7GhEpBqldce2pgpK\nDGY2nWbWNAohnFvyiCLp1AnOPddrDddfHzsaEalGldC/AIU3Jc0CHsjc/ow3LW1MKqhYJk70SSdb\ntsSORESq0bx5lVFjKGqCm5l1Ap4IIYwufUh51yn7FtMnneSjlM46q6yXFZEqt2UL9O0La9d6P0OS\nytX53NQBRNhHoRy0u5uIJOHZZ30pjKSTQim0mRjMNWRmPG/IzHi+H5icfHjld8opsGQJrFwZOxIR\nqSaV0vEMBSSGTFvO0syM5+ztwBDC78sQX9nttJM3JWl3NxEppUrpeIbCm5KeMbMS7TWUfpMmwYwZ\nvsuSiEgpVFWNIWMUMNfMXjSzRWa22MwWJRlYTEOGwMCB8Mc/xo5ERKrBP/7h+78ccEDsSApT6Pp+\nYxONIoWyM6FPOSV2JCJS6ebPh5EjK2dp/w6/H0NLNm70NdOXLEn/9HURSbfLLoOdd4YpU8pzvVjD\nVate9+5w2mnwm9/EjkREKl2lTGzLUo2hFfPn+0S3FSsqpwooIumyfTv06gWvvAK9e5fnmqoxJGjk\nSNhlF3jssdiRiEilyjZHlysplIISQyvMfOiqZkKLSLEqaZhqlhJDG84+G2bN8o27RUTaq5ImtmUp\nMbShb19fWO+222JHIiKVSDWGKqXmJBEpxttve6fz8OGxI2kfJYYCHHecNyUtWBA7EhGpJE8/7TtC\ndu0aO5L2UWIoQKdOqjWISPvNn195/QugxFCwCRPgjjtg8+bYkYhIpai0iW1ZSgwF2mcfz/y/r8rF\nxkWk1EJQjaFD0O5uIlKol1/2/V36948dSfspMbTDySfD88/DCy/EjkRE0q4Sh6lmKTG0w047wZe/\nrN3dRKRtlTixLUuL6LXT88/DscfCq69W3hA0EUlGba3fssc1Nd7s/O1vwyWXlD+eHV1ET4mhnWpr\nvdYwerTvylRT44/X1OSORaTjMoN33vFF89asgQ99KEYMSgxlN2OGj06aNctHHoiIZJnB3Llw4YXw\n7LOxYtCy22V3+unwxBN+XF8fNxYRSZ9KHaaapcRQhIYGnw0NcMwxSg4ikq+SO55BiaEodXWwYYMf\nL1niNxGRrEoeqgpKDEUZNgyGDvXjEGDRorjxiEi6rFsHBx4YO4ridYkdQCXq0QNmz4aePWHOHO9z\neOstmDxZe0OLiG8L3KmCv3ZXcOhx9ejhP0eO9ORw223wjW/4xt8i0rFVcv8CaLhquzU3kQXg8MPh\n2mth993h1lthl13ixCcicZnBAw/AuHExY9A8htTYutWX5161Cu67zye4iEj6tPQFb0cnqq5fD716\n+QJ6AwfuQIA7SIkhZRoavK9h1ix46CEYMCB2RCLSGrPiJ6qGAH/7Gzz3nI9Euv56H75+yCHeD5lt\nci63HU0M6nwusU6d4Mc/9qV2jz7aq5SHHBI7KpHKltQ3/PbYutWHpi9c6Ilg4UIfkbjrrjBiBPTp\nk9vIa+lSP7dS+xpUY0jQXXfBRRfB7bf7vtEisuN25Bt+U/X1Prpww4b8b/dr1+Y+/LM/X3wR9t/f\nk8Ahh+R+7rFH7r2OOcbPr/QagxJDwh57DM44wzumzzordjQila2lD/L2CAHeew/efBNOOMEnrH70\no3DaabBsmX+wb9qU+/DPJoAhQ2DnnZOPrxTUlJRyY8bAn//sIxReew3+/d8110GkGPX1/iENvtXu\n6NH+If/uu7Btm/9sfNz0Z/Z42zbo0sWbfd99199v1Sp4+2244AJPAgMGFPf/NJsMYiaFUoiWGMzs\nFeBtoAHYFkIYGSuWpA0b5nMd/vVf/Q/wuuugc+fYUYlUlgUL4KWX/HjjRu9bGD7cN9Dq2rXwn127\nelJo3PRz8MH+/7LSP9BLJVpTkpm9BHw8hLCulXMqvimpsfXr4TOfgb594Xe/a7taKiI511wDP/yh\nN9OUqg0/iaafUvaBFB9D5S67bZGvX3a77+5DWDt1ghNP9PVURKRtmzb5N/o//tHvl6pjt1qafkot\ndo1hPbAd+FUI4aZmzqmqGkNWQ4Nv+ffQQ/Dd73qHF8QbhieSdldd5Zve3Hln6b+Rp/39iouhQkcl\nmdmHQwivm9kewJ+Ai0IITzQ5J0yZMuX9+zU1NdRU0aflddf5aKUHHvA2zjT8QYmkzfr1Pov47LO9\nGbbUX6CqITHU1tZSm53oAXzve9+rzMSQF4TZFKA+hHBtk8erssbQ2J13+uJ706fDySfHH+Ymkjbf\n/a4P2pg+vXTvmeSEuTR8wavIGoOZ7Qp0CiFsNLMPAY8A3wshPNLkvKpPDODtpqec4iuzxp4YI5Im\na9fC4MHwzDNx1x5qj2pIDLGGq+4FzDSzkInh1qZJoSPp1Ss3ZnrxYt9I/MQT48YkkgZXXQVf/GL6\nk0LjGsiYMTB1qh9Xal9hKpqSWtJRagyNx1P36ePjrW+8ET71qdiRicTz979731tdHXz4w7GjqSwV\n2ZRUqI6SGCB/PPWCBTBpks/s/OlPtXy3dEwXXuhNqtdcEzuSylPJ8xikkcbjqceM8dpD794+s/Pe\ne+PGJlJuL70Ed9/tS9hL+anGEFlboyNmz4Zzz/UtRP/nf7ypSaTanXMO7LcfNBqtLu2gpqQOYPNm\nuPxyuOMO3wjks5+NHZFIcpYuhWOPhRUrvHlV2k+JoQOZMwcmToRDD4Wf/zy3DrxINTntNBg1ylcH\nkOKoj6EDGT3aNwzZZx8frXHPPbEjEimtBQt8uPbXvx47ko5NNYYKNW+e1x6GDYNf/AL23DN2RCI7\nbtw4GD9eiWFHqcbQQR15pC8qtv/+Xnu48874sy1FdsQTT/iCkuefHzsSUY2hCjz1lNceDjoIfvlL\n6NcvdkQi7ROCj8KbOBEmTIgdTeWr1CUxpIRGjvS1ZL7/fV9r6fzzfYc4My3lLZXhT3+CN97wFVQl\nPtUYqsxf/+rfuvbfH378YzjwQK3YKukWgn+5+fa34YwzYkdTHdTHIHkOP9yTw4EHwsc+5o+NHu1L\nboik0b33wrZtPkxV0kGJoQp16+Z7S2dXbK2r81Eea9bEjUukqe3b4YorfC/nTvo0Sg39U1SpYcNg\n6FA/HjzYV2wdPBguuQRWr44bm0jWHXd4M+e4cbEjkcbUx1CFsusvbd0KDz8MY8d6LWL4cJ//8Otf\n+7IakyfDoEGxo5WOats2b+686SZfAkNKR0tiSLu9+aYvyPfLX8IJJ8B//qfXMETK6aabfP7No4/G\njqT6KDFI0TZsgBtugOuu8wlz//VfcMQRsaOSjuCdd3yAxN13+7pIUloalSRF69nTm5NeegmOOw4+\n9znfUvSxxzSLWpJ1440wYoSSQlqpxiDve/dduOUWuPJK2Gsvr0GcdFJudJNIKWzc6H1bjzziy7lI\n6akpSUpu+3av4v/oR9ClC3z609DQ4InjoYc8WXTrppnUUpwf/QgWL4bbb48dSfVSYpDENDTArFk+\nxnzdOm8XXrXKl92YPVuzqaX91q+HAw6AJ5/0PgZJhhKDJC4E3xjom9/M9T185jN+O+ooX35DzU1S\niMsvh9dfh2nTYkdS3ZQYpCzq6+GYY+C55/wb37nn+rLfc+b4fImjjvKlN0aP9mU5dtkldsSSNmvW\n+LyFBQtgwIDY0VQ3JQZJXEsT5rJ9DKtW+a5bc+d6oqir81nX2WRx1FG+65xqFR3bpZfCe+/5HBpJ\nlhKDpM6WLb4M+Jw5uWTRtWsuUXTrBq+95p3c6szuGFav9r6pJUu0X0g5KDFI6oXgcyWySWLuXFi+\n3GsQmzd788L8+erMrmYXXAC9esFVV8WOpGNQYpCK9OijXlPYvt3vjxoF3/qWD43deee4sUlpvfii\n//u+8AL07h07mo5BM5+lIo0alVufafhwOO88Xztn773ha1+Dp5/W7OtqMXUqXHyxkkIlUY1Byq61\nzux994Xf/hZmzPCRTRMm+HaPapeuLNl/49Wr/d/y4ot9CRb1I5WHmpKkKjU0+CS6GTNg5kz4l3/x\nJHHyyb63hKTf0qU+dHnLFk2KLDclBql6GzfCPffA9OmwbBmceaYniREjYkcmLZk5EyZNgrff9iTf\ntSs8/riv4ivJUx+DVL3u3T0RPPaYj2jq2dM7qQ891MfE//Offl59vT+v/a3j2bLF+4j+7d98va3h\nw/3xIUNyOwpK+qnGIBWpoQH+8hevRcya5U1NdXU+2W7oUDVbxLB0KXzhC54Ezj4b/vrXlidFSrLU\nlCQd3qxZPj7+ySdzj+27L4wZA+PH+wfVoEHqm0hKCHDzzb4T4FVX+XIpmuUe144mhi6lDEYkhu7d\nfR2nlSt9PZ7evb3WsG0b3Hqrf5N99VXYbz9PEtnb0KG+7lO3brF/g8q1fj185Ss+YfHxx32yolQ+\n1RikatTX+5ILQ4d+sBnpnXd8gtXSpX7O0qV+e/llGDgwP1kMGQIHHeSJpa7O51uoWeqD5s71gQDj\nx8NPfqKJiWmipiSRHfDuu7mEkb099ZT3VZj5zOxu3WDwYL8NG+ad37vtlrs1vd9Sk1V9fXUkmoYG\nuPpq+OlPfYvOU0+NHZE0pcQgkoDZs+GTn/TVQDt39pVB+/SBDRt8CGb21vh+9rhz5/yk0dDgQ25X\nrfKaS48eMG6cj6w64ww/v1K8/jp86UueUG+91VfNlfRRH4NIidXW+kiaPn28z6JvX//wHjWq7RE1\nIfiQzaYJ5Omn4Yor/JxNm2DFCvjOd2DiRO8oHzTI+zsOOCB3vM8+6UoaDz7oHctf/arvB95Fnx5V\nSzUGkRa01mfRHtlEM326J5o99/SEMHasJ5sXX/REsXKl/8we//Of3v/RNGEccAD07+9JoxzNU+++\nC5dd5vMSbrnFhwZLuqkpSaRCtDfRbN7sSaNpwlixAtau9dFYmzb5XIGePeH00+Hzn4cTTihdzCtX\n+tyE/v19O84+fUr33pIcJQaRDmjTJl92YsIE7yA386anNWu8ZjFsmM86zv4cMAA6tXOdg1tu8aXQ\np0712cyam1A51Mcg0gE9/bTXPvr2zTVPnXkmHHss7LEHLF7sTUw33OA/16/3mkrjZDFsmL+usfp6\nH5U1bZrv6f3oo74AnnQsqjGIVLBCm6fWrfPzsglj8WK/7bRTLlEMGgT//d/wyis+SVDbcFYuNSWJ\nSFFC8L236+rgD3/w2sHKlf5cp07eQX722VrbqBKpKUlEimLmO+btvbdP4tt999zIqT328Jt0TNGW\n3Tazk8zseTN7wcwmx4qjUtTW1sYOITVUFjmlKouaGrjySh/xNGeO/7zyysqqLejvonSiJAYz6wRc\nD4wFhgJfNLPBMWKpFPqjz1FZ5JS6LHr08M10KnHJDv1dlE6sGsNIYEUI4W8hhG3AHcCnk75oe/9w\n2jq/teebe66tx5o+n+QfetrKor33S0llUfx7qywKP7+SyiJWYtgbWNXo/urMY4lK2z9008f0R1/4\n/VJSWRT/3iqLws+vpLKIMirJzD4HjA0hfCVz/2xgZAjh4ibnaUiSiEgRKnFU0t+Bjza63z/zWJ4d\n+cVERKQ4sZqSngYGmdkAM9sJ+AJwX6RYRESkkSg1hhDCdjO7CHgET07TQgjLYsQiIiL5Uj3zWURE\nyi/aBDcREUknJQYREclTUYnBzHY1sxlmdqOZnRk7npjMbF8zu9nM7oodS2xm9mkz+5WZ3W5mJdym\npvKY2WAzu8HM7jKzr8aOJ7bMZ8bTZjYudiwxmdkYM3s887fR5h58FZUYgM8Cd4cQLgBOiR1MTCGE\nl0MI58WOIw1CCPdm5sRcCJwRO56YQgjPhxAuBD4PjI4dTwpMBu6MHUQKBKAe6IZPKG5V1MRgZtPM\n7A0zW9Tk8ZYW2OtPbsb09rIFWgZFlEXV2oGyuBz4RXmiLI9iysLMPgXMAv5YzliT1t6yMLPjgaXA\nWqCq5kS1tyxCCI+HEMYD/wF8v80LhBCi3YBPACOARY0e6wSsBAYAXYGFwODMc2cB4zLHt8WMPXZZ\nNDrn7tixp6EsgKuAT8aOPQ1l0ei8WbHjj1kWwA+Aa4GHgZmx40/D3wWwE3BXW+8fdT+GEMITZjag\nycPvL7AHYGbZBfaeB2YC15vZeOD+sgabsPaWhZn1Bn4IjDCzySGEq8sbcXKKKItvAMcBPc1sUAjh\nV+WNODlFlMUYvMm1G/BAWYNNWHvLIoRweeaxLwP/LGuwCSvi7+Iz+GrWu+ErW7cqjRv1NLfA3kiA\nEMJm4NwYQUXSWlm8hbepdxStlcXPgZ/HCCqS1sriMeCxGEFF0mJZZIUQflvWiOJp7e9iJv7FuiCV\n1vksIiIJS2NiKGiBvQ5CZZGjsshRWeSoLHJKVhZpSAxG/oiBjrzAnsoiR2WRo7LIUVnkJFYWsYer\n3gbMAQ40s1fNbGIIYTvwDXyBvSXAHaEDLLCnsshRWeSoLHJUFjlJl4UW0RMRkTxpaEoSEZEUUWIQ\nEZE8SgwiIpJHiUFERPIoMYiISB4lBhERyaPEICIieZQYpOpkZn4ubudrzjGzfgWcU9RifWZ2gZmd\nXcxrRcotjauripRCe2duTgDqgH+U+H39RSHcWMzrRGJQjUGqVVczu8XMlmb2P94ZwMyuMLP5ZrbI\nzP4389jngMOBW8xsgZl1M7MjzOxJM1toZvPM7EOZ993bzB40s+Vm1uweGGZ2lZnVZV57TeaxKWZ2\nqZl92MyezVznWTN7z8z2MbO+ZnZPJrb5ZqZtOSUaJQapVgcB14cQhuB73X4t8/jPQwijQggHA7ua\n2fgQwu+BvwJnhhAOAxqAO4BvhBBGAMcD72RefwhwOnAw8Hkz27vxRTMbKJ0aQhiWee0PGj8fQng9\nhHBo5jo34TvwrQJ+BlwbQhgFnAbcXNriECmcEoNUq1dDCPMyx7fgWyECHJepASwCjgWGNnpNdqXK\ng4DXQggLAEIIGzMLlAH8OXN/K76fcNNdtN4GtpjZzZlds7Y0F5yZHQ2cR27jqePx3QmfxVfE7G5m\nu7b/1xbZcepjkGrVtC8gmFk34BfAYSGE18xsCrBzC69vafP4rY2Ot9Pk/1AIYbuZjcS3Gj0duChz\nnHtjsw/jtYVPhRCyicOAUSGEba3/WiLJU41BqtUAMxuVOT4TeAJPAgF408y64002WfVAz8zxcqCf\nmX0cwMy6m1nnQi6a6YvYPYTwEHAp3uTU+PkuwF3A5BDCi42eegS4pNF5hxT0W4okQIlBqtXzwNfN\nbCmwO3BDCOFt/Jv6EuBB4KlG588A/tfMFuD/L76AN+0sxD+0uzVzjeZGKPUAZpnZc8DjwLeaPD8a\n+DjwvUZUpA/mAAAAR0lEQVSd0P3wpHC4mT1nZnXABcX80iKloP0YREQkj2oMIiKSR4lBRETyKDGI\niEgeJQYREcmjxCAiInmUGEREJI8Sg4iI5Pn/pOBey1ySHUcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0a71fb2a58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the results\n",
    "plt.errorbar(batch_sizes, mean_times, std_times, marker='.')\n",
    "plt.xscale('log')\n",
    "plt.xlabel('batch size')\n",
    "plt.ylabel('runtime')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Complex graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create a simple tensorflow graph\n",
    "with tf.Graph().as_default() as graph:\n",
    "    placeholder = tf.placeholder(dtype=tf.float32)\n",
    "    weights = tf.constant(np.random.normal(0, 1, (4, 4, 1, 3)), tf.float32)\n",
    "    result = tf.nn.conv2d(tf.expand_dims(placeholder, -1), weights, [1, 2, 2, 1], 'SAME')\n",
    "    \n",
    "session = tf.Session(graph=graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14/14 [02:39<00:00, 20.97s/it]\n"
     ]
    }
   ],
   "source": [
    "num_runs = 5\n",
    "k = 14  # Use a batch size of up to 2 ** k\n",
    "\n",
    "# Create data to feed to tensorflow that is sufficiently large to be problematic\n",
    "data = np.random.normal(0, 1, size=(2 ** k, 32, 32))\n",
    "num_batches = 2 ** np.arange(k)\n",
    "batch_sizes = 2 ** k / num_batches\n",
    "mean_times = []\n",
    "std_times = []\n",
    "\n",
    "# Iterate over the different batch sizes\n",
    "for num in tqdm(num_batches):\n",
    "    times = []\n",
    "    # Use multiple runs to estimate the mean and standard error of the mean\n",
    "    for run in range(num_runs):\n",
    "        start = time()\n",
    "        batches = np.split(data, num)\n",
    "        for batch in batches:\n",
    "            session.run(result, {placeholder: batch})\n",
    "        end = time()\n",
    "        times.append(end - start)\n",
    "        \n",
    "    # Store the results\n",
    "    mean_times.append(np.mean(times))\n",
    "    std_times.append(np.std(times) / np.sqrt(num_runs - 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f0a6c592860>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAETCAYAAAA/NdFSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm8XPP9x/HXJxsiNyghJLZULUkkJNZUGE0iKoSqlFqq\n0dpKbVVLW48kRWirhGh/KEWLKrEUJUGYECSN7JtoisoiEaVyCWly8/398Z0xN9dd5t4528x5Px+P\nedwzM+fO93NPJudzznc15xwiIpI+reIOQERE4qEEICKSUkoAIiIppQQgIpJSSgAiIimlBCAiklKh\nJwAzu9jM5pnZHDO738zahV2miIg0LdQEYGY7AD8G+jjnegFtgJPCLFNERIrTJoIyWgObm9kGoD2w\nPIIyRUSkCaHeATjnlgO/Bd4FlgH/dc49H2aZIiJSnFDvAMxsS+BYYGfgY2CcmZ3snHugzn6aj0JE\npJmcc1bK74fdCDwQeMs596FzrgZ4FOhX347OucAeI0aMCHT/ht4v9vXGnje1b5THoph9ozoWQR8H\nHQsdi0o7FkEIOwG8CxxkZpuamQEDgIUhl0kmkwl0/4beL/b1xp43N9bmas7nF7OvjkXT++hYNP91\nHYvingd+LILOpnUfwAj8SX8OcC/Qtp59nHgjRoyIO4RE0HEo0LEo0LEoyJ03Szo/h94LyDk3ChgV\ndjmVIuyrnXKh41CgY1GgYxEscwHVJZUUhJlLQhwiIuXCzHAJbwQWEZGEUgIQEUkpJQARkZRSAhAR\nSako5gKKRDbrH/ntfGeBTKawLSIiBRXZC8gMEvBniYiERr2ARESkxZQARERSSglARCSllABERFKq\n4hJAdfXGP0VEpH4VlQCqq6F7d7/dv7+SgIhIYyoqAcybB8uW+e0FC2D+/HjjERFJsopKAD17Qpcu\nfrt7d+jRI954RESSrOIGgs2bB3vvDR9/DB07BvKRIiKJo4Fg9ejZ0/98551YwxARSbyKSwB5zz8f\ndwQiIslWsQnguefijkBEJNlCTQBmtruZzTSzGbmfH5vZBWGWmffKK7B2bRQliYiUp1ATgHPuTefc\nvs65PkBf4FPgsTDLzNtrL3jttShKEhEpT5H1AjKzI4CrnHP963mv5F5AddcDqKnx00L/8pdaD0BE\nKk8QvYCiTAB3AdOdc7+v571A1wMAePFFuOIKmDo10I8VEUmEIBJAJCuCmVlbYChwRUP7jBw58ovt\nTCZDpsTL9n79/Gjgjz6CrbYq6aNERGKXzWbJ5qs5AhLJHYCZDQV+5Jw7soH3A78DADjySDjrLDj+\n+MA/WkQkVuU0EOy7wF8iKusLgwZpPICISENCTwBm1h4YCDwadll1DRyo8QAiIg0JPQE459Y45zo5\n5yKfnHnvvWH1ak0LISJSn4odCQzQqhUMGKBqIBGR+lR0AgC1A4iINKTipoOua8kS6NMHVq70dwQi\nIpWgnHoBxWbHHWHrrWH27LgjERFJlopPAOB7A6kaSERkY6lIAIMGqTuoiEhdFd8GAH55yK5dYdUq\n2HTT0IoREYmM2gCKtMUWfkzAK6/EHYmISHKkIgGA2gFEROpKTQJQO4CIyMZS0QYA8L//QadO8NZb\nvluoiEg5UxtAM7RrB/37wwsvxB2JiEgypCYBgNoBRERqS10CUDuAiIiXqgTQowd89plvBxARSbtU\nJQAz3QWIiOSlKgGApocWEclLTTfQvGXLoFcveP99aN06kiJFRAKnbqAt0KULdO4MM2fGHYmISLyi\nWBR+CzN72MwWmtl8Mzsw7DKbonYAEZFo7gBuBp52zu0F9AYWRlBmo9QOICISchuAmXUEZjrnvtrE\nfpG1AQBUV8MOO/hlItu3j6xYEZHAlEMbwK7AB2Z2t5nNMLM7zGyzkMtsUlUV7LMPTJ4cdyQiIvFp\nE8Hn9wHOc869bmZjgCuAEXV3HDly5BfbmUyGTCYTamD5doAjjgi1GBGRQGSzWbLZbKCfGXYV0HbA\na865brnnhwCXO+eOqbNfpFVAAK++Cuedp95AIlKeEl8F5JxbCSwxs91zLw0AFoRZZrH23x/eftsv\nEykikkZR9AK6ALjfzGbhewGNjqDMJrVtC4cdBhMnxh2JiEg8UjcSuLaxY2H2bLjzzsiLFhEpSeKr\ngJIuv0xkAnKgiEjkUp0A9tgDampg8eK4IxERiV6qE4CZFosXkfRKdQIALRMpIumV6kZggBUrYK+9\nfHfQNmEPixMRCYgagQPQuTPsuCNMnx53JCIi0Up9AgBNDy0i6aQEgKaHFpF0Sn0bAMCnn/qqoPfe\ngw4dYgtDRKRoagMIyOabQ9++8PLLcUciIhIdJYActQOISNooAeSoHUBE0kZtADnr10OnTrBwoW8P\nEBFJMrUBBKhNGzj8cE0PLSLpoQRQi9oBRCRNlABqybcDJKBWTEQkdEoAtey2G7RuDW+8EXckIiLh\nUwKoJT89tHoDiUgaKAHUoXYAEUmL0LuBmtk7wMfABmCdc+6AevaJvRto3qpVvirogw/8wvEiIklU\nLt1ANwAZ59y+9Z38k6ZTJ+jWDaZNizsSEZFwRZEALKJyAqNlIkUkDaJYA8sBz5lZDXCHc+4PEZRZ\nkoED4dJLC91Bs1nIZPx2JlPYFhEpZ1G0AWzvnHvPzDoBzwHnO+cm19nHjRgx4ovnmUyGTIxn2TVr\nYLvtYPlyqKryvYMS0kQhIimVzWbJZrNfPB81alTJbQCRzgVkZiOAaufcjXVeT0wjcN6AAXDxxXD0\n0UoAIpI8iW8ENrP2ZtYht705cAQwL8wyg6LuoCJS6cJuA9gOeMzMXK6s+51zz4ZcZiAGDYLTT487\nChGR8Gg66AbU1MC228KcOdC1q6qARCRZEl8FVM5at4ZvfEPTQ4tI5VICaITaAUSkkqkKqBFvvQUH\nHwzvvw+rV/suoSIiSaAqoJB16gQffeS3+/eH6up44xERCZISQCPmzfONwQALFsD8+fHGIyISJCWA\nRvTsCXvt5bd32gl69Ig3HhGRIBWVAMxsOzO7y8yeyT3vbmY/CDe0+FVVwWuvFZ63bh1fLCIiQSv2\nDuAeYAKwQ+75m8BFYQSUNPmG3/32g1rTFYmIlL1iE8A2zrmH8HP745xbD9SEFlUCjR0L990HU6fG\nHYmISDCKTQCfmtnW+KmdMbOD8Kt8pUanTnDzzTB8OHz+edzRiIiUrqhxAGbWBxgL9MRP5tYJOME5\nNyeQIBI6DiAvPxuoc/Dtb/uG4WuvjTsqEUmzIMYBFD0QzMzaAHvgV/ha5JxbV0rBdT67LBIAwIoV\n0Ls3PP009O0bb1wikl6RJQAzaw0MAXah1gyidef1b3EQZZQAwLcF/PrX8Prr0K5dfHGJSHpFORL4\nSeD7wNZAVa1HKp1yih8XcN11cUciItJyxd4BzHHO9QotiATeAWSz/pHfrrsm8LJlsO++8Pzz0Cu0\nIyMiUr8oq4B+BUwMazGXJCaAYvzxj/C738GUKdC2bdzRiEiaRFkFNAW/stdnZrbazKrNbHUpBVeC\n4cNhm23ghhvijkREpPmKvQN4GzgWmBvGpXq53gEAvPuu7w00aRJ07x53NCKSFlHeASwB5pXtWTpE\nO+0EV18NZ5xRmDlURKQcFHsHcA/QDXgGWJt/vdhuoGbWCngdWOqcG1rP+2WdWzZsgAEDYMgQuPTS\nuKMRkTQI4g6gTdO7APB27tEu92iuC4EFQMcW/G7itWoFd90FBxwAQ4fC7rvHHZGISNNCXxLSzLoC\ndwPXApdU4h1A3i23wMMP+/aAVlppQURCFHobgJmNyf180syeqPsosoybgJ+Sm0iukp1/vh8xfOut\ncUciItK0pqqA/pz72aKOjmY2BFjpnJtlZhn8PEL1Gjly5BfbmUyGTH7kVRlp1cqPDejXD44+Grp1\nizsiEakU2WyWbH50akCKbQS+0Dl3c1Ov1fN7o4FTgfXAZvjpIx51zn2vzn4VUQWUd8MNfrK4559X\nVZCIhCPKkcAznHN96rw20zm3b9EFmR0G/KSS2wDyamr8XcDw4XDOOXFHIyKVKPReQGb2XeBkYNc6\ndf5VwIelFFzJWreGu++Gww6Do47yYwVERJKm0TsAM9sZ2BW4Drii1lvVwJzc0pClB1FhdwB5o0f7\nHkHjx/sppUVEghLpgjBhqtQEsG4dHHig7x10xhlxRyOSfE3NwisFUbYBHA/8CtgW35PHAOecC2Rg\nV6UmAIDZs2HgQJg1C7p0iTsakfJRdyEm2ViUCWAxcIxzbmEphTXy+RWbAABGjoRnn4VBg/yXWlc2\nIk0LIgFU8h1FlAngFefc10spqInPr+gE8L//wX77wWWXwamn6spGpBhB/z+ptP93USaAm4HOwONs\nPBnco6UUXuvzKzoBAEyf7nsEzZ4N229fWV9EkTAoATQuysngOgJrgCNqveaAQBJAGvTtCz/4AZx5\npn9eXQ1VqV1VWUSSQL2AIrRqFey4I6xdC717w8svKwmINER3AI2L7A7AzO6mnsncnHPq3NgMixcX\nFo1ZsADmz4eDDoo3JhFJr2KrgJ6qtb0p8C1gefDhVLaePaFHD98OsNVWfltEJC4tqgLKrfA12TnX\nL5AgUlIFBL7uv2NHnwBmzIBddok7IpFkUhVQ46JcE7iur+EHhUkz5ev8L74YfvrTeGMRkXRr8g7A\nzAyoAT6p9fIK4Ern3COBBJGiOwDwVyJr1sBee/lJ4w4/PO6IRJJHdwCNi3IcwDznXM9SCmri8ys+\nAdQ3InHBAj8+YNEiaFNsa4xISigBNC7KBHAvcKtzblophTXy+RWfAOrjnL/6P/FEOPfcuKMRSRYl\ngMZFmQDeAHYD/g18SmEyuF6lFF7r81OZAMD3CDriCFi4EL7ylbijEUkOJYDGRZkAdq7vdefcv0sp\nvNbnpzYBgL/6b9cObm50gU2RdFECaJzWA6gQH3zgG4SzWY0NEIFCd+nVq4MbLa8E8GVasjwBttkG\nrrrKdw2tpC+oSEtUV8PBB/vt/v39cwmHEkBCnHsuLF0KTz4ZdyQi8Xr+eT9NCvifc+fGG08lCzUB\nmNkmZjbVzGaa2VwzGxFmeeWsbVsYMwYuucRPFieSVnffDdtt57fbtIGzzoKHHirMoyXBCb0NwMza\nO+fWmFlr4BXgAufcP+rsk+o2gNqOPRb69YPLL487Eqk05bA61vjxcN55MGUKbLstfPyxnzX36qv9\n9s9/Died1LJxM2oDqOczojrxmll74CXg3LrjCZQAChYv9jOEzp3rF44RCUMST4Zr1/oJE8eMgSFD\nNo7ROZg40SeC5cvhyivhtNP8nXOxkvg3l6IsGoHNrJWZzcRPH/FcWIPJKsVuu/mFY668Mu5IRKJ1\nww2+F9yQIV9+zwwGDoRJk+DOO+GBB+BrX4PbblOVaSmivAPoiF9S8nzn3II677kRIwrNA5lMhkxS\n7kljUF0Ne+wBjz8OBxwQdzRSiZJ2NfzOO37d7NdfL8yQ21SMr70G11wDc+b4iRXPPBM226zh/ZP2\nNzdXNpslm6/DA0aNGlU+VUAAZnYV8Klz7sY6r6sKqI577vFXN6++Cq3UV0sClrST4XHHwf77+zr+\nvGJjnD7dJ4IpU+AnP4FzzoEOHb68X9L+5lIlvgrIzLYxsy1y25sBg4A3wiyzUnzve7BhA9x/f9yR\niITr73/33T0vvbRlv9+3Lzz2GEyYANOmwVe/CqNH+0Fk0riwry23B140s1nAVGCCc+7pkMusCK1a\n+akhrrgCPvmk6f1FytHnn8MFF8DYsbDJJqV9Vq9e8Ne/+h5OCxf6RDByJHz0UWEwmQaVbUxTQSTc\naaf5heRHj447EqkUYUyz0FK//KWfEPGRelYWKbXKZvFiuO46f3fQti28/z707u27lcb9dwehrLqB\nNhqEEkCDli3zVzbTpkG3bnFHI+WuuhoOOcQ3nMZ9MnzrLd/JYcYM2GmnL78fVJ39o4/CsGG+SrVt\nW3jpJd/Vutwlvg1ASteli2/Yamn9qEht8+ZtPM1CfjsOF13kv9v1nfyDNGgQ7L233+7eXRMu1qYE\nUAYuuQRmzfIDYURK0bMnbLml33YO1q2LJ44nn/Qr4V1ySfhlVVX5Ox2onOqfoCgBlIFNN4Xf/hYu\nvBDWr487GilnHTpA+/Z++957/bQKb70VbQyffea/y7feWnrDb7HyJ32d/DemBFAmjjvOT5B1++1x\nRyLlbNGiwvYpp/gR50ce6dekiMr11/tBX4MGRVem1E+NwGVk3jz4xjd8F7ett447GilHY8bAggXw\nhz8UGlivvBJefBFeeKFwdxCW/FxXs2ZB166N76sVwRqnRuCU6dkTvvMdqDVrhkizjB/vr/hrGz0a\ndt/dVweFWcXonO/zf9llTZ/8JRq6Aygz//mPXz5y4sRCzwaRYqxZ46sRly71DcG1/8utWwdHHw07\n7+yrGa2k68r6/e1vfmDj7Nl+Deym6A6gcboDSKGtt/Z3ABdeWFlfZgnfpEnQpw9sscWX32vbFsaN\n85OxXXNN8GWvWVNo+C3m5C/R0B1AGVq/Hvbd14+i/Na34o5GysWFF0Lnzr7Ov6Gr4RUr/IJEv/gF\nnHFG45/XnAVmrroK/vlPePDB4D6zuXQHUM9nJOHEqwTQfBMn+ulvFyzw3URFmrLTTvDNb/qFhho7\nuS5aBIcdBn/8Ixx1VHGf3djJ9Z//9Iu8z57tBzbGRQmgns9IwolXCaBljj/ed6f72c/ijkSS7q23\n/JX98uXFTS8+ZQoccww8/bSfprkpDZ1cnfONzkcc4Uf9xkkJ4MvUBlCmslk/SdzVV/veQVde6Wc+\nrLVehMgXxo+HwYOLX1vioIPgrrtg6FDfdbOlHnvMNzpfcEHLP0PC04KllSUJMhk/D/qDD/r5XFq3\nhsmTNdJR6jd+PJx8cvN+Z+hQeO89fwX/6qt+kfbm+PRTuPhiP+K4OWv3SnR0B1DG5s2DDz/023FP\n7CXJtXat7wHUkpG3Z5/tE8eQIc1fl+Laa/3Moyle3TXxlADKVDYLTzxRGBG8YYO/ZVcVkNQ1ebIf\nO9LS0eOjRvkpyb/zneInj1u0CO64A37zm5aVKdFQI3CZq672V/6vv+7/w02d2vjC2JI+P/0pbL65\nbyNqqXXr/HxUnTvDnXd+eaBY7QZW53x7wze/6auAkkKNwF+mO4AyV1XlG+zOO8/Pcx7F9LpSXsaP\n9yfjUrRtCw89BHPnNp1Ixo3zbQfnn19amRI+JYAKYeaH8D/3HDz8cNzRSFIsXepPxvvtV/pnbb45\nPPUU3H+/v9uszyef+IuQ3/9eDb/lINQEYGZdzewFM5tvZnPNTJ3BQtSxo+8VdN558PbbcUcjSTBh\ngm/8bd06mM/bdlt/RzFypF/Upa6rr4bDD4f+/YMpT8IVahuAmXUGOjvnZplZB2A6cKxz7o06+6kN\nIEA33QR//atf/UhXYel2wgl+QNfppwf7udOm+Z5BTzzhqyDN/Kj0Qw/11USdOwdbXhDUBlDPZ0R5\n4jWzx4GxzrmJdV5XAgiQc/4/fffu8Otfxx2NxGXdOn/FvnBhOCfkp5/28wVNmgR77unXqjj22OQO\n+lIC+LLI2gDMbBdgH2BqVGWmlRnccw/85S/+dl3SaepU2HXX8K7GjzrKryUweLB/vnIl/OhH4ZQl\n4YhkJHCu+mcccKFzrt7hJCNrdS3IZDJkNHqkJNtsA/fd5xf5mD4ddtgh7ogkavUt/hK0YcMKc1F9\n/rlf71ej0cORzWbJBjzQJ/QqIDNrAzwFPOOcu7mBfVQFFJJRo/wt+nPPBdcQKOWhb1/fHnTooeGV\n8dpr/vPXr/ftTS+95NsEkkhVQF8WRRXQH4EFDZ38JVy/+IX/0o8eHXckEqWVK+Ff//LTMIepZ08/\n/gR8m1N+W8pD2L2Avg68BMwFXO7xM+fc+Dr76Q4gRMuX+6vBhx5S97y0+POf4fHH4ZFHwi+rutp3\nQV69OnnVP2EuMBO3susF1GAQSgChe/ppOOccmDmz5XPCSPk45RR/gjvzzGjKq7TqlXKgBCDNcuml\nfpKuJ54IZ9FvSYaaGt/zZ8YMv2ZEFJQAolcubQCSEKNHw/vvw81qjalo06fDdttFd/KX8qUFYVKk\nXTs/NuCgg3xbQN++cUckYYii+6dUBt0BpEy3bnDrrXDiib7RTiqPEoAUS20AKXX22b73xv33qz2g\nknz4Ieyyi6/q23TT6MpVG0D01AYgLTZmjJ+06+67445EgvTcc35gVpQnfylfSgAptdlmfsbQyy/3\nszhKZQhi8RdJDyWAFOveHa6/3rcHfPZZ3NFIqZxT/b80jxJAyp1xBuy9N1x0UdyRSKnmzIEOHeCr\nX407EikXSgApZwa33QYTJ/qpIqR86epfmku9gASA11/387tPmeK7ikr5yWTgssv8v2MUKnmenXKg\nqSAkUGPG+EnEbroJ9t03eRN7ScNWr4YuXWDFCr94u1Q+dQOVwGSz/uQxfz4cdhh87Wtw5ZWFKzxJ\nthde8FM/6+QvzaEEIIC/ZT/2WD+RGMCqVb4qQbfy5UH1/9ISSgAC+Cv9J57wU0Wb+dWdTjkF/va3\nuCOTpjgHzzyj/v/SfGoDkI1UV/tqoD33hOuug3Hj4Mkn/ZgBSaaFC/3V/zvvaFqPNAmiDUCzgcpG\nqqoKa7r+6lf+xJ/JwJ/+pCqGpMpX/+jkL82lKiBp1Omnw6OPwvDhcMstmvAriVT/Ly0V9prAdwFH\nAyudc70a2U9VQAn3zjtwzDHQr5+fTrpt27gjEoA1a/ziL8uW+XV5JT3KoRvo3cDgkMuQCOyyC7zy\nij/RDB7spx2W+GWzfmEfnfylJUJNAM65ycBHYZYh0enY0fcK6tMHDjwQ3ngj7ohE1T9SCrUBSLO0\nbg033OAHiR16qJ9/XuKjBCClUAKQFjnjDN9F9LTT4He/izuadPrXv3y33d69445EylViuoGOHDny\ni+1MJkNGQ1AT79BD4dVXfePwggV+LiE1DkdH3T/TJZvNkg14bpbQB4KZ2S7Ak865vRvZR72AytjH\nH8NJJ8H69X5K6a22ijuidDjmGDj1VL+gj6RP4nsBmdkDwKvA7mb2rpkND7M8iccWW/jRwj17+kFk\nb74Zd0SVb+1amDQJBg6MOxIpZ6FWATnnTg7z8yU52rTx00jvtRf07w8PPAADBsQdVeWaPBl69PBz\nN4m0lBqBJVBnnQUPPugnkrv99rijqVya/E2CoAQggTv8cH+FOmYMXHABfPQRvPaa77EiwVD3TwmC\nZgOV0Dz1lE8AS5fCunV+yoLhw/1IYnXyarklS/yKbStX+nEZkk6JbwSWdOvQwZ/o16/3z1eu9BPL\nTZvmE4K0zIQJcMQROvlL6ZQAJDSZDNx8M/Tq5ccH7LmnHzg2bhxsvz388If+ZKZk0Dzjx6v+X4Kh\nKiAJXX6RmR49CgvN//vfPhE8/DAsXuyXoxw2zPcc0mCyhq1bB9tu6+dh2m67uKOROAVRBaQEILF7\n991CMnjzzY2TQbt2cUeXLC+/DBddBNOnxx2JxE0JQCrOkiWFZLBoEQwd6pPBwIFKBgA//7n/ee21\n8cYh8VMCkIq2ZAk88ohPBgsXFpJBfrRxz56FKqW06NvXt6scckjckUjclAAkNZYuheuvh8cfh+XL\n/dKUHTv6uXCGDUtHt9IVK/xI61Wr/MhrSTd1A5XU6NoVTjjBjyHIz365ejXcdRd8//twySW+R9Fn\nn8UaZqiefda3i+jkL0FRApCykcn40cV77+17CvXu7a+K8zOQXn217yEzeDDceKPveVRJN5Ya/StB\nUxWQlJ36upXm/fe/8MIL/m5gwgSoqfGDpgYP9g3JX/lKPDGXqqbGd/ucNcvfDYmoDUCkEc75xuIJ\nE/zV88sv+4bjwYP9Y//9fXVKdTXMm5fsRuV//AN+8AOYOzfuSCQplABEmuHzz/0kdfm7g6VL/apm\n06f7qqRu3fyi9zvuCO3bJ2ulrVGj4JNP4De/iTsSSQolAJESjBsHt9zi7wzyNtsMNmzwjy239I+t\ntvKP/HZDP/PbrVv7bqtB3lEcfDBcc43WWJCCIBKA+hNIam2zDXz9676a6P33fQNyfrbSgw/27Qkf\nfVT4WXt71Sr/e7XfW7ECPvzQr9YF0KqVTwjbbuvr7Tt23PhRVfXl1+o+2rWDd96BOXO0+LsET3cA\nknqNNSo3RzYLf/4z3HOPv4No1QqOO85fte+2m++2Wl3tfzb2yO/z3//6doz8Q9NpS226AxAJQFWV\nH11cqkzGj9SdPh0WLIDu3X0yaGlSefFFuPden1Sc83cdq1aVHqdIXujjAMzsSDN7w8zeNLPLwy6v\n3GWz2bhDSIRyPQ5VVb5N4aWX/M9S7igOPxzGjoVdd83Stq0f/3DTTem++i/X70VShZoAzKwVcCsw\nGOgBfNfM9gyzzHKnL7hXzschf0cRRANwVRUMG5YNJKFUgnL+XiRR2HcABwD/dM792zm3DngQODbk\nMpv9JWlq/4beL/b1xp6H/YVuzucXs6+ORdP7BH0sNtkkuITSUNml7qvvRdP7JPFYhJ0AugBLaj1f\nmnstVEoADZdd6r46Fk3vo2PR/Nd1LIp7HvSxCLUXkJl9GxjsnDsr9/xU4ADn3AV19lMXIBGRZkp6\nL6BlwE61nnfNvbaRUv8IERFpvrCrgKYBu5nZzmbWDjgJeCLkMkVEpAih3gE452rM7HzgWXyyucs5\ntzDMMkVEpDiJGAksIiLR04IwIiIppQQgIpJSiUwAZtbezO4xs9vN7OS444mTme1qZnea2UNxxxI3\nMzvWzO4ws7+Y2aC444mTme1pZv9nZg+Z2TlxxxO33DljmpkdFXcscTKzw8zspdx349Cm9k9kAgCO\nBx52zp0NDI07mDg55952zv0w7jiSwDn3t9yYknOB78QdT5ycc284584FTgT6xR1PAlwO/DXuIBLA\nAdXAJviBt42KJAGY2V1mttLM5tR5vaGJ4rpSGEFcE0WMUWnBsahYJRyLXwC/iybKaLTkWJjZMcBT\nwNNRxhq25h4LMxsILABWARU1pqi5x8I595JzbghwBfDLJgtwzoX+AA4B9gHm1HqtFbAY2BloC8wC\n9sy9dwrva0S1AAAEIklEQVRwVG77gShijOrR3GNRa5+H4449CccCuB74RtyxJ+FY1Nrvqbjjj/NY\nANcANwITgMfijj8J3wugHfBQU58fyXoAzrnJZrZznZe/mCgOwMzyE8W9ATwG3GpmQ4Ano4gxKs09\nFmb2FeBaYB8zu9w596toIw5PC47Fj4EBQEcz2805d0e0EYenBcfiMHxV6SbA3yMNNmTNPRbOuV/k\nXvse8EGkwYasBd+Lb+FnX94CPxNzo+JcEKa+ieIOAHDOrQHOiCOomDR2LD7E13mnRWPHYiwwNo6g\nYtLYsZgETIojqJg0eCzynHN/ijSi+DT2vXgMfwFdlKQ2AouISMjiTABFTRSXEjoWBToWBToWBToW\nBYEdiygTgLFxC32aJ4rTsSjQsSjQsSjQsSgI7VhE1Q30AeBVYHcze9fMhjvnaoAf4yeKmw886FIw\nUZyORYGORYGORYGORUHYx0KTwYmIpJQagUVEUkoJQEQkpZQARERSSglARCSllABERFJKCUBEJKWU\nAEREUkoJQMpWbiTk3Gb+zulm1rmIfVo06ZyZnW1mp7bkd0WiFudsoCJBaO5Ixu8D84AVAX+u/yXn\nbm/J74nEQXcAUu7amtl9ZrYgtz7upgBmdpWZTTWzOWZ2W+61bwP7AfeZ2Qwz28TM9jezV8xslplN\nMbPNc5/bxcyeMbNFZlbvGgxmdr2Zzcv97q9zr40ws0vMbHszm5krZ6aZrTezHc1sGzMbl4ttqplp\nOUeJjRKAlLs9gFudc93xa6H+KPf6WOfcgc65XkB7MxvinHsEeB042TnXB9gAPAj82Dm3DzAQ+Dz3\n+72BYUAv4EQz61K70NxCPcc553rmfvea2u87595zzu2bK+cP+BXdlgA3Azc65w4ETgDuDPZwiBRP\nCUDK3bvOuSm57fvwS+gBDMhd0c8BDgd61Pqd/MyKewDLnXMzAJxzn+Qm2gKYmHu+Fr/ebN1VmT4G\nPjOzO3OrMH1WX3Bm9nXghxQWOBqIX+1uJn4Gxw5m1r75f7ZI6dQGIOWubl29M7NN8IvG93HOLTez\nEcCmDfx+Q4uIr621XUOd/yvOuRozOwC/ROUw4PzcduGDzbbHX/0f45zLJwgDDnTOrWv8zxIJn+4A\npNztbGYH5rZPBibjT/YO+I+ZdcBXteRVAx1z24uAzmbWF8DMOphZ62IKzbUVbOmcGw9cgq8qqv1+\nG+Ah4HLn3L9qvfUscGGt/XoX9VeKhEAJQMrdG8B5ZrYA2BL4P+fcx/gr7/nAM8A/au1/D3Cbmc3A\nf/9PwlfJzMKfnDepp4z6egRVAU+Z2WzgJeDiOu/3A/oCo2o1BnfGn/z3M7PZZjYPOLslf7RIELQe\ngIhISukOQEQkpZQARERSSglARCSllABERFJKCUBEJKWUAEREUkoJQEQkpf4f5uCsRItu1kIAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0a6462d198>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the results\n",
    "plt.errorbar(batch_sizes, mean_times, std_times, marker='.')\n",
    "plt.xscale('log')\n",
    "plt.xlabel('batch size')\n",
    "plt.ylabel('runtime')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}