bootstrap_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hh/jupyterhub/python2-upstream/local/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "import seaborn\n",
    "mpl.rcParams['figure.figsize'] = (16, 10)\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class BootstrapMetricData:\n",
    "    def __init__(self, data, n_bootstrap_iterations, aggregate='mean'):\n",
    "        self.data = data\n",
    "        self.n_points = len(data)\n",
    "        indexes = np.arange(self.data.shape[0], dtype=np.intp)\n",
    "        self.iterator = BootstrapIterator(indexes, n_bootstrap_iterations)\n",
    "        self.aggregate = aggregate\n",
    "\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "\n",
    "    def next(self):\n",
    "        indexes = next(self.iterator)\n",
    "        data = self.data[indexes]\n",
    "        if self.aggregate == 'sum':\n",
    "            return data.sum()\n",
    "        elif self.aggregate == 'mean':\n",
    "            return data.sum() / self.n_points\n",
    "\n",
    "\n",
    "class BootstrapIterator:\n",
    "    def __init__(self, values, n_bootstrap_iterations):\n",
    "        self.values = values\n",
    "        self.n_values = len(values)\n",
    "        self.n_iterations = 0\n",
    "        self.n_bootstrap_iterations = n_bootstrap_iterations\n",
    "\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "\n",
    "    def next(self):\n",
    "        if self.n_iterations >= self.n_bootstrap_iterations:\n",
    "            raise StopIteration\n",
    "        self.n_iterations += 1\n",
    "        return np.random.choice(self.values, self.n_values, replace=True)\n",
    "    \n",
    "\n",
    "def perform_bootstrap_test(first_array, second_array,n_bootstrap_iterations=10**4, aggregate='mean'):\n",
    "    first_array_results = np.array([result for result in BootstrapMetricData(first_array, n_bootstrap_iterations, aggregate=aggregate)])\n",
    "    second_array_results = np.array([result for result in BootstrapMetricData(second_array, n_bootstrap_iterations, aggregate=aggregate)])\n",
    "    delta = second_array_results - first_array_results\n",
    "    left = np.percentile(delta, 2.5)\n",
    "    right = np.percentile(delta, 97.5)\n",
    "    first_and_second_are_the_same = (left < 0.0 < right)\n",
    "    return first_and_second_are_the_same, (delta, left, right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_epmloyers = 10**5\n",
    "employer_values = np.random.rand(n_epmloyers)\n",
    "employer_values_2 = np.random.rand(n_epmloyers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 45.2 s, sys: 8.21 s, total: 53.4 s\n",
      "Wall time: 53.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "first_and_second_are_the_same, (delta, left, right) = perform_bootstrap_test(employer_values, employer_values_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('first array and second array are the same', True)\n"
     ]
    }
   ],
   "source": [
    "print('first array and second array are the same', first_and_second_are_the_same)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x7f0c61865e50>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA68AAAJSCAYAAAAs4/OaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X90XGWdP/DPZNIWkpTQhKRCq0ayYDlItPWsltUKZaGg\nK1UEKyroWlb3iGLllPKrqKzrgrRWvt3DIgcVFcFt0e6ezZ7jUo8CrnvwC+y237ogrCvdNLKFNGlL\naEihkMz3D2ykJJkpyWTmaeb1+oPT3Ofemc8tT2/mPfe5z5PJ5XK5AAAAgIRVlbsAAAAAKER4BQAA\nIHnCKwAAAMkTXgEAAEie8AoAAEDyhFcAAACSV11oh+eeey6uvPLK2LlzZ+zbty8+/elPx8aNG+Ph\nhx+OGTNmRETERRddFKecckq0t7fH7bffHtlsNpYsWRLnnnvuhJ8AAAAAk1+m0DqvP/7xj+PJJ5+M\niy66KLZv3x6f+MQnYt68eXHWWWfFKaecMrTf3r1745xzzokNGzZEdXV1nHfeeXHnnXfGEUccMeEn\nAQAAwORW8M7re97znqE/b9++PY4++uiIiHhl5t2yZUu0tbVFbW1tRETMmzcvNm3aFKeeemoRywUA\nAKASFQyv+51//vmxY8eOuOWWW+I73/lO3HnnnfGd73wnjjrqqLjmmmuip6cnGhoahvZvaGiI7u7u\nCSkaAACAynLQ4XXdunXx2GOPxWWXXRZXX311HHnkkTFnzpz45je/GTfddFO85S1vOWD/AqORAQAA\n4KAVnG34kUceiaeeeioiIubMmRMDAwNx/PHHx5w5cyIi4rTTTovf/OY38ZrXvOaAO61dXV3R3Nyc\n97UFXGCitPyflmj5Py3lLgOGtLS0REtLS7nLgEnLdR8mv4J3Xh966KHYvn17XH311dHT0xP9/f3x\npS99KS6//PJ47WtfGw888EAcf/zx0dbWFtdcc0309fVFJpOJzZs3x8qVK/O+diaTie7uPUU7GWhq\nmq5PERERg4MvfTk23v6gT1Esg4O5qKrye4/ico36g2Jd9yudPkWxNTVNL9prFQyvH/7wh+Pqq6+O\nj370o/H888/Hl770paipqYlLL700Dj/88KitrY3rrrsupk2bFsuXL4+lS5dGVVVVXHLJJVFXV1e0\nQgEAAKhcBcPrtGnTYs2aNcO2/+hHPxq2bdGiRbFo0aLiVAYAAAC/V/CZVwAAACg34RUAAIDkCa8A\nAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7wCgAAQPKEVwAAAJInvAIAAJA84RUAAIDkCa8AAAAkT3gF\nAAAgecIrAAAAyRNeAQAASJ7wCgAAQPKEVwAAAJInvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgecIr\nAAAAyRNeAQAASJ7wCgAAQPKEVwAAAJInvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNe\nAQAASJ7wCgAAQPKEVwAAAJInvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7w\nCgAAQPKEVwAAAJInvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgedXlLgAAKL2BgYHo6Ng6antLy7GR\nzWZLWBEA5Ce8AkAF6ujYGstWt0dNffOwtv7eHbF2xeJobT2uDJUBwMiEVwCoUDX1zVE3Y1a5ywCA\ng+KZVwAAAJInvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7wCgAAQPKEVwAA\nAJInvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASF51uQsAgEPBwMBAdHRsHbW9\npeXYyGazJawIACqL8AoAB6GjY2ssW90eNfXNw9r6e3fE2hWLo7X1uDJUBgCVQXgFgINUU98cdTNm\nlbsMAKhInnkFAAAgecIrAAAAyRNeAQAASJ7wCgAAQPKEVwAAAJInvAIAAJC8gkvlPPfcc3HllVfG\nzp07Y9++ffHpT3865syZEytWrIhcLhdNTU2xatWqmDJlSrS3t8ftt98e2Ww2lixZEueee24pzgEA\nKKLc4GB0dm7Lu09Ly7GRzWZLVBEAHER4veeee+Kkk06Kiy66KLZv3x6f+MQnYt68eXHBBRfEmWee\nGTfeeGNs2LAh3ve+98XNN98cGzZsiOrq6jjvvPPijDPOiCOOOKIU5wEAFMnePd2xZn1P1NQ/OWJ7\nf++OWLticbS2HlfiygCoZAXD63ve856hP2/fvj2OPvroeOihh+LLX/5yREQsXLgwbrvttmhpaYm2\ntraora2NiIh58+bFpk2b4tRTT52YygGACVNT3xx1M2aVuwwAGFIwvO53/vnnx44dO+Ib3/hGLF26\nNKZMmRIREY2NjbFjx47YuXNnNDQ0DO3f0NAQ3d3dxa8YAACAinPQ4XXdunXx2GOPxWWXXRaZTGZo\ney6Xi0wmE7lc7oD9X/kzAAAAjFXB8PrII49EY2NjvOY1r4k5c+bE4OBgHH744bFv376YOnVqdHV1\nRXNzc8ycOTPuvffeoeO6urpi7ty5BQtoapo+vjOAV9CniIioqnrpS7Zi9Ad9ioiI3bvr8rY3NNTl\n7SvF7JPFUOh8Cil0vpSO/w8vSe3f2KHM3yGpKhheH3roodi+fXtcffXV0dPTE/39/bFgwYK4++67\nY/HixbFx48ZYsGBBtLW1xTXXXBN9fX2RyWRi8+bNsXLlyoIFdHfvKcqJQMRLF1t9ioiIwcGXRn+M\ntz/oU+y3a1dfwfZ8fWVwMBdVVZlk+lOh8zmY41M5l0rmGvUHxbruVzp9imIr5pchBcPrhz/84bj6\n6qvjox/9aDz//PNx7bXXxoknnhiXX3553HXXXXHMMcfEOeecE9lsNpYvXx5Lly6NqqqquOSSS6Ku\nbnzf6gLAZDcwMBAdHVvz7jPasjSFjrWcDQCTScHwOm3atFizZs2w7bfddtuwbYsWLYpFixYVpzIA\nqAAdHVtj2er2qKlvHrE937I0+Y61nA0Ak81BT9gEAEyM8SxLY0kbACpFVbkLAAAAgEKEVwAAAJIn\nvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7wCgAAQPKEVwAAAJInvAIAAJA8\n4RUAAIDkCa8AAAAkT3gFAAAgedXlLgAAmDwGBgaio2Nr3n1aWo6NbDZboooAmCyEVwCgaDo6tsay\n1e1RU988Ynt/745Yu2JxtLYeV+LKADjUCa8AMMEGBgbihRdeiGy2Kh5//L8PaOvs3FamqiZOTX1z\n1M2YVe4yAJhkhFcAmGAdHVtj1zP9kanKxlW3/t8D2nY+8Wg0zj6hTJUBwKFDeAWAEshUZaOqKjvs\njmR/b1eZKgKAQ4vZhgEAAEie8AoAAEDyDBsGgHHKDQ7mnXhpMk7KBAClJrwCwDjt3dMda9b3RE39\nkyO273zi0RJXBACTj/AKAEWQb3kYkzIBwPh55hUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAA\nSJ7wCgAAQPIslQMAk1BucDA6O7eN2p6vDQBSJLwCwCS0d093rFnfEzX1T47YvvOJR6Nx9gklrgoA\nxk54BYCE5buDWujuaU19c9TNmDViW39v17hrG4tCd4RbWo6NbDZbwooAOFQIrwAQEQMDA9HRsXXU\n9nINs813B/VQvHua73z6e3fE2hWLo7X1uDJUBkDqhFcAiIiOjq2xbHV71NQ3j9hezqA42h3Uct09\nHa98d4QBYDTCKwAVI9/d1c7ObUkOswUAXiK8AlAx8t1dPRSH4AJAJRFeAagok20IbjmMZxIpABgr\n4RUAeFVSnESq0IRbZjEGOPQJrwDAq5baHex8Q8LNYgwwOQivAMCkYBZjgMmtqtwFAAAAQCHCKwAA\nAMkTXgEAAEie8AoAAEDyhFcAAACSJ7wCAACQPOEVAACA5AmvAAAAJE94BQAAIHnCKwAAAMkTXgEA\nAEie8AoAAEDyhFcAAACSJ7wCAACQPOEVAACA5FWXuwAAKKaBgYHo6Ng6Yltn57YSVwMAFIvwCsCk\n0tGxNZatbo+a+uZhbTufeDQaZ59QhqoAgPESXgGYdGrqm6Nuxqxh2/t7u8pQDQBQDJ55BQAAIHnC\nKwAAAMkTXgEAAEie8AoAAEDyhFcAAACSJ7wCAACQPOEVAACA5AmvAAAAJE94BQAAIHnCKwAAAMkT\nXgEAAEhe9cHstGrVqti0aVMMDAzEpz71qbjnnnvi4YcfjhkzZkRExEUXXRSnnHJKtLe3x+233x7Z\nbDaWLFkS55577oQWDwAAQGUoGF4feOCBePzxx2PdunXx9NNPxznnnBPz58+Pyy67LE455ZSh/fbu\n3Rs333xzbNiwIaqrq+O8886LM844I4444ogJPQEAAAAmv4Lh9W1ve1u8+c1vjoiI+vr66O/vj8HB\nwcjlcgfst2XLlmhra4va2tqIiJg3b15s2rQpTj311OJXDQAAQEUp+MxrJpOJww47LCIi7rrrrjj1\n1FOjqqoq7rjjjvj4xz8ey5cvj927d0dPT080NDQMHdfQ0BDd3d0TVzkAAAAV46CeeY2I+OlPfxr/\n8A//EN/+9rfj4YcfjiOPPDLmzJkT3/zmN+Omm26Kt7zlLQfs/8o7s6Npapr+6iqGAvQpIiKqqjIR\nUZz+oE8dWnbvrit3CYxDQ0PdmP7NFfr/PtbXPRRM1vN6tYp53a90/g5J1UGF11/84hdx6623xre/\n/e2oq6uL+fPnD7Wddtppce2118ZZZ50V991339D2rq6umDt3bsHX7u7e8+qrhlE0NU3Xp4iIiMHB\nl75AG29/0KcOPbt29ZW7BMZh166+Mf2bK/T/fayvmzrXqD8o1nW/0ulTFFsxvwwpOGy4r68vVq9e\nHbfccktMn/7SG3/uc5+L3/3udxHx0oROxx9/fLS1tcXDDz8cfX198eyzz8bmzZvjrW99a9EKBQAA\noHIVvPP64x//OJ5++un4/Oc/H7lcLjKZTHzgAx+ISy+9NA4//PCora2N6667LqZNmxbLly+PpUuX\nRlVVVVxyySVRV2foFgAAAONXMLwuWbIklixZMmz7+9///mHbFi1aFIsWLSpOZQAAAPB7BYcNAwAA\nQLkd9GzDAAATKTc4GJ2d2/Lu09JybGSz2RJVBEBKhFcAIAl793THmvU9UVP/5Ijt/b07Yu2KxdHa\nelyJKwMgBcIrAJCMmvrmqJsxq9xlAJAg4RWAQ8rAwEB0dGwdtb3QsFMA4NAkvAJwSOno2BrLVrdH\nTX3ziO07n3g0GmefUOKqAICJJrwCcMjJN7S0v7erxNUAAKVgqRwAAACSJ7wCAACQPMOGAYBDQr51\nYE3UBTD5Ca8AwCEh3zqwJuoCmPyEVwDgkDHaZF0m6gKY/DzzCgAAQPKEVwAAAJInvAIAAJA84RUA\nAIDkCa8AAAAkz2zDACRnYGAgOjq2jthmPU8AqEzCKwDJ6ejYGstWt0dNffOwNut5Umz5viyJiGhp\nOTay2WwJKwJgJMIrAEmynielku/Lkv7eHbF2xeJobT2uDJUB8HLCKwBQ8Ub7sgSAdJiwCQAAgOQJ\nrwAAACRPeAUAACB5wisAAADJE14BAABInvAKAABA8oRXAAAAkie8AgAAkDzhFQAAgOQJrwAAACRP\neAUAACB5wisAAADJE14BAABInvAKAABA8oRXAAAAkldd7gIAACZSbnAwOju3jdqerw2AdAivAMCk\ntndPd6xZ3xM19U+O2L7ziUejcfYJJa4KgFdLeAUAJr2a+uaomzFrxLb+3q4SVwPAWHjmFQAAgOQJ\nrwAAACRPeAUAACB5wisAAADJE14BAABInvAKAABA8oRXAAAAkie8AgAAkDzhFQAAgOQJrwAAACRP\neAUAACB5wisAAADJE14BAABInvAKAABA8oRXAAAAkie8AgAAkLzqchcAAJCq3OBgdHZuy7tPS8ux\nkc1mS1QRQOUSXgEARrF3T3esWd8TNfVPjtje37sj1q5YHK2tx5W4MoDKI7wCAORRU98cdTNmlbsM\ngIonvAJQcgMDA9HRsXXU9kLDNAGAyiO8AlByHR1bY9nq9qipbx6xfecTj0bj7BNKXBUAkDLhFYCy\nyDcUs7+3q8TVAACps1QOAAAAyRNeAQAASJ7wCgAAQPKEVwAAAJInvAIAAJA84RUAAIDkCa8AAAAk\nT3gFAAAgecIrAAAAyas+mJ1WrVoVmzZtioGBgfjUpz4VJ510UqxYsSJyuVw0NTXFqlWrYsqUKdHe\n3h633357ZLPZWLJkSZx77rkTXT8AAAAVoGB4feCBB+Lxxx+PdevWxdNPPx3nnHNOzJ8/Py644II4\n88wz48Ybb4wNGzbE+973vrj55ptjw4YNUV1dHeedd16cccYZccQRR5TiPAAAAJjECg4bftvb3hZr\n166NiIj6+vro7++Phx56KE477bSIiFi4cGHcf//9sWXLlmhra4va2tqYNm1azJs3LzZt2jSx1QMA\nAFARCobXTCYThx12WERE/PCHP4xTTz019u7dG1OmTImIiMbGxtixY0fs3LkzGhoaho5raGiI7u7u\nCSobAACASnLQEzb99Kc/jQ0bNsQXvvCFA7bncrnIZDKRy+WGbQcAAIBiOKgJm37xi1/ErbfeGt/+\n9rejrq4uampqYt++fTF16tTo6uqK5ubmmDlzZtx7771Dx3R1dcXcuXMLvnZT0/SxVw8j0KeIiKiq\nykREcfqDPlV8u3fXlbsEKJqGhrqyXidco15SzOt+pfN3SKoKhte+vr5YvXp1fPe7343p01/qyCef\nfHJs3Lgxzj777Ni4cWMsWLAg2tra4pprrom+vr7IZDKxefPmWLlyZcECurv3jP8s4PeamqbrU0RE\nxODgS6M/xtsf9KmJsWtXX7lLgKLZtauvbNcJ16g/KNZ1v9LpUxRbMb8MKRhef/zjH8fTTz8dn//8\n54eGCN9www2xcuXKWL9+fRxzzDFxzjnnRDabjeXLl8fSpUujqqoqLrnkkqir8806AAAA41cwvC5Z\nsiSWLFkybPttt902bNuiRYti0aJFxakMAAAAfu+gJ2wCAACAchFeAQAASJ7wCgAAQPKEVwAAAJIn\nvAIAAJC8grMNAwAwstzgYHR2bhu1vaXl2MhmsyWsCGDyEl4BAMZo757uWLO+J2rqnxzW1t+7I9au\nWBytrceVoTKAyUd4BQAYh5r65qibMavcZQBMep55BQAAIHnCKwAAAMkTXgEAAEie8AoAAEDyhFcA\nAACSJ7wCAACQPEvlADAmAwMD0dGxNe8+LS3HRjabLVFFAMBkJrwCMCYdHVtj2er2qKlvHrG9v3dH\nrF2xOFpbjytxZQDAZCS8AjBmNfXNUTdjVrnLAAAqgGdeAQAASJ7wCgAAQPKEVwAAAJInvAIAAJA8\n4RUAAIDkCa8AAAAkT3gFAAAgedZ5BWBUAwMD0dGxdcS2zs5tJa4GAKhkwisAo+ro2BrLVrdHTX3z\nsLadTzwajbNPKENVAEAlEl4ByKumvjnqZswatr2/t6sM1QAAlUp4BWBC5AYHRx1abMgxAPBqCa8A\nTIi9e7pjzfqeqKl/clibIcdUgnxf4OzX0nJsZLPZElUEcGgTXgGYMIYcU8nyfYETEdHfuyPWrlgc\nra3HlbgygEOT8AoAMEFG+wIHgFfPOq8AAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7wCgAAQPKEVwAA\nAJInvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASF51uQsAAKhEucHB6OzcNmp7\nS8uxkc1mS1gRQNqEVwCAMti7pzvWrO+Jmvonh7X19+6ItSsWR2vrcWWoDCBNwisAQJnU1DdH3YxZ\n5S4D4JDgmVcAAACSJ7wCAACQPOEVAACA5AmvAAAAJE94BQAAIHnCKwAAAMkTXgEAAEie8AoAAEDy\nhFcAAACSJ7wCAACQPOEVAACA5AmvAAAAJE94BQAAIHnCKwAAAMmrLncBAJTXwMBAdHRsHbGts3Nb\niasBABiZ8ApQ4To6tsay1e1RU988rG3nE49G4+wTylAVAMCBhFcAoqa+OepmzBq2vb+3qwzVAAAM\n55lXAAAAkie8AgAAkDzhFQAAgOQJrwAAACRPeAUAACB5wisAAADJE14BAABInvAKAABA8g4qvP7m\nN7+JM844I+68886IiLjqqqvi7LPPjo997GPxsY99LH7+859HRER7e3ucd9558aEPfSg2bNgwcVUD\nAABQUaoL7bB37974yle+EieffPIB2y+77LI45ZRTDtjv5ptvjg0bNkR1dXWcd955ccYZZ8QRRxxR\n/KoBAACoKAXvvE6bNi2+9a1vRXNzc979tmzZEm1tbVFbWxvTpk2LefPmxaZNm4pWKAAAAJWrYHit\nqqqKqVOnDtt+xx13xMc//vFYvnx57N69O3p6eqKhoWGovaGhIbq7u4tbLQAAABWp4LDhkbzvfe+L\nI488MubMmRPf/OY346abboq3vOUtB+yTy+WKUiAAAACMKbzOnz9/6M+nnXZaXHvttXHWWWfFfffd\nN7S9q6sr5s6dW/C1mpqmj6UEGJU+RUREVVUmIorTHyZ7n9q9u67cJQAjaGioO6jrz2S/Rh2sYl73\nK52/Q1I1pvD6uc99LlasWBGvfe1r44EHHojjjz8+2tra4pprrom+vr7IZDKxefPmWLlyZcHX6u7e\nM5YSYERNTdP1KSIiYnDwpdEf4+0PldCndu3qK3cJwAh27eoreP2phGvUwSrWdb/S6VMUWzG/DCkY\nXh955JH46le/Gtu3b4/q6urYuHFjXHjhhXHppZfG4YcfHrW1tXHdddfFtGnTYvny5bF06dKoqqqK\nSy65JOrqfJsPAADA+BUMryeeeGJ8//vfH7b9jDPOGLZt0aJFsWjRouJUBgAAAL9XcLZhAAAAKDfh\nFQAAgOQJrwAAACRPeAUAACB5wisAAADJE14BAABInvAKAABA8oRXAAAAkie8AgAAkLzqchcAAMCB\ncoOD0dm5bdT2lpZjI5vNlrAigPITXgEAErN3T3esWd8TNfVPDmvr790Ra1csjtbW48pQGUD5CK8A\nAAmqqW+Ouhmzyl0GQDI88woAAEDyhFcAAACSJ7wCAACQPOEVAACA5AmvAAAAJM9swwAAh5CXrwG7\ne3dd7NrVN2wf68ACk5HwCgBwCMm3BmyEdWCByUt4BQA4xFgDFqhEnnkFAAAgecIrAAAAyRNeAQAA\nSJ7wCgAAQPKEVwAAAJInvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7wCgAA\nQPKEVwAAAJInvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7wCgAAQPKEVwAA\nAJJXXe4CAJhYAwMD0dGxddT2zs5tJawGAGBshFeASa6jY2ssW90eNfXNI7bvfOLRaJx9QomrAgB4\ndYRXgApQU98cdTNmjdjW39tV4moAAF494RUAYBLJDQ7mfRygpeXYyGazJawIoDiEVwCASWTvnu5Y\ns74nauqfHNbW37sj1q5YHK2tx5WhMoDxEV4BJoF8kzKZkAkqT75HBQAOVcIrwCSQb1ImEzIBAJOB\n8AowSYx2p8WETADAZFBV7gIAAACgEOEVAACA5AmvAAAAJE94BQAAIHnCKwAAAMkTXgEAAEie8AoA\nAEDyhFcAAACSJ7wCAACQvOpyFwAAQBoGBgaio2PrqO0tLcdGNpstYUUAfyC8AgAQEREdHVtj2er2\nqKlvHtbW37sj1q5YHK2tx5WhMgDhFQCAl6mpb466GbPKXQbAMJ55BQAAIHnuvAIkwrNmQMpyg4PR\n2bkt7z6uU8BEEl4BEuFZMyBle/d0x5r1PVFT/+SI7a5TwEQTXgES4lkzIGWuUUA5eeYVAACA5Amv\nAAAAJE94BQAAIHnCKwAAAMkTXgEAAEie8AoAAEDyDiq8/uY3v4kzzjgj7rzzzoiIeOqpp+LCCy+M\nCy64IC699NJ44YUXIiKivb09zjvvvPjQhz4UGzZsmLiqAQAAqCgFw+vevXvjK1/5Spx88slD29au\nXRsXXnhh3HHHHfG6170uNmzYEHv37o2bb745vve978Xtt98e3/3ud+OZZ56Z0OIBAACoDAXD67Rp\n0+Jb3/pWNDc3D2178MEHY+HChRERsXDhwrj//vtjy5Yt0dbWFrW1tTFt2rSYN29ebNq0aeIqBwAA\noGIUDK9VVVUxderUA7bt3bs3pkyZEhERjY2NsWPHjti5c2c0NDQM7dPQ0BDd3d1FLhcAAIBKVD2W\ngzKZzNCfc7lcZDKZyOVyB+zzyp9H09Q0fSwlwKj0KSIiqqpeuk4Voz+Uqk/t3l2Xt72hoW7UWgod\nC7DfRF5L8r32RCvmdb/S+TskVWMKrzU1NbFv376YOnVqdHV1RXNzc8ycOTPuvffeoX26urpi7ty5\nBV+ru3vPWEqAETU1TdeniIiIwcGXvkAbb38oZZ/atatv1Lbc4GD8v//3yKj7dHZum6iygElm166+\nUa9r+a5D433tiVas636l81mKYivmlyFjCq8nn3xybNy4Mc4+++zYuHFjLFiwINra2uKaa66Jvr6+\nyGQysXnz5li5cmXRCgWoZHv3dMea9T1RU//kiO07n3g0GmefUOKqAABKp2B4feSRR+KrX/1qbN++\nPaqrq2Pjxo3xta99La688spYv359HHPMMXHOOedENpuN5cuXx9KlS6OqqiouueSSqKszjA2gWGrq\nm6NuxqwR2/p7u0pcDQBAaRUMryeeeGJ8//vfH7b9tttuG7Zt0aJFsWjRouJUBgAAAL9XcLZhAAAA\nKDfhFQAAgOSNacImAF69gYGB6OjYOmq7GYMBAEYnvAKUSEfH1li2uj1q6ptHbDdjMADA6IRXgBIy\nYzAAwNh45hUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7wCgAAQPKEVwAAAJJnnVcAgAqR\nGxyMzs5to7bnawMoN+EVAKBC7N3THWvW90RN/ZMjtu984tFonH1CiasCODjCKwBABampb466GbNG\nbOvv7SqE/XDnAAAUL0lEQVRxNQAHzzOvAAAAJE94BQAAIHnCKwAAAMkTXgEAAEie8AoAAEDyhFcA\nAACSZ6kcAAAm1MDAQHR0bM27T0vLsZHNZktUEXAoEl4BAJhQHR1bY9nq9qipbx6xvb93R6xdsTha\nW48rcWXAoUR4BQBgwtXUN0fdjFnlLgM4hHnmFQAAgOQJrwAAACTPsGGAIso3KUln57YSVwMAMHkI\nrwBFlG9Skp1PPBqNs08oQ1UAAIc+4RWgyEablKS/t6sM1QAATA7CKwAA45YbHBz18QiPTQDFILwC\nADBue/d0x5r1PVFT/+SwNo9NAMUgvAIAUBQemwAmkqVyAAAASJ7wCgAAQPKEVwAAAJInvAIAAJA8\n4RUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7wCgAAQPKEVwAAAJInvAIAAJA84RUAAIDk\nCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7wCgAAQPKqy10AwKFkYGAgOjq2jtre2bmthNUAAFQO\n4RXgVejo2BrLVrdHTX3ziO07n3g0GmefUOKqAAAmP+EV4FWqqW+OuhmzRmzr7+0qcTUAAJXBM68A\nAAAkT3gFAAAgeYYNA4esfJMnvfDCC1Fd7RIHADBZ+GQHHLLyTZ606+j+aDiipgxVAQAwEYRX4JA2\n2uRJmapsGaoBAGCieOYVAACA5AmvAAAAJM+wYQAAkpVvcr6IiJaWYyOb9agIVALhFQCAZOWbnK+/\nd0esXbE4WluPK0NlQKkJrwAAJG20yfmAyiK8AgBQVrnBwejs3DZi22jbgcojvAIAUFZ793THmvU9\nUVP/5LC2nU88Go2zTyhDVUBqhFcAAMputKHB/b1dZagGSJGlcgAAAEie8AoAAEDyhFcAAACS55lX\ngFcYGBiIjo6tERGxe3dd7NrVN9Rm1ksAgPIYU3h98MEHY9myZXHcccdFLpeLN77xjfEXf/EXsWLF\nisjlctHU1BSrVq2KKVOmFLtegAnX0bE1lq1uj5r65mFtZr0EACiPMd95fdvb3hZr164d+vmqq66K\nCy+8MBYtWhQ33nhjbNiwIc4///yiFAlQama9BABIy5ifec3lcgf8/OCDD8bChQsjImLhwoVx//33\nj68yAAAA+L0x33l9/PHH4+KLL47e3t74zGc+E88999zQMOHGxsbo7u4uWpEAAABUtjGF19e//vXx\n2c9+Nt797nfH7373u/jYxz4WL7744lB7LpeLTCZTtCIBAOCVcoODQxPpvfDCCxER8fjj/33APi0t\nx0Y2my15bUDxjSm8zpw5M9797ndHRMRrX/vaOOqoo+Lhhx+Offv2xdSpU6OrqyuampoO6rWamqaP\npQQYlT5VOXbvrsvbns1Wjak/FHpdANKwd093rFnfEzX1T8buWc9HRMRVt/7fofb+3h3x/es/Escf\nf3y5Sjwk+SxFqsYUXv/5n/85uru7Y+nSpdHd3R07d+6MD3zgA3H33XfH4sWLY+PGjbFgwYKDeq3u\n7j1jKQFG1NQ0XZ+qIC9fwmYkAwODY+oPhV4XgHTsn2Cvquqlu6uvnGxv164+nw1eBZ+lKLZifhky\npvB62mmnxfLly+NnP/tZvPjii/FXf/VXMWfOnLjiiivirrvuimOOOSbOOeecohUJAABAZRtTeK2t\nrY1bbrll2Pbbbrtt3AUBAADAK415qRwAAAAolTEvlQOQtFzEwMCLw2ad3G9gYCAiMpHNDv8Ob//M\nlQAApEN4BSalXG4gep8dOGDWyZfb+cSjcfj0xqipbx6xrXH2CRNdIgAAr4LwCkxamarssFkn9+vv\n7RqaoXKkNgAA0uKZVwAAAJInvAIAAJA84RUAAIDkCa8AAAAkT3gFAAAgecIrAAAAyRNeAQAASJ7w\nCgAAQPKEVwAAAJInvAIAAJA84RUAAIDkCa8AAAAkr7rcBQAAwETIDQ5GZ+e2UdtbWo6NbDZbwoqA\n8RBeAQCYlPbu6Y4163uipv7JYW39vTti7YrF0dp6XBkqA8ZCeAUAYNKqqW+Ouhmzyl0GUASeeQUA\nACB5wisAAADJE14BAABInmdeAQCoOIVmIo4wGzGkRngFAKDi5JuJOMJsxJAi4RWYUAMDA9HRsTXv\nPr7ZBqAczEQMhxbhFZhQHR1bY9nq9qipbx6x3TfbAAAcDOEVmHC+2QYAYLyEVyBZhYYcF5poAwCA\nyUN4BZJVaMjxzicejcbZJ5S4KgAAykF4BZKWb8hxf29XiasBoFIUWkrHZINQesIrAAC8Qr6ldEw2\nCOUhvAIAwAhMOAhpqSp3AQAAAFCI8AoAAEDyhFcAAACS55lXYNzyrcdqLVYAAIpBeAXGLd96rNZi\nBQCgGIRXoChGm5HRWqwATDaF1oCNsA4sTAThFQAAXoV8a8BGRDz79FNx2flz43Wve/2I7YItjI3w\nCgAAr1K+NWD7e7tizfotI4bb/t4dsXbF4mhtPW6iS4RJR3gFAIAiyxdugbERXoGyyvfckJmKAQDY\nT3gFyirfc0NmKgYAYD/hFSg7MxUDAFBIVbkLAAAAgEKEVwAAAJJn2DAAAJRIvokKI6wBC/kIrwAA\nUCL5Jiq0BizkJ7wCAEAJWQMWxsYzrwAAACRPeAUAACB5wisAAADJE14BAABInvAKAABA8sw2DAAA\nCSi0BmyEdWCpbMIrAAAkIN8asBHWgQXhFSaRgYGB6OjYmrc9IhPZ7MhPDPg2FwDKyxqwMDrhFSaR\njo6tsWx1e9TUN4/YvvOJR+Pw6Y0jthf6NjdfMC40xAkAAMZLeIVJJt83tv29XWP+RjdfMN75xKPR\nOPuEV/2aAABwsIRXSEyhob8REzO8t9AkEZ2d20YNvv29XUWtBQAYrtDvao//MNkJr5CYQkN/J2qy\nhkKTRLi7CgDlle939bNPPxWXnT83Xve61496vHDLoU54hQSVa7KGQkOOAYDyyjcKas36LWYqZlIT\nXgEAYBIwUzGT3cjrZQAAAEBChFcAAACSZ9gwAABMcmYqZjIQXgEAYJLLN1OxyZw4VAivAABQAcY6\noVOhNejdtaVUhFcYo3wX8oGBgYjIRDY7+mPlY73Q5xv2k284EADASF7+2WL37rrYtavvgPbOzm2/\nX4Zn+Br07tpSSsIrjFFHx9ZYtrp9xAv5zicejcOnN47YFjG+C32+YT87n3g0Gmef8KpfEwCoXPk+\nW0T84fOFZXgoN+GVilZoGExE/juk+RYKn8i11vK9LwDAq5Xvc8t4Pl8cakOOx/vZkIlV9PB6/fXX\nx5YtWyKTycTVV18dJ510UrHfAoom393TCENhAADGI99nrRQ/Z/lsmLaihteHHnootm3bFuvWrYvH\nH388Vq5cGevWrSvmWzBJlfNbrnzfNHq+FABgdIWW4Ons3Daho9FGMlEj6yi/oobXX/7yl3H66adH\nRERra2s888wz8eyzz0ZtbW0x34ZDVL4LSb6JACIinn36qbjs/Lnxute9fsTX3T850kiTDOSbPKlQ\nAPV8KQDA6A72edmR5Au+45n8slx3T8s1mWclKWp47enpiTe96U1DP8+YMSN6enqE1wT9+WevjkzN\nzBHbDo9n4uZVXyj6exaa4CjfRAD9vV2/D7cjh8h8kyPlaz+YAOr5UgCA0Y31edlCNwnGM/llOe6e\nlmsyz0pS1PCay+WG/ZzJZIr5FhRJbug/I7c+/vh/F/09C93l7O/dMWrb3j274vDpjcUu6aDeN2Lk\nPjzWtv3vmW848kTUVK5jy1VTbuZgRET07f7for5vin/H5To2xZrGc+xE15QbHIjBGN4nD9XzOZSO\nTbGm8RybYk3lOvblbYM1AxFx4L+xQ/l8KuXY8Xy+G+tnqfF8Dit0LBMrk3tl4hyHm266KZqbm2PJ\nkiUREXH66adHe3t71NTUFOstAAAAqECjD7oeg3e84x2xcePGiIj49a9/HTNnzhRcAQAAGLeiDhue\nO3dunHjiiXH++edHNpuNL37xi8V8eQAAACpUUYcNAwAAwEQo6rBhAAAAmAjCKwAAAMkTXgEAAEhe\nUSds2u/FF1+MK6+8MrZv3x7ZbDauv/76mD179gH7tLe3x+233x7ZbDaWLFkS55577qjH9fX1xaWX\nXhq9vb3xmte8JtasWRNTpkyZiNJJVLH71H7r1q2LW2+9Ne65555SnxJlVuw+9dhjj8Vf//VfR1VV\nVdTX18eaNWti2rRpZTo7Sun666+PLVu2RCaTiauvvjpOOumkobb7778/brzxxshms/Gud70rLr74\n4lGPeeqpp2LFihWRy+WiqakpVq1a5XddhSpmn7rqqqvixRdfjClTpsTq1aujsXFi1kwnbcXqU/v9\n4he/iE9+8pPx2GOPlfxcSEOx+tSLL74YV1xxRXR2dkZdXV387d/+bUyfPn30N85NgH/8x3/MffnL\nX87lcrncv/3bv+U+//nPH9De39+fO/PMM3N9fX255557Lvfe974319vbO+pxq1atyn3ve9/L5XK5\n3N/93d/lfvWrX01E2SSs2H0ql8vldu7cmVu6dGnutNNOK92JkIxi96kLLrgg95//+Z+5XC6Xu+GG\nG3I/+MEPSng2lMuDDz6Y+8u//MtcLpfL/fa3v8196EMfOqD9Pe95T+6pp57KDQ4O5j7ykY/kfvvb\n3456zJVXXpnbuHFjLpfL5b7+9a/n/v7v/76EZ0Iqitmnrrjiitzdd9+dy+VyuTvuuCO3atWqEp4J\nqShmn8rlcrnnn38+d8EFF+QWLFhQupMgKcXsU3feeWfub/7mb3K5XC5311135e6555687z0hw4Z/\n+ctfxumnnx4REX/yJ38SmzZtOqB9y5Yt0dbWFrW1tTFt2rSYN29e/Md//Mew4zZv3hwREffee2+8\n973vjYiIiy+++IBkT2UoVp96+XGrV6+OZcuWle4kSEqx+9Qtt9wSb3rTmyIioqGhIZ5++ukSng3l\n8vL+0NraGs8880w8++yzERHxu9/9Lo488siYOXNmZDKZOOWUU+KXv/zliMf09fXFgw8+GAsXLoyI\niIULF8b9999fnpOirIrVp5599tm49tprY9GiRRHx0nWpt7e3PCdFWRWzT0W89PvuggsuMDKkghXz\nd9+9994bZ599dkREfPCDHxz6PTiaCQmvPT090dDQEBERmUwmqqqq4sUXXxyxPeKlC2p3d/ew4zKZ\nTLzwwgvR09MT69ati49+9KNx7bXXxgsvvDARZZOwYvWp/cc98MADcdhhh0VbW1vkrBZVkYrdp2pr\nayMior+/P/7pn/4pzjzzzBKeDeXyyn4yY8aM6OnpGbFtpD60f3tPT08899xzQx8GGxsbo7u7u0Rn\nQUqK0af2H3PYYYdFJpOJwcHB+MEPfjB0I4DKUsw+1dHREf/1X/8VZ555ps9PFayYv/v+93//N37+\n85/HhRdeGMuXL49nnnkm73uP+5nXH/7wh/GjH/0oMplMRETkcrn41a9+dcA+g4ODQ+3793m5XC53\nQPsrtz///PPxzne+My6++OL4whe+ED/84Q/jIx/5yHhLJ1ET3acGBwfjpptuiptvvnkCqidFE9mn\nXn5cf39/XHzxxXHRRRfFscceW+zTIEH5+slIbaO9xv4vbEd6HSpLMftUxEvXqBUrVsT8+fNj/vz5\nE1AxqStmn7ruuuvimmuumZhCOWQUo08NDg5GVVVV5HK5aG1tjc9+9rPxjW98I2655Za4/PLLR33v\ncYfXD37wg/HBD37wgG1XXXVV9PT0xBvf+MahOxnZbHaofebMmXHfffcN/dzV1RVz586N5ubmA47L\n5XJRXV0dRx99dLS1tUVExDve8Y548MEHx1s2CZvoPvXYY49FT09PfPKTn4xcLhc9PT2xfPnyWLNm\nTUnOj9KbyD61/7iBgYH4zGc+E4sXL473v//9E39SJGHmzJlD3zZHROzYsSOOOuqoobaX3z3t6uqK\n5ubmmDJlyrBjmpqa4vDDD499+/bF1KlTo6urK5qamkp3IiSjWH1q/zFXXXVVvOENb4jPfOYzJToD\nUlOsPjVlypT4n//5n6GJ5bq7u+PCCy+M73//+6U7GZJQjD7V3d0dRx11VBx11FHxx3/8xxER8c53\nvjNuuummvO89IcOG3/GOd8Tdd98dERH33HNPvP3tbz+g/c1vfnM8/PDD0dfXF88++2xs3rw53vrW\nt4563Pz58+OBBx6IiIhHHnkk3vCGN0xE2SSsmH2qra0t/uVf/iXWrVsX69evj6OOOkpwrUDFvk7d\neuut8fa3vz0+8IEPlPZEKKt3vOMdsXHjxoiI+PWvfx0zZ86MmpqaiIiYNWtWPPvss7F9+/Z48cUX\n47777ot3vvOdw45pbm6OmpqaOPnkk4e2b9y4MRYsWFCek6KsitGn9h/T3t4eU6dOjc9+9rNlOx/K\nr1h96uijj46f/OQnQ5+fmpqaBNcKVczffe9617viX//1XyPi4HJeJjcBA9YHBwdj5cqVsW3btpg2\nbVp89atfjZkzZw59uHvzm98cP/nJT+Jb3/pWVFVVxYUXXhh/9md/Nupxu3btihUrVsTzzz8fjY2N\nccMNN8Rhhx1W7LJJWLH71Mv96Z/+afzsZz8r05lRLsXuUwsWLIjZs2dHdXV1ZDKZmD9//tDU8Exu\nX//61+PBBx+MbDYbX/ziF+PXv/51TJ8+PU4//fT493//9/ja174WERFnnXVW/Pmf//mIx7zxjW+M\n7u7uuOKKK2Lfvn1xzDHHxPXXX3/AaAAqx3j71Je+9KU4/vjj4/zzz499+/ZFbW1tZDKZ+KM/+qP4\n4he/WMYzo1yKdZ16OZ+fKlux+tRzzz0Xl19+efT09ERtbW3ccMMNBzwb+0oTEl4BAACgmCZk2DAA\nAAAUk/AKAABA8oRXAAAAkie8AgAAkDzhFQAAgOQJrwAAACRPeAUAACB5wisAAADJ+/+9IwfKksaT\nUgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0c61c91c90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(delta, bins=100)\n",
    "plt.axvline(0.0, color='k')\n",
    "plt.axvline(left, color='g')\n",
    "plt.axvline(right, color='g')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2 (upstream libs)",
   "language": "python",
   "name": "python2-upstream"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}