IR+Evaluation+and+Reference+Collections.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "%matplotlib inline\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from IPython.display import display, HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "recall = np.zeros( 20, dtype=np.float32)\n",
    "\n",
    "recall[0:2] = 1/11\n",
    "\n",
    "recall[2:7] = 2/11\n",
    "\n",
    "recall[7:8] = 3/11\n",
    "\n",
    "recall[8:12] = 4/11\n",
    "\n",
    "recall[12:14] = 5/11\n",
    "\n",
    "recall[14:19] = 6/11\n",
    "\n",
    "recall[-1] = 7/11\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.09090909,  0.09090909,  0.18181819,  0.18181819,  0.18181819,\n",
       "        0.18181819,  0.18181819,  0.27272728,  0.36363637,  0.36363637,\n",
       "        0.36363637,  0.36363637,  0.45454547,  0.45454547,  0.54545456,\n",
       "        0.54545456,  0.54545456,  0.54545456,  0.54545456,  0.63636363], dtype=float32)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "precision = np.zeros(20, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  1.,  2.,  2.,  2.,  2.,  2.,  3.,  4.,  4.,  4.,  4.,  5.,\n",
       "        5.,  6.,  6.,  6.,  6.,  6.,  7.], dtype=float32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision[0:2] = 1\n",
    "precision[2:7] = 2\n",
    "precision[7:8] = 3\n",
    "precision[8:12] = 4\n",
    "precision[12:14] = 5\n",
    "precision[14:19] = 6\n",
    "precision[-1] = 7\n",
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  1.00000000e+00,   1.25000000e-01,   7.40740741e-02,\n",
       "         3.12500000e-02,   1.60000000e-02,   9.25925926e-03,\n",
       "         5.83090379e-03,   5.85937500e-03,   5.48696845e-03,\n",
       "         4.00000000e-03,   3.00525920e-03,   2.31481481e-03,\n",
       "         2.27583068e-03,   1.82215743e-03,   1.77777778e-03,\n",
       "         1.46484375e-03,   1.22124975e-03,   1.02880658e-03,\n",
       "         8.74763085e-04,   8.75000000e-04])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prec = precision / np.arange(1,21)\n",
    "prec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Precision - Recall Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiAAAAF5CAYAAACm4JG+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XuYXXV97/H3NyRcEiQq4aJSESuCHCkwozyJolCiIlC0\n3qCjnlqEUijeUhEFrVy0tWhJDtRGObWCERhLfSyFVoUGqVi5WBPAgqBwhCIgMRQcVEITyPf8sdZ0\nJsNMklmz92/v2Xm/nmc/M/u3f2ut7/5lZ+Yzv3WLzESSJKmkGZ0uQJIkbXkMIJIkqTgDiCRJKs4A\nIkmSijOASJKk4gwgkiSpOAOIJEkqzgAiSZKKM4BIkqTiDCCSJKm4rgggEfGqiLgiIh6IiPUR8YbN\nWOaQiFgREU9ExI8j4l0lapUkSVPXFQEEmAPcApwMbPLmNBHxAuCfgGuA/YDzgC9ExGvbV6IkSWqV\n6Lab0UXEeuB3M/OKjfQ5Bzg8M39rVNsgMDczjyhQpiRJmoJumQGZrPnA8jFtVwELOlCLJEmapOka\nQHYFVo1pWwXsEBHbdKAeSZI0CTM7XUALRf113H1KEbEjcBhwL/BEoZokSeoF2wIvAK7KzP9qxQqn\nawB5CNhlTNvOwGOZuXaCZQ4DLmlrVZIk9bZ3AJe2YkXTNYDcABw+pu11dftE7gW4+OKLeclLXtKm\nsrrH298O++4Lp5329NcWLVrEkiVLyhfVZRyHEY5FxXEY4VhUHIfKHXfcwTvf+U6of5e2QlcEkIiY\nA7yIkd0oL4yI/YBHMvOnEfEp4LmZOXytj88D76nPhvkisBB4K7CxM2CeAHjJS15CX19fO95GV5k9\nG3baCcZ7q3Pnzt0ixmBTHIcRjkXFcRjhWFQch6dp2SEM3XIQ6suAm4EVVMdwnAusBM6qX98V+I3h\nzpl5L3Ak8Bqq64csAo7LzLFnxkiSpC7UFTMgmfltNhKGMvPYCZbpb2ddkiSpPbplBkSSJG1BDCBb\noIGBgU6X0BUchxGORcVxGOFYVByH9um6S7G3S0T0AStWrFixRRxQ1NcH8+fD0qWdrkSSNN2tXLmS\n/v5+gP7MXNmKdToDIkmSijOASJKk4gwgkiSpOAOIJEkqzgAiSZKKM4BIkqTiDCCSJKk4A4gkSSrO\nACJJkoozgEiSpOIMIJIkqTgDiCRJKs4AIkmSijOASJKk4gwgkiSpOAOIJEkqzgAiSZKKM4BIkqTi\nDCCSJKk4A4gkSSrOACJJkoozgEiSpOIMIJIkqTgDiCRJKs4AIkmSijOASJKk4gwgkiSpOAOIJEkq\nzgAiSZKKM4BIkqTiDCCSJKk4A4gkSSrOACJJkoozgEiSpOIMIJIkqTgDiCRJKs4AIkmSijOASJKk\n4gwgkiSpOAOIJEkqzgAiSZKKM4BIkqTiDCCSJKk4A4gkSSrOACJJkoozgEiSpOIMIJIkqTgDiCRJ\nKq5rAkhEnBwR90TEmoi4MSJevon+H4iIOyPi8Yi4LyIWR8Q2peqVJEnNdUUAiYhjgHOBM4ADgFuB\nqyJi3gT93w58qu6/N/Bu4Bjgz4oULEmSpqQrAgiwCLggM5dl5p3AicDjVMFiPAuAf8vMv8vM+zJz\nOTAIHFimXEmSNBUdDyARMQvoB64ZbsvMBJZTBY3xXA/0D++miYgXAkcA/9zeaiVJUivM7HQBwDxg\nK2DVmPZVwF7jLZCZg/XumX+LiKiX/3xmntPWSiVJUkt0fAZkIwLIcV+IOAQ4nWpXzQHAm4HfiYiP\nFatOkiQ11g0zIA8DTwG7jGnfmafPigw7G1iWmRfWz2+PiO2BC4BPbmxjixYtYu7cuRu0DQwMMDAw\nMNm6JUnqOYODgwwODm7QNjQ01PLtdDyAZOa6iFgBLASuAKh3qywEzp9gsdnA+jFt6+tFoz6GZFxL\nliyhr69v6oVLktSDxvujfOXKlfT397d0Ox0PILXFwJfqIPI9qrNiZgMXAUTEMuD+zDy97n8lsCgi\nbgFuAvakmhX5x42FD0mS1B26IoBk5mX1QaVnU+2KuQU4LDNX1112A54ctcgnqGY8PgE8D1hNNXvi\nMSCSJE0DXRFAADJzKbB0gtcOHfN8OHx8okBpkiSpxbr5LBhJktSjDCCSJKk4A4gkSSrOACJJkooz\ngEiSpOIMIJIkqTgDiCRJKs4AIkmSijOASJKk4gwgkiSpOAOIJEkqzgAiSZKKM4BIkqTiDCCSJKk4\nA4gkSSrOACJJkoozgEiSpOIMIJIkqTgDiCRJKs4AIkmSijOASJKk4gwgkiSpOAOIJEkqzgAiSZKK\nM4BIkqTiDCCSJKk4A4gkSSrOACJJkoozgEiSpOIMIJIkqTgDiCRJKs4AIkmSijOASJKk4gwgkiSp\nOAOIJEkqzgAiSZKKM4BIkqTiDCCSJKk4A4gkSSrOACJJkoozgEiSpOIMIJIkqTgDiCRJKs4AIkmS\nijOASJKk4gwgkiSpOAOIJEkqzgAiSZKKM4BIkqTiDCCSJKk4A4gkSSquawJIRJwcEfdExJqIuDEi\nXr6J/nMj4q8j4sF6mTsj4vWl6pUkSc3N7HQBABFxDHAucALwPWARcFVEvDgzHx6n/yxgOfAQ8Gbg\nQWB34BfFipYkSY11RQChChwXZOYygIg4ETgSeDfw6XH6Hwc8E5ifmU/VbfeVKFSSJE1dx3fB1LMZ\n/cA1w22ZmVQzHAsmWOwo4AZgaUQ8FBH/ERGnRUTH348kSdq0bpgBmQdsBawa074K2GuCZV4IHApc\nDBwO7AksrdfzyfaUKUmSWqUbAshEAsgJXptBFVBOqGdLbo6I5wGnsIkAsmjRIubOnbtB28DAAAMD\nA1OvWJKkaW5wcJDBwcEN2oaGhlq+nW4IIA8DTwG7jGnfmafPigz7GbC2Dh/D7gB2jYiZmfnkRBtb\nsmQJfX19U6lXkqSeNd4f5StXrqS/v7+l2+n4MROZuQ5YASwcbouIqJ9fP8Fi3wVeNKZtL+BnGwsf\nkiSpO3Q8gNQWAydExO9HxN7A54HZwEUAEbEsIv58VP/PATtGxHkRsWdEHAmcBny2cN2SJKmBRrtg\nImIr4A+oZil2ZkyQycxDJ7O+zLwsIuYBZ1PtirkFOCwzV9dddgOeHNX//oh4HbAEuBV4oP5+vFN2\nJUlSl2l6DMh5VAHkn4HbmPhg0c2WmUupzmQZ77WnBZrMvAl4xVS3K0mSymsaQH4PODozv97KYiRJ\n0pah6TEga4G7W1mIJEnacjQNIOcC76/PVpEkSZqUprtgDgJ+Gzg8Im4H1o1+MTPfPNXCJElS72oa\nQH4B/EMrC5EkSVuORgEkM49tdSGSJGnLMaVLsUfETlRXIE3gx6Ou2yFJkjShRgehRsSciPgi1T1Z\nrgO+AzwYEX8bEbNbWaAkSeo9Tc+CWQwcDBwFPLN+vLFuO7c1pUmSpF7VdBfMW4C3Zua/jmr7ekSs\nAS4DTppqYZIkqXc1nQGZDawap/3n9WuSJEkTahpAbgDOiohthxsiYjvgjPo1SZKkCTXdBfN+4JvA\n/RFxK9VZMPsDTwCHtag2SZLUo5peB+S2iNgTeCewNxDAV4BLMnNNC+uTJEk9qPF1QOqg8TctrEWS\nJG0hNjuARMQbgG9k5rr6+wll5hVTrkySJPWsycyAXA7sSnWmy+Ub6ZfAVlMpSpIk9bbNDiCZOWO8\n7yVJkiarZUEiIp7ZqnVJkqTe1vReMB+OiGNGPf974JGIeCAi9mtZdZIkqSc1nQH5I+CnABHxWuA1\nwOuBbwCfaU1pkiSpVzU9Dfc51AEE+B3gssy8OiLuBW5qRWGSJKl3NZ0BeRT4jfr71wPL6+8Dz4CR\nJEmb0HQG5GvApRFxF7Aj1a4XqC7HfncrCpMkSb2raQBZBNxLNQtyamb+qm5/DrC0BXVJkqQe1vRe\nMOuAvxyn/f9MuSJJktTzvBS7JEkqzkuxS5Kk4rwUuyRJKs4gIUmSimt6KfbzI+J947S/JyI8EFWS\nJG1U0xmQtwDfHaf9euCtzcuRJElbgqYBZEdgaJz2x4B5zcuRJElbgqYB5G6qS7CPdTjwk+blSJKk\nLUHTK6EuBj4bETsB36rbFgIfBD7QisIkSVLvanol1C9GxDbAR4E/rZvvBU7KzGUtqk2SJPWopjMg\nZObngM/VsyBrRt0PRpIkaaMaXwckImZGxGuANwNRtz03IrZvVXGSJKk3NZoBiYjdgW8Czwe2Af4F\n+CXw4fr5ia0qUJIk9Z6mMyDnAd8HngWsGdX+D1QHo0qSJE2o6TEgBwGvzMy1ETG6/V7geVMtSpIk\n9bamMyBbMf4db3ej2hUjSZI0oaYB5Go2vN5H1gefngV8fcpVSZKkntZ0F8wHgasi4ofAtsClwJ7A\nw8BAi2qTJEk9qumFyO6PiP2AY4D9gO2BvwUuycw1G11YkiRt8SYdQCJiFnAB8InMvAS4pOVVSZKk\nnjbpY0Aycx3VxcckSZIaaXoQ6j8Cv9vKQiRJ0paj6UGodwEfj4hXAiuAX49+MTPPn2phkiSpdzUN\nIMcBvwD668doCRhAJEnShJqeBbPH8PdRXwo1M7NVRUmSpN42lbvhHhcRtwFPAE9ExG0RcfwU1ndy\nRNwTEWsi4saIePlmLvd7EbE+Ir7WdNuSJKmsRgEkIs6muiHdlcDb6seVwJL6tcmu7xjgXOAM4ADg\nVqoLnc3bxHK7A58BrpvsNiVJUuc0nQE5CfjDzDwtM6+oH6cBJwB/3GB9i4ALMnNZZt4JnAg8Drx7\nogUiYgZwMfBx4J4G25QkSR3SNIDMAr4/TvsKJnlcSX1hs37gmuG2+niS5cCCjSx6BvDzzLxwMtuT\nJEmd1zSAfJlqFmSsE5j8lVHnUd1Zd9WY9lXAruMtUJ/+eyzQ+JgTSZLUOU1PwwU4LiJeB9xYP58P\n/AawLCIWD3fKzD9puP6gOqV3w8bqrrtfptoF9GjDdUuSpA5qGkBeCqysv//N+uvq+vHSUf0259Tc\nh4GngF3GtO/M02dFhre3O3Dl8CnA1DM5EbEW2CszJzwmZNGiRcydO3eDtoGBAQYGvImvJEmDg4MM\nDg5u0DY0NNTy7UQ3XL4jIm4EbsrM99fPA7gPOD8zPzOm79bAi8as4s+o7sj7PuCuzHxynG30AStW\nrFhBX19fG95Fd+nrg/nzYenSTlciSZruVq5cSX9/P0B/Zq7cVP/NMZVdMK20GPhSRKwAvkd1Vsxs\n4CKAiFgG3J+Zp2fmWuCHoxeOiF9QHbt6R9GqJUlSI10RQDLzsvqaH2dT7Yq5BTgsM1fXXXYDnjar\nIUmSpqeuCCAAmbkUGHeHQWYeuollj21LUZIkqS0aX4pdkiSpKQOIJEkqzgAiSZKKM4BIkqTiDCCS\nJKk4A4gkSSrOACJJkoozgEiSpOIMIJIkqTgDiCRJKs4AIkmSijOASJKk4gwgkiSpOAOIJEkqzgAi\nSZKKM4BIkqTiDCCSJKk4A4gkSSrOACJJkoozgEiSpOIMIJIkqTgDiCRJKs4AIkmSijOASJKk4gwg\nkiSpOAOIJEkqzgAiSZKKM4BIkqTiDCCSJKk4A4gkSSrOACJJkoozgEiSpOIMIJIkqTgDiCRJKs4A\nIkmSijOASJKk4gwgkiSpOAOIJEkqzgAiSZKKM4BIkqTiDCCSJKk4A4gkSSrOACJJkoozgEiSpOIM\nIJIkqTgDiCRJKs4AIkmSijOASJKk4gwgkiSpOAOIJEkqzgAiSZKK65oAEhEnR8Q9EbEmIm6MiJdv\npO/xEXFdRDxSP/5lY/0lSVJ36YoAEhHHAOcCZwAHALcCV0XEvAkWORi4FDgEmA/8FLg6Ip7T/mol\nSdJUdUUAARYBF2Tmssy8EzgReBx493idM/N/Z+bnM/MHmflj4Hiq97KwWMWSJKmxjgeQiJgF9APX\nDLdlZgLLgQWbuZo5wCzgkZYXKEmSWq7jAQSYB2wFrBrTvgrYdTPXcQ7wAFVokSRJXW5mpwvYiABy\nk50iPgIcDRycmWvbXpUkSZqybgggDwNPAbuMad+Zp8+KbCAiTgFOBRZm5u2bs7FFixYxd+7cDdoG\nBgYYGBjY7IIlSepVg4ODDA4ObtA2NDTU8u10PIBk5rqIWEF1AOkVABER9fPzJ1ouIj4EnA68LjNv\n3tztLVmyhL6+vqkVLUlSjxrvj/KVK1fS39/f0u10PIDUFgNfqoPI96jOipkNXAQQEcuA+zPz9Pr5\nqcDZwABwX0QMz578KjN/Xbh2SZI0SV0RQDLzsvqaH2dT7Yq5BTgsM1fXXXYDnhy1yElUZ718dcyq\nzqrXIUmSulhXBBCAzFwKLJ3gtUPHPN+jSFGSJKktuuE0XEmStIUxgEiSpOIMIJIkqTgDiCRJKs4A\nIkmSijOASJKk4gwgkiSpOAOIJEkqzgAiSZKKM4BIkqTiDCCSJKk4A4gkSSrOACJJkoozgEiSpOIM\nIJIkqTgDiCRJKs4AIkmSipvZ6QLUHjNmwBVXwHbbwcKF8OpXw/bbd7oqSZIqzoD0qL/8yyp4/N3f\nwZFHwrOeBQcdBGecAdddB2vXdrpCSdKWzBmQHnXIIdUjE+66C665pnp89rNw9tkweza86lVVSFm4\nEPbfv5o1kSSpBH/l9LgIePGL4aST4KtfhdWrYcUKOPPM6vUzz4T+fthpJ3jLW2DpUvjRj6rgIklS\nuzgDsoWZMQP6+qrHhz5U7Yq58caRGZL3vx+efBKe97yR2ZGFC6vnkiS1igFkC7f11tUBqq9+NZx1\nFvzyl/Cd74wEkmXLqn577TUSRg45BJ797I6WLUma5gwg2sAzngFHHFE9AB5+GK69tgojV19d7aKJ\nqGZQDj20CiQHHQRz5nS2bknS9GIA0UbNmwdve1v1ALjvvpHZkS9/GT7zGZg1CxYsGJkhOfDAqk2S\npIl4EKom5fnPh2OPhYsvhgcfhNtvh3PPrU7zXby4mg159rOrU38XL4Zbb4X16ztdtSSp2xhA1FgE\n7LMPvPe9cPnl1e6am26C00+vDm796Eer03t32QWOPhouuADuvtszbCRJ7oJRC82cWe1+OfBAOO00\neOIJuOGGkV02J58MTz1VzaIM76459FB4znM6XbkkqTQDiNpm223ht3+7enzyk/DYY/Dtb48Ekgsv\nrPrts89IIDn4YHjmMztbtySp/QwgKmaHHeCoo6oHwKpV8K1vVWHkyivhr/6quk5Jf/9IIHnlK6v7\n2UiSeosBRB2zyy4wMFA9AO65Z2R25ItfhL/4C9hmG3jFK0YCycteVu3qkSRNbx6Eqq6xxx5w/PEw\nOAgPPQQ/+EEVQubMgXPOqU713XFHeMMb4Lzz4LbbPKBVkqYr/5ZUV4qAffetHh/4AKxbB9///sgM\nyamnVmfa7LzzyAXRDj/cS8ZL0nRhANG0MHyxswUL4GMfg8cfh+uvHwkkl11WXX9k9epOVypJ2hwG\nEE1Ls2fDa15TPaA6gPWUUzpbkyRp83kMiHrCVlt1ugJJ0mQYQCRJUnEGEEmSVJwBRJIkFWcAkSRJ\nxRlAJElScQYQSZJUnAFEkiQVZwCRJEnFGUAkSVJxBhBJklScAUSSJBVnAJEkScUZQCRJUnEGEEmS\nVJwBRJIkFdc1ASQiTo6IeyJiTUTcGBEv30T/t0XEHXX/WyPi8FK1TneDg4OdLqErOA4jHIuK4zDC\nsag4Du3TFQEkIo4BzgXOAA4AbgWuioh5E/RfAFwK/A2wP3A5cHlE7FOm4unN/1AVx2GEY1FxHEY4\nFhXHoX1mdrqA2iLggsxcBhARJwJHAu8GPj1O//cD38jMxfXzMyLidcB7gD8uUK+6UCasX9++/htb\nT6dFjP9VkrpVxwNIRMwC+oE/H27LzIyI5cCCCRZbQDVjMtpVwBvbUqS63syZsG4dbLXV5JabbP/p\naqKAMvz1qadg66033a/dXzf22rp1cNxxcM45m36/krpfxwMIMA/YClg1pn0VsNcEy+w6Qf9dW1ua\npotjjoFttql+kW6u88+H972vNdvv5IzD8AzMRF83p8/nPgcnnrjpflPZxlS/fvzj8OlPw0UX0VKj\n/+0efRR2neJPkVZ9FtauhaEhOOAA2G671qwTNr++226Dgw9u3fo2x/r1sOOOMH9+69Y5VXff3d7Q\nu912rfs5NN10QwCZSACTmdzeVP9tAe64446p1NQThoaGWLlyZafLaLl9951c/7lzh9h//94bhya+\n8pUh5s/v7rF48Yvhuuvau42vf32II45oPg6t3B33q1/BfffB9tuXryUTZs4cYs6csp+Jq6+uZuKu\nvbboZjfq178e4lOfat847LADHHRQ21bfMqN+d27bqnVGdngHdr0L5nHgLZl5xaj2i4C5mfmmcZb5\nT+DczDx/VNuZwBsz84AJtvN24JLWVi9J0hblHZl5aStW1PEZkMxcFxErgIXAFQAREfXz8ydY7IZx\nXn9t3T6Rq4B3APcCT0ytakmStijbAi+g+l3aEh2fAQGIiKOBLwF/BHyP6qyYtwJ7Z+bqiFgG3J+Z\np9f9FwDfBj4C/DMwUH/fl5k/7MBbkCRJk9DxGRCAzLysvubH2cAuwC3AYZm5uu6yG/DkqP43RMQA\n8Gf14y6q3S+GD0mSpoGumAGRJElblq64EqokSdqyGEAkSVJxPRNAvJndiMmMRUTsExFfrfuvj4ie\nuSTOJMfh+Ii4LiIeqR//sqnP0HQyybF4U0T8e0Q8GhG/ioibI+KdJettl8n+nBi13O/V/z++1u4a\nS5nkZ+Jd9ft/qv66PiIeL1lvuzT43TE3Iv46Ih6sl7kzIl5fqt52meTn4dpRn4PRjysns82eCCDe\nzG7EZMcCmA38P+DDwM+KFFlAg3E4mOozcQgwH/gpcHVEPKf91bZXg7H4L+CTVOOwL3AhcGFEvLZA\nuW3TYByGl9sd+AzQ5sugldNwLIaorjY9/Ni93XW2W4PfHbOA5cDzgTdTXa37D4EHihTcJg0+D29i\nw8/CS4GngMsmteHMnPYP4EbgvFHPA7gfOHWC/l8BrhjTdgOwtNPvpfRYjFn2HuB9nX4PnR6Huv8M\nqh+47+z0e+n0WNTLrADO6vR7KT0O9efgO8CxVEHsa51+H50YC+BdwCOdrrsLxuFEqrMut+p07Z0c\nh3GW/wDwC2C7yWx32s+AjLqZ3TXDbVmNyKZuZrd8TNtVG+k/LTQci57TonGYA8wCHml5gQW1Yiwi\nYiHwYqpr70xLUxiHM4CfZ+aF7a2wnCmMxfYRcW9E3BcR037GuOE4HEX9x2pEPBQR/xERp0XEtP1d\n2qKfl+8GBjNzzWS2PW0HbZSN3cxuottK9erN7JqMRS9qxTicQzWtOjaoTjeNxiIidoiIX0bEWuBK\n4L2Z+a32ldl2kx6HiHgl1czH8e0trbgmn4kfUf2SeQPVFaVnANdHxPPaVWQBTcbhhcDbqN7/4cAn\ngA8Cp7epxhKm9PMyIg4E/hfwhcluuCsuRNYmrb6Z3XTWy+9tMjZrHCLiI8DRwMGZubbtVXXGpsbi\nl8B+wPZUtz1YEhE/ycyeOQ6iNu44RMT2wJeBP8zMR4tX1RkTfiYy80aqafqqY8QNwB3ACVSzRL1k\nY/83ZlD9Yj6hniW4uQ5hp1AdN9VLNvf3xnHAbZm5YrIb6IUA8jDVwS+7jGnfmacnumEPTbL/dNFk\nLHpR43GIiFOAU4GFmXl7e8orqtFY1D9cf1I//UE93X4a0/dAzMmOw29SHWR5ZcT/3HB+BkA9K7RX\nZt7Tplrbbco/JzLzyYi4GXhRi2srqck4/AxYW///GHYHsGtEzMzMJydYrptN5efldsAxwMeabHja\n74LJzHVUB8gtHG6rf2AsBK6fYLEbRvevbepmdl2v4Vj0nKbjEBEfAj5KdRuAm9tdZwkt/EzMALZp\nbXXlNBiHO6jOANqfaiZoP6qbZX6r/v6nbS65bVrxmaiPeXgp0/jMuYbj8F2eHrr2An42TcPHVD8P\nxwBb0/RO850++rZFR/AeDawBfh/YG7iA6lTCnerXlwF/Pqr/AmAt8CdUH54zqe6Qu0+n30sHxmIW\n1Q/U/amOeTinfv6bnX4vhcfh1Poz8CaqvwSGH3M6/V46MBYfAV4D7FH3/yDw38CxnX4vJcdhnOV7\n6SyYyX4m/pTqj7Q9qE7THAR+TXXD0I6/n4LjsBvV2XHnAXsCR1LNqH+k0++l5DiMWu47wKVNt9sL\nu2BIb2b3PyY7FsBzgZsZ2dd3Sv34NnBokaLboME4nEQVxr46ZlVn1euYthqMxRzgr+v2NcCdwDsy\nc+zYTCsNxqFnNRiLZwH/l+qgxEep/mJekJl3lqu69Rr87rg/Il4HLKG6VsYD9fefLlp4izX5vxER\newKvoAqmjXgzOkmSVNy0PwZEkiRNPwYQSZJUnAFEkiQVZwCRJEnFGUAkSVJxBhBJklScAUSSJBVn\nAJEkScUZQCRJUnEGEEnTTkQcHBHrI2KH+vm7IuLRTtclafMZQCRNV2PvI+F9JaRpxAAiqaUiYlan\na5DU/QwgkqYkIq6NiL+KiCURsRr4ZkTMjYgvRMTPI2IoIpZHxG+NWe6oiPheRKyJiNUR8dVRr70j\nIv49Ih6LiJ9FxCURsVPxNyepbQwgklrh94H/pro994nA3wM7AocBfcBKYHlEPBMgIo4Evgb8E7A/\ncCjw/VHrmwV8DPgt4I3A7sCFJd6IpDIi092mkpqLiGuBHTKzv37+SqpgsXNmrhvV7y7gnMz8QkR8\nF7g7M9+1mdt4GXAT8IzMfDwiDga+BTwrMx+LiHcBSzLz2a19d5LaxRkQSa0wevZiP+AZwCMR8cvh\nB/AC4IV1n/2pAsS4IqI/Iq6IiP+MiMeAf61fen7LK5fUETM7XYCknvDrUd9vDzwIHAzEmH6/qL+u\nmWhFETEb+CbwDeDtwGqqXTDfBLZuUb2SOswAIqnVVgK7Ak9l5n0T9PkBsBD40jiv7Q08GzgtMx8A\niIgD21GopM5xF4yklsrM5cANwOUR8dqI2D0iXhERn4yIvrrbWcBARJwZEXtHxL4R8aH6tfuAtcD7\nImKPiHgD1QGpY42dXZE0jRhAJE3VeEeyHwFcB3wR+BFwKdXxG6sAMvPbwNuAo4CbgeXAgfVrDwN/\nALwVuB0WeiQHAAAAVklEQVQ4FfjgZm5X0jThWTCSJKk4Z0AkSVJxBhBJklScAUSSJBVnAJEkScUZ\nQCRJUnEGEEmSVJwBRJIkFWcAkSRJxRlAJElScQYQSZJUnAFEkiQV9/8B64sSlh0RQhIAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe4891eba10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(recall, prec)\n",
    "plt.xlabel('recall')\n",
    "plt.ylabel('precision')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "r_pre = precision / (np.ones(20)*7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.14285714,  0.03571429,  0.03174603,  0.01785714,  0.01142857,\n",
       "        0.00793651,  0.0058309 ,  0.00669643,  0.00705467,  0.00571429,\n",
       "        0.00472255,  0.00396825,  0.00422654,  0.00364431,  0.00380952,\n",
       "        0.00334821,  0.00296589,  0.0026455 ,  0.00237436,  0.0025    ])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_pre"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## R-precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAF5CAYAAABEPIrHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XuYXWV59/HvnWQ4RCAcQhLITFGKHEUgKZS0CgpVCmpR\nK+IoryhVRFExaotaLRWrFRQiiGkpfRVSZXyxKGABY6EWKXIQwkHloFDknJBwCMdADvf7x9pDdiYz\nmZk9e2at2fv7ua517dlrPXvte7OvYX551vM8KzITSZKkqphQdgGSJEn1DCeSJKlSDCeSJKlSDCeS\nJKlSDCeSJKlSDCeSJKlSDCeSJKlSDCeSJKlSDCeSJKlSDCeSJKlSKhNOIuL4iLg3Ip6PiOsiYt8N\ntN09Iv691n5NRHx8gHbbR8S/RcSyiHguIm6NiFmj9ykkSdJIVSKcRMSRwGnAScA+wK3AwoiYOsBL\nJgP3ACcCjwxwzi2Ba4AXgEOA3YBPAU80tXhJktRUUYUb/0XEdcD1mXlC7XkADwBnZuapg7z2XmBe\nZp7ZZ/9XgTmZeeAolS1JkkZB6T0nEdEBzAau7N2XRWK6ApgzglO/BbgxIi6IiCURsSgiPjCyaiVJ\n0mgrPZwAU4GJwJI++5cAM0Zw3h2BDwN3AW8E/hk4MyKOGsE5JUnSKJtUdgEbEMBIrjlNAG7IzC/U\nnt8aEXtQBJbvrvdmEdtQjE35PbBiBO8rSVK72QR4ObAwMx8b6cmqEE6WAauB6X32T2P93pTheAS4\no8++O4C3D9D+EOB7I3g/SZLa3XuA80d6ktLDSWaujIibgIOBS+ClAbEHA2du6LWDuAbYpc++XYD7\nBmj/e4Dvfve77LbbbiN4W1XJ3LlzmTdvXtllqEn8PluL32fruOOOOzjqqKOg9rd0pEoPJzWnA+fV\nQsoNwFyK6cLnAkTEAuDBzPxc7XkHsDvFpZ+NgJkRsRfwTGbeUzvnPOCaiPgscAHwx8AHgA8OUMMK\ngN12241Zs1wKpVVMmTLF77OF+H22Fr/PltSUYRGVCCeZeUFtTZOTKS7v3AIckplLa006gVV1L9ke\nuJm1Y1I+XduuAg6qnfPGiHgb8FXgC8C9wAmZ+f1R/jiSJGkEKhFOADJzPjB/gGMH9Xl+H0OYaZSZ\nlwGXNaVASZI0JqowlViSJOklhhO1tO7u7rJLUBP5fbYWv08NxHCilub//FqL32dr8fvUQAwnkiSp\nUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwn\nkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSpUgwnkiSp\nUgwnkiSpUgwnkiSpUgwnkiSpUgwnYyQTVq8uuwpJkqrPcDIGHn0UNtkELrus7EokSaq+yoSTiDg+\nIu6NiOcj4rqI2HcDbXePiH+vtV8TER8f5NyfrbU7vfmVD26bbWDNGnjwwTLeXZKk8aUS4SQijgRO\nA04C9gFuBRZGxNQBXjIZuAc4EXhkkHPvC3ywds5STJwIM2fCAw+UVYEkSeNHJcIJMBc4OzMXZOad\nwHHAc8Ax/TXOzBsz88TMvAB4caCTRsRmwHeBDwBPNr/soevqMpxIkjQUpYeTiOgAZgNX9u7LzASu\nAOaM8PTfAn6cmf81wvOMWGenl3UkSRqKSWUXAEwFJgJL+uxfAuzS6Ekj4l3A3sAfNV5a83R1wS9/\nWXYVkiRVXxXCyUACyIZeGNEJfAN4Q2auHM5r586dy5QpU9bZ193dTXd3dyOlvKSrq+g5yYSIEZ1K\nkqTS9PT00NPTs86+5cuXN/U9qhBOlgGrgel99k9j/d6UoZoNbAvcFPFSFJgIHBARHwU2rl06Ws+8\nefOYNWtWg287sM5OeOEFWLYMtt226aeXJGlM9PcP9kWLFjF79uymvUfpY05qPRs3AQf37qsFioOB\nXzR42iuAPSku6+xV226kGBy710DBZDR1dRWPDoqVJGnDqtBzAnA6cF5E3ATcQDF7ZzJwLkBELAAe\nzMzP1Z53ALtTXPrZCJgZEXsBz2TmPZn5LHB7/RtExLPAY5l5x9h8pHXVh5NR6JiRJKllVCKcZOYF\ntTVNTqa4vHMLcEhmLq016QRW1b1ke+Bm1o5J+XRtuwo4aKC3aXbdw7HtttDR4YwdSZIGU4lwApCZ\n84H5Axw7qM/z+xjmJam+5xhrEyYU4068rCNJ0oaVPuaknbgQmyRJgzOcjCEXYpMkaXCGkzFkz4kk\nSYMznIyh3oXY1qwpuxJJkqrLcDKGOjth5UpYunTwtpIktSvDyRhyITZJkgZnOBlDhhNJkgZnOBlD\nU6fCxhs7Y0eSpA0xnIyhCBdikyRpMIaTMeZ0YkmSNsxwMsbsOZEkacMMJ2Osd60TSZLUP8PJGOvq\ngocegtWry65EkqRqMpyMsc5OWLUKliwpuxJJkqrJcDLGetc68dKOJEn9M5yMMRdikyRpwwwnY2zr\nrWGTTQwnkiQNxHAyxiKcsSNJ0oYYTkrgQmySJA3McFICF2KTJGlghpMSeFlHkqSBGU5K0NUFDz/s\nQmySJPXHcFKCzs4imDzySNmVSJJUPYaTErgQmyRJAzOclMCF2CRJGpjhpARbbgmTJxtOJEnqj+Gk\nBC7EJknSwAwnJXEhNkmS+leZcBIRx0fEvRHxfERcFxH7bqDt7hHx77X2ayLi4/20+WxE3BART0XE\nkoj4UUTsPLqfYuhciE2SpP5VIpxExJHAacBJwD7ArcDCiJg6wEsmA/cAJwIDTch9LfBN4I+BPwM6\ngJ9GxKZNLL1hXtaRJKl/k8ouoGYucHZmLgCIiOOANwHHAKf2bZyZNwI31tqe0t8JM/Ow+ucR8T7g\nUWA28D9NrL0hXV3FOierVsGkqnwLkiRVQOk9JxHRQREYruzdl5kJXAHMaeJbbQkk8HgTz9mwzk5Y\ns6ZYKVaSJK1VejgBpgITgSV99i8BZjTjDSIigG8A/5OZtzfjnCPlQmySJPWvyhcUgqKnoxnmA7sD\nfzpYw7lz5zJlypR19nV3d9Pd3d2kUgouxCZJGo96enro6elZZ9/y5cub+h5VCCfLgNXA9D77p7F+\nb8qwRcRZwGHAazNz0LvZzJs3j1mzZo30bQe1xRaw2WaGE0nS+NLfP9gXLVrE7Nmzm/YepV/WycyV\nwE3Awb37apdhDgZ+MZJz14LJ4cDrM/P+kZyr2VyITZKk/lWh5wTgdOC8iLgJuIFi9s5k4FyAiFgA\nPJiZn6s976C4TBPARsDMiNgLeCYz76m1mQ90A38BPBsRvT0zyzNzxVh9sA1xITZJktZXiXCSmRfU\n1jQ5meLyzi3AIZm5tNakE1hV95LtgZtZOybl07XtKuCg2r7jasf/u8/bvR9Y0OSP0JDOTvjVr8qu\nQpKkaqlEOAHIzPkUA1f7O3ZQn+f3Mcglqcws/ZLVYLq64PLLy65CkqRqqfwf8FbW1QWLF8OLL5Zd\niSRJ1WE4KVFnJ2S6EJskSfUMJyVyITZJktZnOCmRC7FJkrQ+w0mJNt+8WIzNcCJJ0lqGk5K5EJsk\nSesynJTMhdgkSVqX4aRknZ2GE0mS6hlOSuZlHUmS1mU4KVlXFyxZAi+8UHYlkiRVg+GkZJ2dxeND\nD5VbhyRJVWE4KZkLsUmStC7DScl6e04cFCtJUsFwUrLNNoMttzScSJLUy3BSAc7YkSRpLcNJBbgQ\nmyRJaxlOKsCF2CRJWstwUgFe1pEkaS3DSQV0dcHSpbBiRdmVSJJUPsNJBfROJ7b3RJIkw0kluBCb\nJElrGU4qwIXYJElay3BSAZMnw9ZbG04kSQLDSWU4Y0eSpILhpCJciE2SpILhpCJciE2SpILhpCK8\nrCNJUqEy4SQijo+IeyPi+Yi4LiL23UDb3SPi32vt10TEx0d6zrJ1dcFjj8Fzz5VdiSRJ5apEOImI\nI4HTgJOAfYBbgYURMXWAl0wG7gFOBB5p0jlL5UJskiQVKhFOgLnA2Zm5IDPvBI4DngOO6a9xZt6Y\nmSdm5gXAi804Z9lciE2SpELp4SQiOoDZwJW9+zIzgSuAOVU552hzITZJkgqlhxNgKjARWNJn/xJg\nRoXOOao22QSmTjWcSJJUhXAykAByHJyzaZyxI0kSTCq7AGAZsBqY3mf/NNbv+Rj1c86dO5cpU6as\ns6+7u5vu7u4GSxk6F2KTJFVdT08PPT096+xbvnx5U9+j9HCSmSsj4ibgYOASgIiI2vMzx/qc8+bN\nY9asWY287Yh1dsLVV5fy1pIkDUl//2BftGgRs2fPbtp7lB5Oak4HzqsFihsoZtpMBs4FiIgFwIOZ\n+bna8w5gd4rLNBsBMyNiL+CZzLxnKOesIi/rSJJUkXCSmRfU1h85meJSzC3AIZm5tNakE1hV95Lt\ngZtZO37k07XtKuCgIZ6zcrq64Ikn4Nln4WUvK7saSZLKUYlwApCZ84H5Axw7qM/z+xjCYN4NnbOK\n6qcT77prubVIklSWKs/WaTsuxCZJ0gh6TiJiS2A/ihkw64SczFwwwrra0syZxaMzdiRJ7ayhcBIR\nbwG+B7wMeJp11w5JwHDSgI03hmnTDCeSpPbW6GWd04BvA5tn5paZuVXdtnUT62s7ztiRJLW7RsPJ\nTODMzHyumcXIhdgkSWo0nCwE/qiZhajQ2Wk4kSS1t0YHxF4KfC0idgd+BaysP5iZl4y0sHblZR1J\nUrtrNJycU3v8u36OJcUdgdWAri5Yvhyefho237zsaiRJGnsNXdbJzAkb2AwmI1C/EJskSe3IRdgq\npnchNsOJJKldNRxOIuLAiPhxRNwdEb+LiEsi4rXNLK4dbb89RDjuRJLUvhoKJxFxFHAF8BxwJnAW\n8DxwZUS8u3nltZ+NNoLp0+05kSS1r0YHxP4t8DeZOa9u3xkR8UngC8D5I66sjbnWiSSpnTV6WWdH\n4Mf97L8EeEXj5QicTixJam+NhpMHgIP72X9w7ZhGwIXYJEntrNHLOqcBZ0bE3sAvKNY2eQ3wPuCE\n5pTWvnov62QWg2MlSWonDYWTzPyniFgMfAp4Z233HcCRmXlxs4prV11d8Mwz8NRTMGVK2dVIkjS2\nGu05ITN/BPyoibWopn4hNsOJJKnduAhbBbkQmySpnQ255yQiHgd2zsxlEfEExTiTfmXm1s0orl1t\ntx1MmOCMHUlSexrOZZ25wNN1Pw8YTjQyHR0wY4Y9J5Kk9jTkcJKZ59X9fO6oVKOXuBCbJKldNbp8\n/ayI2LPu+eERcVFEfCUiNmpeee3LhdgkSe2q0QGxZwM7A0TEjsD/o7jPzhHAqc0prb25EJskqV01\nGk52Bm6p/XwEcFVmvptiEba/bEJdba9+ITZJktpJo+Ek6l77Z8BltZ8fAKaOtCgV4eS55+DJJ8uu\nRJKksdVoOLkR+HxE/B/gQODS2v5XAEuaUVi7q1+ITZKkdtJoOPkEMAs4C/hyZt5d2/8OinvtaIRc\niE2S1K4aCieZeVtm7pmZUzLzi3WH/ho4upFzRsTxEXFvRDwfEddFxL6DtD8iIu6otb81Ig7tc/xl\nEXFWRDwQEc9FxG8i4kON1FaG7baDiROdsSNJaj9NXb4+M1dk5srhvi4ijqS40/FJwD7ArcDCiOh3\n/EpEzAHOB84B9gYuAi6KiN3rms0D3gi8G9gV+AZwVkS8ebj1lWHixCKg2HMiSWo3Qw4nEfF4b1iI\niCdqz/vdGqhjLnB2Zi7IzDuB4yimJh8zQPsTgMsz8/TMvCszTwIWAR+tazMHOC8zr87M+zPzHIrQ\ns18D9ZXChdgkSe2o9OXrI6IDmA18pXdfZmZEXEERMPozh6Knpd5C4PC6578A/iIivpOZD0fE64FX\n1tqNCy7EJklqR1VYvn4qMJH1Z/ksAXYZ4DUzBmg/o+75x4B/AR6MiFXAauCDmXnNiCseI52dcPPN\nZVchSdLYGk7PyUsi4jBgdWYu7LP/jcDEzLy8CbUFw+ud6dv+48AfA28G7gcOAOZHxMOZ+V9NqG/U\n1S/EFlF2NZIkjY2GwgnwVeAz/eyfUDs2nHCyjKJXY3qf/dMYeM2UxRtqHxGbAF8GDs/Mn9SO/zoi\n9gE+DQwYTubOncuUKVPW2dfd3U13d/fgn6TJurpgxQp4/HHYZpsxf3tJktbT09NDT0/POvuWL1/e\n1PdoNJy8Eri9n/13AjsN50SZuTIibgIOBi4BiIioPT9zgJdd28/xN9T2A3TUtr49L6sZZBDwvHnz\nmDVr1nA+wqipX4jNcCJJqoL+/sG+aNEiZs+e3bT3aHQq8XJgx3727wQ828D5TgeOjYj3RsSuwD8D\nk4FzASJiQUR8pa79GcChEfHJiNglIv6eYlDtWQCZ+TRwFfC1iDgwIl4eEe8D3gv8sIH6SuFCbJKk\ndtRoz8nFwDci4m2ZeQ9AROxEMYPmkuGeLDMvqE1TPpnics0twCGZubTWpBNYVdf+2ojoprh082Xg\ndxSXcOp7c44E/hH4LrA1cB/w2cz8l+HWV5bp02HSJGfsSJLaS6Ph5G+AnwB3RkTvn85O4GqKMR3D\nlpnzgfkDHDuon30XAhdu4HyPAn/VSC1VMXEibL+9PSeSpPbSUDjJzOUR8ScU4zz2Ap4HbsvMnzez\nOLkQmySp/TTac0JmJvDTiPg58ELtuZrMhdgkSe2moQGxETEhIr4QEQ8BzwCvqO3/UkSM60spVdPZ\nac+JJKm9NDpb5/PA+yjGnrxYt//XwAdGWJPq9Pac2C8lSWoXjYaT9wLHZub3KNYO6XUrxR2A1SRd\nXfDCC7BsWdmVSJI0NhoNJzOBuwc4X0fj5aiv+oXYJElqB42Gk9uB1/az/x2At6prIhdikyS1m0Zn\n65wMnBcRMykCztsjYheKyz1vblZxgmnToKPDGTuSpPbRUM9JZl5MEUL+jGK5+pOB3YC3ZOZ/Nq88\nTZgAM2facyJJah/D7jmJiInAn1IsuvaG5pekvlyITZLUTobdc5KZq4GfAls1vxz1x4XYJEntpNEB\nsb+m/7sSaxS4EJskqZ2MZBG2r0fEmyNiu4jYon5rZoFa23OyZk3ZlUiSNPoana1zWe3xEqB+7dKo\nPZ84kqK0rq4uWLkSli6F6dPLrkaSpNHVaDh5fVOr0AbVL8RmOJEktbqGwklmXtXsQjSw+oXY/uiP\nyq1FkqTR1mjPCRGxFfBXFOubJHAH8J3MfLxJtalm221ho42csSNJag8NDYiNiAOA3wMfp5hSvHXt\n53trx9REEc7YkSS1j0Z7Tr4F/D/gw7V1T3oXZ5tfO7Znc8pTLxdikyS1i0anEu8EnNYbTOClxdlO\nrx1Tk7kQmySpXTQaThZRjDXpazfg1sbL0UC8rCNJaheNXtY5EzgjInYCrqvt2x84HvhMRLy6t2Fm\n3jayEgVFz8lDD8Hq1TDRVWQkSS2s0XDSU3s8dYBjiQuyNVVXF6xaBY8+CtttV3Y1kiSNnkbDySua\nWoUGVb8Qm+FEktTKGl2E7b5mF6INq1+Ibb/9yq1FkqTR1OiA2JdExFMR4R2KR9k228AmmzhjR5LU\n+kYcTijGlmiUuRCbJKldNCOcaIy4EJskqR0MO5xEREdEXBkRr6zt+i7w1EgLiYjjI+LeiHg+Iq6L\niH0HaX9ERNxRa39rRBzaT5vdIuLiiHgyIp6JiOsjonOktZbFhdgkSe1g2OEkM1cC9euYfDgzl42k\niIg4EjgNOAnYh2Iht4URMXWA9nOA84FzgL2Bi4CLImL3ujZ/CFwN3A4cQLGk/peAFSOptUxe1pEk\ntYNGL+t8l+KOxM0yFzg7Mxdk5p3AccBzwDEDtD8BuDwzT8/MuzLzJIpVaz9a1+YfgEsz87OZeVtm\n3puZ/zHSIFWmri54+OFiITZJklpVo+ucTAKOiYg3ADcCz9YfzMxPDvVEEdEBzAa+Uvf6jIgrgDkD\nvGwORU9LvYXA4bVzBvAm4NSI+AlFb8y9wD9m5sVDra1qurqKYLJ4McycWXY1kiSNjkZ7Tl5F0VPx\nFLAzxR//3m3vYZ5rKsUqskv67F8CzBjgNTMGaT8N2Aw4EbgMeAPwI+CHEfHaYdZXGfULsUmS1Koa\nXYTt9c0upB+9y9830r43dF2UmWfWfr4tIv6E4pLR1QOdZO7cuUyZMmWdfd3d3XR3dw+jlNFRvxDb\n/vuXW4skqT319PTQ09Ozzr7ly5c39T0avazTTMuA1cD0PvunsX7vSK/Fg7RfBqwC7ujT5g7gTzdU\nzLx585g1a9YgJZdjq61g8mRn7EiSytPfP9gXLVrE7Nmzm/Yepa9zUpv9cxNwcO++2piRg4FfDPCy\na+vb17yhtr/3nL8EdunTZmdg3C6970JskqR2UIWeE4DTgfMi4ibgBorZO5OBcwEiYgHwYGZ+rtb+\nDOCqiPgkcCnQTTGo9oN15/wa8P2IuBr4GXAo8GbgwFH/NKPIhdgkSa2uEuEkMy+orWlyMsXlmluA\nQzJzaa1JJ8Vlmt7210ZEN/Dl2vY74PDMvL2uzUURcRzwOYowcxfw9sy8diw+02jp6oI77yy7CkmS\nRk8lwglAZs4H5g9w7KB+9l0IXDjIOc+l1vvSKjo74T//s+wqJEkaPaWPOdHwdHXBI4/AqlWDt5Uk\naTwynIwzXV2wZk0RUCRJakWGk3HGhdgkSa3OcDLO1C/EJklSKzKcjDNTpsBmm7kQmySpdRlOxhkX\nYpMktTrDyTjkQmySpFZmOBmHDCeSpFZmOBmHOjsdcyJJal2Gk3GoqwsWL4YXXyy7EkmSms9wMg51\ndUEmPPxw2ZVIktR8hpNxqHchNi/tSJJakeFkHHIhNklSKzOcjENbbFFshhNJUisynIxTztiRJLUq\nw8k45VonkqRWZTgZpwwnkqRWZTgZp7ysI0lqVYaTcaqrC5YsgRdeKLsSSZKay3AyTvVOJ37ooXLr\nkCSp2Qwn45QLsUmSWpXhZJxyITZJUqsynIxTm20GW25pOJEktR7DyTjW1eVlHUlS6zGcjGOdnfac\nSJJaj+FkHHMhNklSKzKcjGNe1pEktSLDyTjW2QlLl8KKFWVXIklS81QmnETE8RFxb0Q8HxHXRcS+\ng7Q/IiLuqLW/NSIO3UDbsyNiTUR8vPmVl6d3OrG9J5KkVlKJcBIRRwKnAScB+wC3AgsjYuoA7ecA\n5wPnAHsDFwEXRcTu/bR9K7Af0HJrqRpOJEmtqBLhBJgLnJ2ZCzLzTuA44DngmAHanwBcnpmnZ+Zd\nmXkSsAj4aH2jiJgJnAm8G1g1atWXpHeVWAfFSpJaSenhJCI6gNnAlb37MjOBK4A5A7xsTu14vYX1\n7SMigAXAqZl5RzNrrorJk2HrrQ0nkqTWUno4AaYCE4ElffYvAWYM8JoZQ2j/GeDFzDyrGUVWlTN2\nJEmtZlLZBWxAANlI+4iYDXycYvzKsMydO5cpU6ass6+7u5vu7u7hnmpMuBCbJGks9fT00NPTs86+\n5cuXN/U9qhBOlgGrgel99k9j/d6RXosHaf8aYFvggeLqDlD0zpweEZ/IzB0HKmbevHnMmjVr6NWX\nrKsLrr227CokSe2iv3+wL1q0iNmzZzftPUq/rJOZK4GbgIN799XGixwM/GKAl11b377mDbX9UIw1\neTWwV932MHAqcEizaq8CL+tIklpNFXpOAE4HzouIm4AbKGbvTAbOBYiIBcCDmfm5WvszgKsi4pPA\npUA3xaDaDwJk5hPAE/VvEBErgcWZ+btR/zRjqLMTHnsMnnuuGCArSdJ4V4lwkpkX1NY0OZnics0t\nwCGZubTWpJO6qcCZeW1EdANfrm2/Aw7PzNs39DajUnzJ6tc62XnncmuRJKkZKhFOADJzPjB/gGMH\n9bPvQuDCYZx/wHEm45nhRJLUakofc6KRmTmzeHTGjiSpVRhOxrlNN4WpU+FXv4I1a8quRpKkkTOc\ntIB994XTToMddoBPfAKuucagIkkavwwnLeDHP4arroK3vhUuuABe85piLMrHPw5XXw2rV5ddoSRJ\nQ2c4aQETJ8IBB8A3v1kMjL36anjHO+CHPyz2d3bCRz9aBBiDiiSp6gwnLWbChKLn5Iwz4P77i0s8\n73oXXHwxvO51xQDaj3wEfvYzWNVy92mWJLUCw0kLmzAB/uRPYN48uO++Ypn7o46CSy+Fgw6C7beH\n446DK64wqEiSqsNw0iYmTID994evfx1+/3u4/np43/tg4UJ4wxtgu+3g2GPhpz+FlSvLrlaS1M4M\nJ20oAvbbD049Ff73f+HGG+Gv/gquvBIOOQRmzIAPfAB+8hODiiRp7BlO2lwEzJ4NX/0q3H03LFoE\nH/pQMXj20ENh+nT4/OchW3Lxf0lSFRlO9JII2Gcf+MpX4Le/hVtugWOOgS9/udgnSdJYqMy9dVQt\nEbDXXsW2xRZF78krXgHvfnfZlUmSWp3hRIP6wheKsSnvf3+xuNtrX1t2RZKkVuZlHQ0qAv7lX+BP\n/7RYhfa3vy27IklSKzOcaEg22qhYcXb6dDjsMFi6tOyKJEmtynCiIdtyS7jsMnj66aIHZcWKsiuS\nJLUiw4mG5eUvL240ePPNcPTR3v1YktR8hhMN2377wfe+Bz/4AXzuc2VXI0lqNYYTNeRtbyuWwj/l\nFDjnnLKrkSS1EqcSq2Fz58I998CHPwx/8AfF0veSJI2UPSdqWASccQb8+Z/DEUfAbbeVXZEkqRUY\nTjQikybB978PO+0Eb3oTPPxw2RVJksY7w4lGbLPN4D/+o/j5zW+GZ54ptx5J0vhmOFFTbL89XHpp\ncWfjd70LVq0quyJJ0nhlOFHTvPrVxfTin/wEPvEJyCy7IknSeGQ4UVMdcgjMnw/f+hZ84xtlVyNJ\nGo+cSqymO/bYYorxpz5VrCj7treVXZEkaTypTM9JRBwfEfdGxPMRcV1E7DtI+yMi4o5a+1sj4tC6\nY5Mi4pSIuC0inomIhyLivIjYbvQ/iQD+8R/hHe+A97wHbrih7GokSeNJJcJJRBwJnAacBOwD3Aos\njIipA7SfA5wPnAPsDVwEXBQRu9eaTK7t/2LtfG8DdgEuHsWPoToTJsB558Hee8Nb3gL33lt2RZKk\n8aIS4QSxoOEhAAATA0lEQVSYC5ydmQsy807gOOA54JgB2p8AXJ6Zp2fmXZl5ErAI+ChAZj6VmYdk\n5oWZ+bvMvKF2bHZEdI7+xxHAppvCxRcXU43f9CZ44omyK5IkjQelh5OI6ABmA1f27svMBK4A5gzw\nsjm14/UWbqA9wJZAAk82XKyGbdtt4bLLYPFi+Mu/hBdfLLsiSVLVlR5OgKnARGBJn/1LgBkDvGbG\ncNpHxMbAV4HzM9MlwsbYLrvARRfBNdcUg2WdYixJ2pAqhJOBBEVPx4jaR8Qk4Ae1Yx9pTmkargMO\ngG9/uxiH8g//UHY1kqQqq8JU4mXAamB6n/3TWL93pNfiobSvCyZdwEFD6TWZO3cuU6ZMWWdfd3c3\n3d3dg71Ug3jPe+B//xf+7u9gxx2L55Kk8aWnp4eenp519i1fvryp7xFZgT72iLgOuD4zT6g9D+B+\n4MzM/Fo/7b8PbJqZh9ftuwa4NTM/UnveG0x2BF6fmY8PUsMs4KabbrqJWbNmNemTqa9MOOYYOP98\n+OlP4cADy65IkjRSixYtYvbs2QCzM3PRSM9Xlcs6pwPHRsR7I2JX4J8ppgOfCxARCyLiK3XtzwAO\njYhPRsQuEfH3FINqz6q1nwhcCMwCjgI6ImJ6besYqw+l9UXA2WfDa15TLM52111lVyRJqppKhJPM\nvAD4FHAycDPwauCQzFxaa9JJ3WDXzLwW6AaOBW4B3g4cnpm317V/c+3xFuBh4JHa44Zm9GgMbLQR\nXHghbLddsdz95Zc7SFaStFYlwglAZs7PzJdn5qaZOSczb6w7dlBmHtOn/YWZuWut/aszc2Hdsfsy\nc2KfbULt8edj+bnUvy23LKYYd3XBYYfB618P119fdlWSpCqoTDhR+9lhB/j5z+E//gMeewz2379Y\nC+XOO8uuTJJUJsOJShVRrB57yy3FNOObboI99oAPfhAeeqjs6iRJZTCcqBImToT3vrcYIHvaafCj\nH8FOO8GJJ7rsvSS1G8OJKmXjjeETnyjWQ/nrv4ZvfatYE+WUU+D558uuTpI0FgwnqqQttoCTT4Z7\n7ikWa/v854uelHPOgVWryq5OkjSaDCeqtOnT4ayz4I47iiXwjz0WXvUq+OEPnX4sSa3KcKJxYaed\noKenGDC7ww7FrJ45c+C//7vsyiRJzWY40bgyaxYsXAhXXglr1hTroxx6aDHbR5LUGgwnGpcOOqhY\ntO0HPygGz+6zz9obC0qSxjfDicatCHjHO+DXvy7u1/Ozn8Guu8LHPgaPPlp2dZKkRhlONO51dBQD\nZe++G774Rfi3fyumH590Etx/f9nVSZKGy3CiljF5Mnz2s8X04w9/uFgbZYcdYM894TOfKZbKdxqy\nJFWf4UQtZ5tt4Gtfg8WL4YILYPZs+M534MADYdtt4cgjYcECL/1IUlVNKrsAabRsuSUccUSxrVlT\nTEO+9NLibshHH12MWdl33+KuyG96UzETaIJxXZJK5/+K1RYmTCiCyN//PdxwQ9Gr8p3vFJd95s0r\njm2/Pbz//cUMoOXLy65YktqX4URtafr0ovfkggtg6dJiMbejj4Zf/hLe+U6YOhVe97ri8tBvfuNq\ntJI0lgwnansdHcV4lFNOKaYl//738M1vwuabFzN+XvUqeMUr4CMfKS4LPfdc2RVLUmtzzInUxw47\nwHHHFduKFUWvyqWXFts//RNsskmxMu1rXlPMBNpzz+I1EWVXLkmtwXAibcAmm8Cf/3mxnXkm3HVX\nMaD2ssvg1FPXjk3ZfPOih6U3rOy5Z/F8m23KrV+SxiPDiTREEcUKtLvuCp/8ZDEO5cEH4Ve/Wrtd\nd10x0HblyuI12223bmDZc0/Yffci9Iy1556Dxx+HJ5+E55+HF14oeoZWrFj7c3/7hvPziy8W07W7\nuqCzc93Hrq5irM/EiWP/2SWNL4YTqUERa//oHnbY2v0rV8Jvf7tuaPnhD+G004rjEybAK1+5fmjZ\nccehTWV+4QV47LEiaAz02N++FSuG9rk6OorwtPHGxeNAP2+6KWy11br7Jk0q1o958EG4+WZ44IF1\n33fSpGJW1EDhpbOzCDBO6Zbam+FEarKODthjj2J717vW7n/66WLmT31oOfPMIjxAscLtHnsUl4Om\nT4cnnug/fAw0IHfLLYvLSFtvXTzOnAmvfvXa51tvXWxbbVUEi/4Cx8YbNzcYZBZ1P/BAEVgeeGDd\nn2+6qXh84YV1//vNnNl/eJkxo+iZ2Xbb4lKa43yk1mQ4kcbI5pvD/vsXW6/MYs2V3rDy61/DbbcV\nIaQ3UEybVlxKqg8evUGj9+ettqrm5ZKIosZttoG99+6/TSYsW7Y2sPQNMddfXzy++OK6r9t442LK\n97RpawNL363+2JQphhlpvDCcSCWKKMalbLcdvPGNZVdTjoi1AWKfffpvk1msR7N4cfHY33b//UVP\nzNKlRW9N37VpOjqKMDNQeJk6teh9qt+mTCkuRUkaW/7aSaq8iCJITJs2tParVhUBZenSYgxMf2Fm\n8eKil2rp0qKnas2a/s+12WZFUNlqq/XDS9+tb5stthi4RysTVq8ueoRWrmzs8cUXi8+66aZFnZtv\nvv7jy17mGB6NP4YTSS1n0qS1YWaPPQZvv3p1MS38ySeHtt1777rPn3lm4HNvsUURFFatWj9cjJWX\nvayoYaAAM9CxzTYrwtXq1UX9vY/1Pzf6uGpVEZomT167bbrpus83tL+jw8t0rcxwIqntTZy4dhxP\nI1atWj/cPPHEuuGlo6PYNtqouY+TJhVTw595ptiefnrdxw3te/LJYjxP3+P1A5SH899w4sSinoEe\n++5bs6ao/bnniu3ZZ4vwMtT3GyzQ9M4q6x343d823ONVHNvVigwnkjRCkyatHfhbhs03L7ZmWbly\nbVjJHDx0TJjQvF6MlSuLoFIfWuq3gfb3d+zJJ9euw7NiRXG8/vlQp9fXmzRp7cy23m2jjdZ9Ptj+\nobxmo43WDaL120D7Jk4c296kzCJgrl7dWKDdkMqEk4g4Hvg0MAO4FfhYZv5yA+2PAE4GXg78FvhM\nZl7ep83JwAeALYFrgA9n5t2j8gFUST09PXR3d5ddhprE73NsdHQU42e22mp036e/77OjoxiIPGXK\n6L43FH9c6xcSHCzI1O9/4YV1txdfXH/fihXw1FPr7++v/apVI/88EYMHmQkTijDRGyr6/jycY6N5\nQ9RKhJOIOBI4DTgWuAGYCyyMiJ0zc1k/7ecA5wMnApcC7wYuioh9MvP2WpsTgY8CRwP3Av9QO+du\nmTmGV3tVJv+YtRa/z9ZS9vcZsfZyTdnWrFkbVOoHPPcdAN3I8/p9q1evvQQ3YULzfn7gAfjCF5r3\n36MS4YQijJydmQsAIuI44E3AMcCp/bQ/Abg8M0+vPT8pIt5IEUY+UtfmS5n549o53wssAd4KXDBa\nH0SSpOGaMKEY37LppmVX0phFi5obTkqfYBYRHcBs4MrefZmZwBXAnAFeNqd2vN7C3vYRsSPF5aH6\ncz4FXL+Bc0qSpAooPZwAU4GJFL0a9ZZQBIz+zBik/XQgh3lOSZJUAVW5rNOfoAgYzWy/oTabANxx\nxx3DeEtV3fLly1m0aFHZZahJ/D5bi99n66j729mUETxVCCfLgNUUvR31prF+z0evxYO0X0wRRKb3\nOcc04OYBzvlygKOOOmooNWscmT17dtklqIn8PluL32fLeTnwi5GepPRwkpkrI+Im4GDgEoCIiNrz\nMwd42bX9HH9DbT+ZeW9ELK61ua12zi2APwa+NcA5FwLvAX4PNDD7XZKktrUJRTBZ2IyTRY7mROWh\nFhHxTuA84EOsnUr8DmDXzFwaEQuABzPzc7X2c4CrgM9QTCXurv08q24q8d9QTDV+H0Xg+BKwB7CH\nU4klSaqu0ntOADLzgoiYSrGo2nTgFuCQzFxaa9IJrKprf21EdANfrm2/Aw7vDSa1NqdGxGTgbIpF\n2K4GDjWYSJJUbZXoOZEkSepVhanEkiRJLzGcSJKkSjGc1ETE8RFxb0Q8HxHXRcS+Zdek4YuIkyJi\nTZ/t9sFfqaqIiNdGxCUR8VDt+/uLftqcHBEPR8RzEfGfEbFTGbVqcIN9nxHxnX5+Zy8rq15tWER8\nNiJuiIinImJJRPwoInbu02bjiPhWRCyLiKcj4t8jYtpw3sdwwjo3HjwJ2IfirsgLa4N0Nf78mmJg\n9Yza9ppyy9EwvYxiUPzx9LNoYt1NPT8E7Ac8S/H7utFYFqkh2+D3WXM56/7OenfH6not8E2KpTn+\nDOgAfhoR9XcF+gbF/fH+EjgA2B64cDhv4oBYICKuA67PzBNqzwN4ADgzM/u78aAqKiJOopi5Navs\nWjRyEbEGeGtmXlK372Hga5k5r/Z8C4rFFo/OTG/qWWEDfJ/fAaZk5tvLq0yNqv0j/lHggMz8n9rv\n41LgXZn5o1qbXYA7gP0z84ahnLfte04avPGgqu2VtS7keyLiuxHRVXZBao6IeAXe1LMVva52ieDO\niJgfEVuXXZCGbEuKHrHHa89nUyxTUv87ehdwP8P4HW37cEJjNx5UdV1HsfDeIcBxwCuAn0fEy8os\nSk0zA2/q2WouB94LHAT8DXAgcFmtB1sVVvuOvgH8T906YzOAF2v/aKg3rN/RSizCVlHDvfGgKiAz\n65dO/nVE3ADcB7wT+E45VWkM+Ps6TvW5FPebiPgVcA/wOuBnpRSloZoP7M7QxvUN63fUnpPGbjyo\ncSIzlwO/BZzN0Rrqb+pZz9/XFpGZ91L8f9nf2QqLiLOAw4DXZebDdYcWAxvVxp7UG9bvaNuHk8xc\nCfTeeBBY58aDI76zosoVEZsBfwg8UnYtGrnaH67em3oC69zU09/XFhARncA2+DtbWbVgcjjw+sy8\nv8/hmyhuN1P/O7oz8AfUbs47FF7WKZwOnFe7O3LvjQcnA+eWWZSGLyK+BvyY4lLOTOCLFL8oPWXW\npaGrjQ/aiaKHBGDHiNgLeDwzH6C4xv35iLibtTf1fBC4uIRyNYgNfZ+17SSKaaaLa+1OoejtbMrd\nbdVcETGfYqr3XwDPRkRvL+byzFyRmU9FxP8FTo+IJ4CngTOBa4Y6UwecSvySiPgIxWCs3hsPfiwz\nbyy3Kg1XRPRQzMPfhmI62/8Af1v7F7fGgYg4kGKsQd//OZ2XmcfU2vw9cCxrb+p5fGbePZZ1amg2\n9H0CHwEuAvam+C4fpgglf1d341dVSG06eH/B4f2ZuaDWZmPg6xQhZmPgJxS/o48O+X0MJ5IkqUra\nfsyJJEmqFsOJJEmqFMOJJEmqFMOJJEmqFMOJJEmqFMOJJEmqFMOJJEmqFMOJJEmqFMOJJEmqFMOJ\npDETEd+JiB+WXYekajOcSGq6iNghItZExKvLrkXS+GM4kTQagv5vDjZuRMSEiIjBW0pqNsOJpA2K\niJ9FxBkRcUpEPBYRj0TESYO87H9rj7fUelD+q885PxURD0fEsog4KyImDlLDZyJicUQsj4h/jYh/\njIib+9R4ep/X/Cgivl33fKOI+HpEPBgRz0TEtbU75vYePzoinoiIt0TEb4AVwGsi4sWImNbn3GdE\nxH8P8t9AUoMMJ5KG4r3AM8B+wN8AfxcRB2+g/X4UvScHATOAt9cdOwjYEXhd7bzvq239ioh3AicB\nnwH+CHgE+AjD75n5FvDHwDuBPYEfAJdHxB/WtZlM8fn+CtgDuBG4B/g/dfVMorgV/LeRNCoMJ5KG\n4rbM/FJm3pOZ/0bxR3tD4WRp7fHxzHw0M5+sO/Y48NHM/G1mXgZcOsi5TgDOycxzM/N3mfkF4Pbh\nFB8Rf0ARgI7IzF9k5r2ZeTpwDfD+uqaTgA9n5nW193qeIoTUt/kLYGOKcCNpFBhOJA3FbX2ePwJM\nA4iIf4qIp2vbU0M4128ys77X46VzDWA34IY++64dwvvUexUwEfhtXa1PAwcA9T0nL2bmr/u89lzg\nlRGxX+350cAFteAiaRRMKrsASePCyj7Pk7X/uPkC8LUmnWsgg13CWUNxGaleR93PmwGrgFm1tvWe\nqft5vcCRmUsj4sfA+yPi98ChFKFG0igxnEgakcxcBizrs/vF2uMGB7oO0R3A/sD36vbt36fNUmC7\n3icRMYGit6R3IO7NtVqmZ+Y1DdTwr8D3gYeAuzPzugbOIWmIvKwjaTQ8StEL8ecRMS0ithjBuc4A\njomI90XEKyPiixSDVev9F/CmiDgsInYB/gnYsvdgZv4OOB9YEBFvi4iXR8R+tVlAhw6hhoXAcuBv\ncSCsNOoMJ5IGM+z1SjJzNfAx4EMUvQ0XNfzmmRcAXwJOoRiI20URPup9Gzivtv03xQyb/+rT5n3A\nAuDrwJ3Ajyhm/9w/hBqSYuzJRODfGvkckoYu1h2XJknVV1tn5fDMnDWG7/mvwNTMfOtYvafUrhxz\nIkkbULsk9Wrg3cCbSy5HaguGE0nasIuBfYH5mdn3UpGkUeBlHUmSVCkOiJUkSZViOJEkSZViOJEk\nSZViOJEkSZViOJEkSZViOJEkSZViOJEkSZViOJEkSZXy/wEk7kjTF7LiGAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe489123950>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.arange(1, 21), r_pre)\n",
    "plt.xlabel('n-th query')\n",
    "plt.ylabel('r-precision')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n-th query</th>\n",
       "      <th>r-precision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.035714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.031746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.017857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.011429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.007937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0.005831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>0.006696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>0.007055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>0.005714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>0.004723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>0.003968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>0.004227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>0.003644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>0.003810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>0.003348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>0.002966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>0.002646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>0.002374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>0.002500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    n-th query  r-precision\n",
       "0            1     0.142857\n",
       "1            2     0.035714\n",
       "2            3     0.031746\n",
       "3            4     0.017857\n",
       "4            5     0.011429\n",
       "5            6     0.007937\n",
       "6            7     0.005831\n",
       "7            8     0.006696\n",
       "8            9     0.007055\n",
       "9           10     0.005714\n",
       "10          11     0.004723\n",
       "11          12     0.003968\n",
       "12          13     0.004227\n",
       "13          14     0.003644\n",
       "14          15     0.003810\n",
       "15          16     0.003348\n",
       "16          17     0.002966\n",
       "17          18     0.002646\n",
       "18          19     0.002374\n",
       "19          20     0.002500"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.DataFrame({'n-th query': np.arange(1,21), 'r-precision':r_pre})\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Average Precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.36363634"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "averPrec = np.mean(np.unique(recall))\n",
    "display(averPrec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Harmonic Mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>harmonic mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.133333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.148148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.111111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.085106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.066667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.080000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.086957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.072072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.060606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.051613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.055556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.048309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.050847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.044944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.040000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.035821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.032258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.034063</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    harmonic mean\n",
       "0        0.166667\n",
       "1        0.133333\n",
       "2        0.200000\n",
       "3        0.148148\n",
       "4        0.111111\n",
       "5        0.085106\n",
       "6        0.066667\n",
       "7        0.080000\n",
       "8        0.086957\n",
       "9        0.072072\n",
       "10       0.060606\n",
       "11       0.051613\n",
       "12       0.055556\n",
       "13       0.048309\n",
       "14       0.050847\n",
       "15       0.044944\n",
       "16       0.040000\n",
       "17       0.035821\n",
       "18       0.032258\n",
       "19       0.034063"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from scipy.stats import hmean\n",
    "harmonic_mean = hmean(np.vstack((precision, recall)), axis=0)\n",
    "df2 = pd.DataFrame({ 'harmonic mean': harmonic_mean})\n",
    "display(df2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}