dermatrack_dnn_v1_7__w40_clsregr_filt_augm_custom_lrlowlow__continue0_BASIC_20180623_203022.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "custom": {
     "type": "setup"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from dermatrack import TMP_DIR\n",
    "import dermatrack.ml_engine.predictions.cloud_api as api\n",
    "import dermatrack.ml_engine.predictions.utils as utils\n",
    "import dermatrack.ml_engine.scripts.jobs as jobs\n",
    "import dermatrack.gcloud.storage as gstorage\n",
    "from dermatrack.ml_engine.scripts.notebook_evaluation import get_evaluation_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "custom": {
     "title": "setJobName",
     "type": "setup"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model name:  dermatrack_dnn\n",
      "Model version:  v1_7__w40_clsregr_filt_augm_custom_lrlowlow__continue0\n",
      "Model description:  Custom estimator, with window step 5, augmented size of 3 and filtered 1 at 3, also augmented batch size to 400, diminuished learning rate even more\n"
     ]
    }
   ],
   "source": [
    "JOB_NAME = 'dermatrack_dnn_v1_7__w40_clsregr_filt_augm_custom_lrlowlow__continue0_BASIC_20180623_203022'\n",
    "params = jobs.load_job(JOB_NAME)\n",
    "print('Model name: ', params['model']['name'])\n",
    "print('Model version: ', params['model']['version'])\n",
    "print('Model description: ', params['model']['description'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "custom": {
     "title": "setJobName",
     "type": "setup"
    }
   },
   "outputs": [],
   "source": [
    "DF_DIR = os.path.join(TMP_DIR, 'predictions_v2', params['jobName'])\n",
    "DF_NAME = 'evaluation_df.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "custom": {
     "title": "setGlobVar",
     "type": "setup"
    }
   },
   "outputs": [],
   "source": [
    "df = get_evaluation_df(params, file_name=DF_NAME, out_dir=DF_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "custom": {
     "type": "metrics"
    }
   },
   "source": [
    "## Metrics time\n",
    "\n",
    "Compute metrics on predictions\n",
    "\n",
    "N.B. Each of these cells must have in metadata:\n",
    "{\n",
    "  \"custom\": {\n",
    "    \"type\": \"metrics\"\n",
    "  }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "custom": {
     "type": "metrics"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(os.path.join(DF_DIR, DF_NAME))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "custom": {
     "title": "apr",
     "type": "metrics"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9578699463561429,\n",
       " 'precision': 0.8971815107102593,\n",
       " 'recall': 0.9548836093112552}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "metrics = utils.metrics(y_pred=df['y_pred'], y_true=df['y_true'], as_dict=True)\n",
    "metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "custom": {
     "title": "confusionMatrix",
     "type": "metrics"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVsAAAEmCAYAAADMczPyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XecFdXdx/HPF1AsqKiIBUEs2EIM\nomKLJWrsiuaJeawgMfYYo7FGE1HEaOITo7EFY0GNPTF2kahYEQUEu4IdRQUpggq48Hv+mLPksi67\nd9e9987uft+85sWdM+03d+7+7rlnzswoIjAzs9JqU+kAzMxaAydbM7MycLI1MysDJ1szszJwsjUz\nKwMnWzOzMmi1yVbS0pLukzRT0p3fYT2HSHqkKWOrFEnbSXozL9uT1F1SSGpXrpiaC0nvSdolvf6t\npL+XYBtXS/pdU6+3tVLe+9lKOhg4GdgQmAWMAwZHxNPfcb2HAScA20RE1XcONOckBdAjIiZWOpbF\nkfQe8IuI+E8a7w68CyzR1MdI0g3ApIg4uynXWy4136smWN/haX0/bIr12bflumYr6WTgL8AFwKpA\nN+BKoG8TrH4t4K3WkGiL4dpj6fi9NQAiIpcDsAIwGzigjnnakyXjj9PwF6B9mrYjMAn4DfAZMBkY\nkKadC8wDvknbOAIYCNxcsO7uQADt0vjhwDtktet3gUMKyp8uWG4b4AVgZvp/m4JpI4BBwDNpPY8A\nnRazb9Xxn1YQ/37AnsBbwDTgtwXz9wFGAjPSvJcDS6ZpT6Z9+TLt7/8WrP904BPgpuqytMy6aRu9\n0/gawFRgxyKO3VDgN+l1l7Tt49L4emm9qrG9m4AFwNcpxtMKjkF/4IO0/bOKPP6LHJdUFmn7R6Vj\nPy9t677F7EcAxwATgOnAFfz312Ab4Gzg/XR8bgRWqPHZOSLF/WRB2QDgw7S+Y4AtgJfScbu8YNvr\nAo8Bn6f9/gfQsWD6e8Au6fVA0mc3HffZBUMVMDBNOwN4m+yz9xqwfyrfCJgDzE/LzEjlNwDnF2zz\nSGBiOn73AmsU8155SO9RpQNYbGCwe/qgtKtjnvOA54DOwCrAs8CgNG3HtPx5wBJkSeorYMWaH9DF\njFf/cbQDlgW+ADZI01YHvpdeH076owZWSh+0w9JyB6XxldP0EenDvj6wdBq/cDH7Vh3/71P8RwJT\ngFuA5YDvpT+QddL8mwFbpe12B14Hfl2wvgDWq2X9F5ElraUpSH5pniPTepYBhgEXF3nsfk5KYMDB\naZ9vL5h2T0EMhdt7j5RAahyDa1J8PwDmAhsVcfwXHpfa3gNqJJLF7EcA9wMdyX5VTQF2L9iPicA6\nQAfgX8BNNeK+keyzs3RB2dXAUsCu6fj9O8XfhSxp75DWsR7w43RsViFL2H+p7b2ixme3YJ5eKeZN\n0/gBZF+abci+cL8EVq/j/Vr4HgE7kSX93immvwJPFvNeeciGPDcjrAxMjbp/5h8CnBcRn0XEFLIa\n62EF079J07+JiAfJvrU3aGQ8C4CekpaOiMkR8Wot8+wFTIiImyKiKiJuBd4A9imY5/qIeCsivgbu\nIPuDWJxvyNqnvwFuAzoBl0bErLT9V4FNACJiTEQ8l7b7HvA3YIci9umciJib4llERFxDVlMZRfYF\nc1Y966v2BLCdpDbA9sAfgW3TtB3S9IY4NyK+jojxwHiypAv1H/+mcGFEzIiID4DH+e/xOgT4c0S8\nExGzgTOBA2s0GQyMiC9rvLeDImJORDxCluxuTfF/BDwFbAoQERMjYng6NlOAP1P/8VxI0ipkifyE\niHgxrfPOiPg4IhZExO1kx7ZPkas8BLguIsZGxNy0v1undvVqi3uvjHy32X4OdKqnvWsNsp9x1d5P\nZQvXUSNZf0VWC2mQiPiSrCZwDDBZ0gOSNiwinuqYuhSMf9KAeD6PiPnpdfUf7KcF07+uXl7S+pLu\nl/SJpC/I2rk71bFugCkRMaeeea4BegJ/TX9k9YqIt8m+2HoB25HVeD6WtAGNS7aLe8/qO/5NoSHb\nbkd2bqHah7Wsr+bxW9zx7CzpNkkfpeN5M/UfT9KySwB3AbdExG0F5f0kjZM0Q9IMsuNa1Dqpsb/p\nC+ZzGv/ZbnXynGxHkv3M2q+OeT4mO9FVrVsqa4wvyX4uV1utcGJEDIuIH5PV8N4gS0L1xVMd00eN\njKkhriKLq0dELA/8lqxdtC51dkWR1IGsHfRaYKCklRoQzxPAT8najT9K4/2AFcl6lDQ4nlrUdfwX\nOZ6SFjmejdhWMduuYtHk+V228Ye0/CbpeB5K/cez2l/J2mUX9rSQtBbZZ/aXZM1aHYFXCtZZX6yL\n7K+kZcl+fZbjs90i5DbZRsRMsvbKKyTtJ2kZSUtI2kPSH9NstwJnS1pFUqc0/82N3OQ4YHtJ3SSt\nQPYzCQBJq0raN33A5pLV2ubXso4HgfUlHSypnaT/BTYmq9mV2nJk7cqzU6372BrTPyVrX2yIS4Ex\nEfEL4AGy9kYAJA2UNKKOZZ8g+8N+Mo2PIOtq93RBbb2mhsZY1/EfD3xPUi9JS5G1a36XbdW27ZMk\nrZ2+lC4ga5duqt4ty5FOVknqApxazEKSjib79XBwRCwomLQsWUKdkuYbQFazrfYpsKakJRez6luA\nAen9bE+2v6NSk5UVIbfJFiAi/kzWx/Zssg/Jh2R/wP9Os5wPjCY7m/syMDaVNWZbw4Hb07rGsGiC\nbEPWq+FjsjOxOwDH1bKOz4G907yfk51R3zsipjYmpgY6hexk1CyyGsztNaYPBIamn5A/q29lkvqS\nnaQ8JhWdDPSWdEga70rWq2JxniBLGNXJ9mmymuaTi10iq82dnWI8pb4YqeP4R8RbZCfQ/kPWNlmz\nX/a1wMZpW/+m4a4j60HxJFnvlDlkXyZN5Vyyk1Ezyb7o/lXkcgeRfYl8LGl2Gn4bEa8B/0f2i/FT\n4PssevweIzsH8Imkb31eI+JR4HfAP8l6u6wLHNiYHWutcn9Rg+WTpHHAzukLxszq4WRrZlYGuW5G\nMDNrKZxszczKwMnWzKwMmv0NMtRu6dCSy1U6DGuEXht1q3QI1kjvv/8en0+dWmy/36K0XX6tiKpv\nXchYq/h6yrCI2L0pt19qzT/ZLrkc7TeotyeT5dATz1xW6RCskXbYttirfIsXVV8X/bc8Z9wVxV75\nlhvNPtmaWUshUMtt2XSyNbN8EKAmbZnIFSdbM8uPNm0rHUHJONmaWU64GcHMrDzcjGBmVmLCNVsz\ns9KTa7ZmZmXhmq2ZWampRfdGaLlfI2bWvFT3sy1mqG9V0nWSPpP0SkHZSpKGS5qQ/l8xlUvSZZIm\nSnpJUu+CZfqn+SdI6l9Qvpmkl9Myl0n1B+Vka2b5oTbFDfW7gexJI4XOAB6NiB7Ao2kcYA+gRxqO\nInueH+mZe+cAW5I9hfic6gSd5jmqYLl679PgZGtmOaEmS7YR8STZI6wK9QWGptdD+e/DZPsCN0bm\nOaCjpNWB3YDhETEtIqYDw4Hd07TlI2JkZE9fuJG6H0wLuM3WzPKkTdG9ETpJGl0wPiQihtSzzKoR\nMRkgIiZL6pzKu7DoY+cnpbK6yifVUl4nJ1szy4eG9bOdGhGbN+GWa4pGlNfJzQhmlh9NdIJsMT5N\nTQCk/z9L5ZPInhZdbU2yJ2nXVb5mLeV1crI1s5xIXb+KGRrnXqC6R0F/4J6C8n6pV8JWwMzU3DAM\n2FXSiunE2K7AsDRtlqStUi+EfgXrWiw3I5hZfjTRRQ2SbgV2JGvbnUTWq+BC4A5JRwAfAAek2R8E\n9gQmAl8BAwAiYpqkQcALab7zIqL6pNuxZD0elgYeSkOdnGzNLB++WxPBIiLioMVM2rmWeQM4fjHr\nuQ64rpby0UDPhsTkZGtm+eHLdc3MysA3ojEzKzXfPNzMrDxcszUzKzEJ2rTclNRy98zMmh/XbM3M\nysBttmZmZeCarZlZicm9EczMysM1WzOz0hLQpo1rtmZmpSVqv1NsC+Fka2Y5IYp4bmKz5WRrZrnh\nZGtmVgZOtmZmZeBka2ZWaj5BZmZWekLu+mVmVg5uRjAzKwMnWzOzUnObrZlZebhma2ZWYvIVZGZm\n5eFka2ZWagK1cbI1Mys512zNzMrAydbMrMR8gszMrFxabq51si2Hq885hD2278mUabPY/IALAFhx\n+WW46aKfs9YaK/H+x9M49LRrmTHrawC226wHfzr1f1iiXVs+nzGbXX9xKQArdFiaq845mI3XXZ0I\nOObcfzDqpXf5/XF7sfcOm7AgginTZnHUOTczecrMiu1vazF//nx22LYPq6+xBnf+6z6OOXIAzzz1\nJMuvsAIAVw25jk1+0AuAp54cwRmnnsw333zDyit34qHhj1cy9HySmxHsO7rpvue4+vYn+PugfgvL\nThnwY0Y8/yYXXz+cUwb8mFMG7MrZl93DCh2W5tLf/oy+x1/Jh59MZ5UVOyxc5uLTfsojz77Gwade\nyxLt2rLMUksCcMnQRznvygcAOO6gHTjzqD341eDbyruTrdBVl1/G+htsyKxZXywsG3TBRez3k58u\nMt+MGTM4+cRf8q97HqRrt25M+eyzcofabLTkG9G03D3LkWfGvs20mV8tUrb3jptw832jALj5vlHs\n86NNAPjfPTbnnkfH8+En0wGYMn02AMstuxQ/7L0uN9w9EoBvquYzc3ZWE5715ZyF611m6fZERGl3\nyPho0iSGPfwg/QccUe+8d95+K/v03Z+u3boBsErnzqUOr/lSkUMz5GRbIZ1XXo5PpmY1ok+mfsEq\nKy0HQI+1OtNx+WUYds2JPPOP0zh47z4ArN1lZaZOn82Qcw9l5K2nc+XvD15YswUYePw+THhoEAfu\nsTmDrnqg/DvUypxx6kmcN/jCb9XEzhv4O7beohdnnHoyc+fOBWDihLeYMWM6e+66E9tvswW3/OPG\nSoTcLEgqamiOcplsJe0naeNKx1EJ7dq2ofdGXdn/hKvY9/grOPPI3VmvW2fatWtLrw27cs2dT7H1\nQRfx1ddzOeXnP1643MAr7qPHHr/jtodGc8z/bl/BPWj5Hnrwfjp17symvTdbpHzgeRcwZvxrjHh6\nFNOnT+OS//sjAFVVVYwbO5Y7776Pu+99iD/+YTATJrxVidBzrdhE62RbB2Uasq39gBadbD/7fBar\ndVoegNU6Lc+UabMA+OizGTzy7Ot8NWcen8/4kqfHTmST9bvw0afT+eizGbzwyvsA3P2fcfTasOu3\n1nvHQy+w3869yrcjrdCokc/y0P330XODdRjQ72CeHPE4vxhwGKutvjqSaN++PYf2O5wxo58HoEuX\nNdll191YdtllWblTJ7b94Xa88tL4Cu9FPjVVspV0kqRXJb0i6VZJS0laW9IoSRMk3S5pyTRv+zQ+\nMU3vXrCeM1P5m5J2+y77VrJkK6m7pNclXQmMBQ6TNFLSWEl3SuqQ5rtQ0muSXpJ0saRtgH2BP0ka\nJ2ndUsVYSQ888TKH7rMlAIfusyX3j3gJgPtGvMS2m65L27ZtWHqpJdiiZ3feePcTPv18FpM+mU6P\ntbL2vh37bMAb73wCwLrdVlm43r122IS33vu0zHvTugwcdAFvvP0Br7z5DtffeAvb7/gj/n79TXwy\neTIAEcH9997Dxhv3BGCvffZl5DNPU1VVxVdffcXoF55ngw03quQu5FZTJFtJXYBfAZtHRE+gLXAg\ncBFwSUT0AKYD1Q3uRwDTI2I94JI0H+nX9YHA94DdgSsltW3svpW6N8IGwADg98C/gF0i4ktJpwMn\nS7oc2B/YMCJCUseImCHpXuD+iLirtpVKOgo4CoAlOtQ2S64M/cPhbLdZDzp17MDEhwcx6OoHufj6\n4dx80c/pv9/WfDh5Ooecdi0Ab777KcOffY0X7jiTBQuCG+5+ltfezv6IT77oTq6/4HCWbNeW9z6a\nylHn3AzA+b/qS4+1OrNgQfDB5GnuiVAhvxhwGFOnTiEi+P4mP+Avf70KgA023IhdfrwbW2/RizZt\n2tDv8CPY+Hs9KxxtTjVdC0E7YGlJ3wDLAJOBnYCD0/ShwEDgKqBveg1wF3C5sozeF7gtIuYC70qa\nCPQBRjYmIJXqzHWqij8eEWtL2hu4AZiUJi9JFvDRwBhgNPAAWYKdJ+kG6ki2hdos0znab/CzJo/f\nSu+zkZdVOgRrpB227cPYMaObtPG0/Wo9Ys1DivtMvPPnPd8HphYUDYmIIdUjkk4EBgNfA48AJwLP\npdorkroCD0VET0mvALtHxKQ07W1gS7IE/FxE3JzKr03L1JuXalPqmu2X6X8BwyPioJozSOoD7ExW\nXf8l2bePmbUyAhpw7mtqRGxe63qkFclqpWsDM4A7gT1qmbW6plnbVqOO8kYpV2+E54BtJVV/qywj\naf3UbrtCRDwI/BqoPrMzC1iuTLGZWS40WW+EXYB3I2JKRHxD1oS5DdBRUnUFc03g4/R6EtAVIE1f\nAZhWWF7LMg1WlmQbEVOAw4FbJb1Elnw3JEuo96eyJ4CT0iK3AadKerGlniAzs2+Tihvq8QGwVarU\nieyX82vA40D15X39gXvS63vTOGn6Y5G1r94LHJh6K6wN9ACeb+y+lawZISLeA3oWjD8GbFHLrH1q\nWfYZWnjXLzP7tqboQxsRoyTdRdYLqgp4ERhCdl7oNknnp7Jr0yLXAjelE2DTyJo0iYhXJd1Blqir\ngOMjYn5j4/K9EcwsH4qrtRYlIs4BzqlR/A61V+7mAAcsZj2DyU60fWdOtmaWCwLa+LE4Zmal52Rr\nZlZqTdiMkEdOtmaWC1k/25abbZ1szSwnmu8dvYrhZGtmudGCc62TrZnlh2u2ZmYlJrk3gplZWbTg\niq2TrZnlh5sRzMzKoAXnWidbM8sJuWZrZlZyDbx5eLPjZGtmOeGLGszMysJdv8zMSs03ojEzKz3f\niMbMrEycbM3MyqAF51onWzPLD9dszcxKzSfIzMxKT8hdv8zMyqFNC67aOtmaWW604FzrZGtm+SDf\niMbMrDxacJPt4pOtpOXrWjAivmj6cMysNWutNdtXgSC7iq5a9XgA3UoYl5m1MqKVniCLiK7lDMTM\nrCU3I7QpZiZJB0r6bXq9pqTNShuWmbU6yu5nW8zQHNWbbCVdDvwIOCwVfQVcXcqgzKx1koobmqNi\neiNsExG9Jb0IEBHTJC1Z4rjMrJVptW22Bb6R1IbspBiSVgYWlDQqM2uVWnCuLarN9grgn8Aqks4F\nngYuKmlUZtYqNWWbraSOku6S9Iak1yVtLWklScMlTUj/r5jmlaTLJE2U9JKk3gXr6Z/mnyCpf2P3\nrd6abUTcKGkMsEsqOiAiXmnsBs3MaiNB26btjnAp8HBE/DQ1fS4D/BZ4NCIulHQGcAZwOrAH0CMN\nWwJXAVtKWgk4B9ic7Nf9GEn3RsT0hgZTVG8EoC3wDTCvAcuYmTWIihzqXU92Udb2wLUAETEvImYA\nfYGhabahwH7pdV/gxsg8B3SUtDqwGzA8IqalBDsc2L0x+1ZMb4SzgFuBNYA1gVskndmYjZmZ1aUJ\nmxHWAaYA10t6UdLfJS0LrBoRkwHS/53T/F2ADwuWn5TKFlfeYMWcIDsU2CwivgKQNBgYA/yhMRs0\nM6tN1huh6Nk7SRpdMD4kIoYUjLcDegMnRMQoSZeSNRnUtfmaal5BW1jeYMUk2/drzNcOeKcxGzMz\nW6yGXbAwNSI2r2P6JGBSRIxK43eRJdtPJa0eEZNTM8FnBfMXXjW7JvBxKt+xRvmIYoMstNhmBEmX\nSPoz2UUMr6Zq+DXAy8CMxmzMzKwuTXVRQ0R8AnwoaYNUtDPwGnAvUN2joD9wT3p9L9Av9UrYCpiZ\nmhmGAbtKWjH1XNg1lTVYXTXb6h4HrwIPFJQ/15gNmZnVp4kvxT0B+EfqifAOMICsgnmHpCOAD4AD\n0rwPAnsCE8kqmANg4UVcg4AX0nznRcS0xgRT141orm3MCs3MGkM0bdeviBhH1mWrpp1rmTeA4xez\nnuuA675rPPW22UpaFxgMbAwsVRDA+t9142ZmhVrwBWRF9Zm9Abie7H3YA7gDuK2EMZlZKyRl90Yo\nZmiOikm2y0TEMICIeDsizia7C5iZWZNq7Xf9mqus1fptSccAH/HfjsBmZk2mud6rthjFJNuTgA7A\nr8jablcAfl7KoMysdWrBubaoG9FUdwqexX9vIG5m1qQkNfWNaHKlrqfr3k0dl6VFxE9KElEDbbpR\nN54ZdXmlw7BGuHPch/XPZLk0/et5JVlva21GcAYzs7JqybcUrOuihkfLGYiZtW6i9dZszczKqgU3\n2TrZmll+ONkCktpHxNxSBmNmrVd2wULLzbbFPKmhj6SXgQlp/AeS/lryyMys1WnbprihOSom7MuA\nvYHPASJiPL5c18yaWPakhpZ7b4RimhHaRMT7Nar380sUj5m1Ys200lqUYpLth5L6ACGpLdkNed8q\nbVhm1ho100prUYpJtseSNSV0Az4F/pPKzMyajJpxE0Exirk3wmfAgWWIxcxauRaca4t6UsM11HKP\nhIg4qiQRmVmr1dr72f6n4PVSwP6A7yBiZk2qqZ9BljfFNCPcXjgu6SZgeMkiMrPWSa7Z1rQ2sFZT\nB2Jmphb8yMdi2myn89822zbANOCMUgZlZq1PdlFDpaMonTqTbXr22A/InjsGsCA9X93MrMm15GRb\n5wUbKbHeHRHz0+BEa2YlI6mooTkq5uq45yX1LnkkZtaqSS37RjR1PYOsXURUAT8EjpT0NvAlWdNK\nRIQTsJk1qdZ6BdnzQG9gvzLFYmatWGs+QSaAiHi7TLGYWSvXgiu2dSbbVSSdvLiJEfHnEsRjZq2W\naNNK+9m2BTpAC957M8uN7Om6lY6idOpKtpMj4ryyRWJmrVsrvly3Be+2meVNa74Rzc5li8LMjJbd\n9Wux3YMjYlo5AzEzyx5nXv9Q3LrUVtKLku5P42tLGiVpgqTbJS2Zytun8YlpeveCdZyZyt+UtNt3\n2bdmei2GmbU0IktIxQxFOhF4vWD8IuCSiOgBTAeOSOVHANMjYj3gkjQfkjYme0rN94DdgSvTcxgb\nxcnWzPJBTXdvBElrAnsBf0/jAnYC7kqzDOW/F2z1TeOk6Tun+fsCt0XE3Ih4F5gI9Gns7jnZmllu\nqMgB6CRpdMFQ8zFdfwFOAxak8ZWBGekWBACTgC7pdRfS02fS9Jlp/oXltSzTYI25ebiZWZPLLtct\n+gTZ1IjYvNb1SHsDn0XEGEk7Fqy+pqhnWl3LNJiTrZnlRhP1/NoW2FfSnmTPTVyerKbbseAGW2sC\nH6f5JwFdgUmS2gErkD0kobq8WuEyDeZmBDPLieLaa+trs42IMyNizYjoTnaC67GIOAR4HPhpmq0/\ncE96fW8aJ01/LN27+17gwNRbYW2gB9kNuhrFNVszy4Xq3ggldDpwm6TzgReBa1P5tcBNkiaS1WgP\nBIiIVyXdAbwGVAHHR8T8xm7cydbMcqOpn8IQESOAEen1O9TSmyAi5gAHLGb5wcDgpojFydbMcqPl\nXj/mZGtmeaGmr9nmiZOtmeWCgLZOtmZmpddyU62TrZnlSAuu2DrZmlk+ZF2/Wm62dbI1s9xwzdbM\nrOSEXLM1Mys912zNzEpMctcvM7OyaMG51snWzPLDbbZWEkf/4uc89OD9rNK5M2PGvQLA+HHjOOH4\nY5g7Zw7t2rXjL3+9ki369GHmzJn8vP+hfPjBB1TNr+LXJ51Cv8MHVHgPWpd5c+dwwVEH8M0381hQ\nVcUWO+/JT47+Da+98Ay3XjqYqm/m0X2j7/OLs/9E23bteOCmqxn50L8BmD+/io/fm8gVj4yjwwod\nGXbrtYz4960QwQ77HcTuB/+iwntXednNwysdRen4frYVdFj/w7nn/ocXKTvrzNM463fnMGrMOH43\n8DzOOvM0AP521RVsuNHGPD92PMP+M4IzTvsN8+bNq0TYrdYSS7bnjKtuY/Atwxh0y8O8NPIJJowf\nzZCBJ3Pc4Mv5w+3/odNqa/L0A9ljrvY67BjOv+Vhzr/lYX52/Ols2HsrOqzQkUkT32TEv29l4ND7\nOP+WYYx7+lE++eDdCu9dPqjIf82Rk20F/XC77VlppZUWKZPEF198AcDMmTNZfY01FpbPnjWLiODL\n2bNZcaWVaNfOP0zKSRJLLbMsAPOrqphfVUWbtm1pt+SSrL7WOgD03PKHvPDYQ99aduQj97LVrvsC\n8PF7E1jv+71pv9TStG3Xjg17b8WYEQ9/a5nWqCkfZZ43/mvNmT/931/YZ6/dOPP0U1iwYAGPP/ks\nAMcc90t+uv++rNNtDWbNmsVNt9xOmzb+riy3BfPn8/vD9uLTSe+xywH9WOd7vZhfVcU7r41nnY1/\nwAuPPsi0Txd9csrcOV/z8sgR9Dt1EABd1t2AO6/6E7NmTGfJpZZi/LOPs/ZGm1Rid3KnudZai1HS\nv1ZJz6b/u0s6uIj535PUqZQx5d2Qv13FHy++hInvfsgfL76EY4/KHm0//JFhbPKDXrzzwceMGj2O\nk0785cIasJVPm7ZtOf+Wh/nLA6N459XxfPT2Wxw3+HJuueQ8Bvbfh6WW7UCbtovWYV58cjg9Ntmc\nDit0BKDL2j3Yu9+x/PGXh3Dxrw6jW4+NaNu2bSV2J1eEaKvihuaopMk2IrZJL7sD9SZbg3/cNJT9\n9v8JAP/z0wMY/UL2yKObhl5P3/1/giTWXW89undfmzffeKOSobZqyy63AhtuthUvjRxBj0024+xr\n/snAofexwaZbslq37ovMO2r4fWy1W99FynboeyCDbn6Qs4bcRYflO7Jq17XLGH1OFdmE0Exzbclr\ntrPTywuB7SSNk3SSpLaSLpb0sqSXJJ1QsNgJksamaRuWMr48Wn2NNXjqyScAGPH4Y6y3Xg8Aunbt\nxojHHgXg008/5a233mTtddapWJyt0RfTP+fLWTMBmDdnDq8+/zSrd1+XL6ZNBeCbeXN5YOiV7PST\nQxcu89XsL3hj7HNstsOui64rLTP1k48Y/fjDbL3bvmXai3xTkUNzVK422zOAUyJibwBJxwJrA5tG\nRJWkwrNEUyOit6TjgFOAb/WJkXQUcBRA127dSh58qfQ79CCeemIEU6dOZd3ua/K735/LFVddw6kn\nn0hVVRXtl1qKy68aAsAZZ/2jKfb4AAALPElEQVSOo444nM17fZ8gGHzBRXTq1KpbXMpuxtTPGDLw\nZGLBfBYsWMCWu+zNptvtwq2XDmbc048SCxaw0/8cysZbbLtwmTGPD6PnltvTfullFlnXZacfzeyZ\n02nbbgn6nTaIZZfvWO7dyZ2s61dzTaX1U/bE3hKtXJodER0k7ciiyfafwNURMbzG/O8B20bER5K2\nBAZHxC51bWOzzTaPZ0aNLs0OWEndOe7DSodgjfT7fnvx7msvNWlm3Oj7m8b1dz9e1Lxb91hxTERs\n3pTbL7VK9UYQsLgsPzf9Px/3ljBrXVpuxbZs/WxnAcsVjD8CHCOpHUCNZgQza6XaSEUNzVG5ku1L\nQJWk8ZJOAv4OfAC8JGk87qlgZvgEWaNFRIf0/zfAzjUmn5yGwvm7F7weDexYyvjMLGeaayYtgttE\nzSwXslpry822TrZmlg/N+IKFYjjZmllutOBc62RrZjnSgrOtk62Z5UTz7dZVDCdbM8uF5tytqxhO\ntmaWHy042zrZmllutOSuX77Vv5nlRlPdz1ZSV0mPS3pd0quSTkzlK0kaLmlC+n/FVC5Jl0mamG77\n2rtgXf3T/BMk9W/svjnZmlluNOHlulXAbyJiI2Ar4HhJG5Pd7vXRiOgBPJrGAfYAeqThKOAqWHjf\nlnOALYE+wDnVCbqhnGzNLB+KzbRFZNuImBwRY9PrWcDrQBegLzA0zTYU2C+97gvcGJnngI6SVgd2\nA4ZHxLSImA4MB3ZvzO65zdbMcqFUNw+X1B3YFBgFrBoRkyFLyJI6p9m6AIU3WJ6UyhZX3mBOtmaW\nGw1ItZ0kFT41YEhEDPnW+qQOwD+BX0fEF1p8Mq9tQtRR3mBOtmaWH8Vn26n1PalB0hJkifYfEfGv\nVPyppNVTrXZ14LNUPgnoWrD4msDHqXzHGuUjio6ygNtszSw3VOS/eteTVWGvBV6PiD8XTLoXqO5R\n0B+4p6C8X+qVsBUwMzU3DAN2lbRiOjG2ayprMNdszSw3mrDJdlvgMOBlSeNS2W/JnvR9h6QjyB5g\ncECa9iCwJzAR+AoYABAR0yQNAl5I850XEdMaE5CTrZnlRlPl2oh4uo7V1XyQAZE9+fb4xazrOuC6\n7xqTk62Z5YKAOk5gNXtOtmaWD755uJlZebTgXOtka2Y50oKzrZOtmeVEcd26misnWzPLDbfZmpmV\nmJ/UYGZWJu76ZWZWBi041zrZmll+tOBc62RrZjnhixrMzMql5WZbJ1szy4Xs3giVjqJ0nGzNLDda\ncK51sjWz/CjFM8jywsnWzPKj5eZaJ1szy48WnGudbM0sH+SuX2Zm5eG7fpmZlUPLzbVOtmaWH22c\nbM3MSs03DzczK7mWfgVZm0oHYGbWGrhma2a50ZJrtk62ZpYbbrM1Mys1X9RgZlZ6Lf0EmZOtmeWG\nmxHMzMrANVszszJowbnWydbMcqQFZ1snWzPLjZbcZquIqHQM34mkKcD7lY6jRDoBUysdhDVaSz5+\na0XEKk25QkkPk71nxZgaEbs35fZLrdkn25ZM0uiI2LzScVjj+PhZId8bwcysDJxszczKwMk234ZU\nOgD7Tnz8bCG32ZqZlYFrtmZmZeBka2ZWBk62ZmZl4GSbc1JLvjWHWevhZJtjkpYA+khaUtJBknpV\nOiYrnr8orZB7I+SYpDWAE4GNgfWBH0bElMpGZQ0laTVgVkR8WelYrHJcs82xiPgYGA1sC9wBfFXZ\niKwYkraWtFZ6/RvgYeBSScdXNjKrJCfbHJO0BzAJ6AusAhwrae00rdgbdlj5/QR4SNLuwPeBI4B/\nAntKOrmikVnF+BaLOSXpVGAP4NiIeFPSHOBXQJWkzsDGkg6KiK8rGqgtJKlNRCyIiFMlzQL+BgyN\niDGSlgJmAadLWjoiBlc2Wis312xzSNJWwH4RsRMwQdKmwJfAqUAHYEPgHCfa/JCkiFiQXm8YEeeR\nJdufSeoaEXOA54A/A70krVTBcK0CfIIsh1KyvRy4EVgX2AD4EbBTRDwjacmImFfJGK12kk4Ctif7\nRfKJpIuAXYCfRMT7ktoBS/iLsvVxzTZHJPWRtCrwOvBXYDvgrnST5N8D3QCcaPMptbEfCBwZEZ8A\nRMTpZCfIHk813Con2tbJbbY5IelXwP8AzwLrAQMiYmia1g84HNi3YgHat9TyC2MZYGxETE01WFJy\nPSu14S5RkUAtF5xsc0BSH2B/YEfgGmAB8KWkZcnaZ38NHBAREyoWpC1C0vLAXpIeAbYB2gOzgfaS\nOkfEZ2m+g4F5EXFh5aK1PHCbbYVJOhroTNak8zmwN9nJsTmSfgSMAtpHxPQKhmkFJLWLiCpJhwFn\nAfMj4nvpy/F64E3gXSCAM4C9ImJi5SK2PHCbbQVJ2gvYCrgP2BM4JiJ2T4n2aLKrx9o60eaHpFXI\n+swCTAZWBCZJ6pKuEDsJmAn0BHYiOzHmRGuu2VaKpC7ASGBERPRLtaS9yGpEHwMDgP4R8XIFw7Ra\nSFoG6AM8T9YO24+sx8E5ETFOUs+IeMW9RqyQ22wrJCI+kvRr4GpJ+0XETZLGA0cDXwCHRsRrlY3S\nahMRX0laEXiD7AqxK4HlgMGSxgA9JR3hXyRWyDXbCpO0N3ABMCgi7qx0PFa8dDnuX4HNI2KmpKPI\neoycHhGvVjY6yxsn2xxI/TOHAL+OiH/WN7/lh6Q9gf8Dto2IaZLaR8TcSsdl+eNkmxOSfgy8HRHv\nVDoWaxhJfYFzgd4A1ZftmhVysjVrApI6RMTsSsdh+eVka2ZWBu5na2ZWBk62ZmZl4GRrZlYGTrZm\nZmXgZNuKSJovaZykVyTdmS47bey6dpR0f3q9r6Qz6pi3o6TjGrGNgZJOKba8xjw3SPppA7bVXdIr\nDY3RrFhOtq3L1xHRKyJ6AvOAYwonKtPgz0RE3FvPLQQ7Ag1OtmYtiZNt6/UUsF6q0b0u6UpgLNBV\n0q6SRkoam2rAHSC7PFXSG5KeJnuCLKn8cEmXp9erSrpb0vg0bANcCKybatV/SvOdKukFSS9JOrdg\nXWdJelPSf8geB1QnSUem9YyX9M8atfVdJD0l6a10WTSS2kr6U8G2j/6ub6RZMZxsW6H0FIE9gOo7\nim0A3BgR1Q+WPBvYJSJ6A6OBk9PTYa8B9iF7XM9qi1n9ZcATEfEDsiuqXiW7p+vbqVZ9qqRdgR5k\nd87qBWwmaXtJm5E9VmZTsmS+RRG786+I2CJt73Wyx4ZX6w7sQHY3tavTPhwBzIyILdL6j1R6PLxZ\nKfmuX63L0pLGpddPAdcCawDvR8RzqXwrYGPgGUkAS5LdCnJD4N3qp0VIuhk4qpZt7ER2y0EiYj4w\nM90hq9CuaXgxjXcgS77LAXdHxFdpG/cWsU89JZ1P1lTRARhWMO2OdOnsBEnvpH3YFdikoD13hbTt\nt4rYllmjOdm2Ll9HRK/CgpRQvywsAoZHxEE15utF9uSBpiDgDxHxtxrb+HUjtnED2ZMtxks6nOzR\nQtVqrivStk+IiMKkjKTuDdyuWYO4GcFqeg7YVtJ6kN0oW9L6ZPduXVvSumm+gxaz/KPAsWnZtulZ\nXbPIaq3VhgE/L2gL7iKpM/AksL+kpSUtR9ZkUZ/lgMmSlgAOqTHtAEltUszrkD2uZhhwbJofSeun\nx9mYlZRrtraIiJiSaoi3Smqfis+OiLfS/VofkDQVeJrs0S81nQgMkXQEMB84NiJGSnomda16KLXb\nbgSMTDXr2WQ3Sx8r6XZgHPA+WVNHfX5H9py298naoAuT+pvAE8CqZI8cmiPp72RtuWOVbXwKsF9x\n745Z4/lGNGZmZeBmBDOzMnCyNTMrAydbM7MycLI1MysDJ1szszJwsjUzKwMnWzOzMvh/61Qvyn/v\n1XcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1903da70b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from dermatrack.models import model_selection as metrics\n",
    "from sklearn.metrics import precision_score, accuracy_score, recall_score, confusion_matrix\n",
    "\n",
    "cm = metrics.plot_confusion_matrix(confusion_matrix(df['y_true'], df['y_pred']), classes=['rest', 'itch'],\n",
    "                          title='Confusion matrix, without normalization')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "custom": {
     "title": "precisionRecallCurve",
     "type": "metrics"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XuYZFV97vHv29XXmW5mBgYIzAwM\nd5kgqEy4PCaKB4PAEfAYj0K8YQhoEqI5MSbmJFEkMUY9xmMiJpBgVEQReJSMBMNRBBEDwvBwR0Du\nM9wvMz3Tc+mZ6f6dP9beVHVN9+6a7tp9qXk/z1NPV9Xee61Vq6v2b6+19l5bEYGZmdlY2qa7AGZm\nNrM5UJiZWSEHCjMzK+RAYWZmhRwozMyskAOFmZkVcqCY5SSdKemm6S5Hs0m6T9Jx46yzj6QBSZUp\nKlbpJD0u6c3Z8/MkfXO6y2TmQDENJHVJuljSE5LWS7pD0knTXa5GZDuyTdkO+jlJ/yapt9n5RMSv\nRsQN46zzZET0RsRQs/PPdtJbs8+5VtJ/STq22fnsLCR9TdI2SXvXvd+Uepb029nvaYOkqyTtWrDu\nKZLuzfL8L0nLapadKWkoW5Y/jtvR8rQaB4rp0Q6sAt4IzAP+Crhc0tJpLNOOOCUieoHXAb8G/GX9\nCkpm+/frO9nnXAhcD1wxzeVpOkntU5DHXOC3gH7g3aOsktfz7sBNwHclaQfS/1XgQuC9wJ7ARuAr\nY6x7EHAp8CFgPvB9YEVdPdycHYDkjxsaLUurmu0/5FkpIjZExHkR8XhEDEfE1cBjwJFjbSNpiaTv\nSnpB0kuSvjzGel+StErSOkm3S/qNmmVHSVqZLXtO0t9n73dL+maW7lpJt0nas4HP8RTwA+CwLJ0b\nJH1a0s9IP9b9Jc3LWk/PSHpK0t/UdhVJOlvSL7KW1f2SXpe9X9sFM1a5l0qK/EcuaW9JKyS9LOlh\nSWfX5HOepMslfSPL6z5Jy8f7jNnn3EbauSyStHtNmm+VdGfNkfDhNctG/X9JOkDSj7P3XpR0qaT5\njZSjnqTTsvzXSXpE0on1dVfz2b9ZV2dnSXoS+LGk/5R0bl3ad0l6e/b8VZJ+mNXrg5LeuYNF/S1g\nLXA+8P6xVoqIrcDXgV8BdtuB9N8NfD8iboyIAdKB19sl9Y2y7luAn0bETdn/9bPAItJBm43BgWIG\nyHbKBwP3jbG8AlwNPAEsJX2xLxsjuduA1wC7At8CrpDUnS37EvCliNgFOAC4PHv//aSWzRLSD/RD\nwKYGyr0EOBm4o+bt9wLnAH1Zeb8ObAMOBF4LnAD8brb9/wTOA94H7AKcCrw0SlZjlbvet4HVwN7A\nO4C/lXR8zfJTSfU2H1gBjBpsR/mcnVkZXwLWZO+9Dvgq8EFSnV1IOjLtGuf/JeAzWRkPJdX5eY2U\no65MRwHfAD6WfZ43AI/vQBJvzPJ/C+l7ckZN2suAfYH/yFoDP8zW2SNb7yvZUXze5XP3OHm9n/S/\nuQx4VX4wMMpn6gLOBFZHxIuSfj0LwmM9fj3b9FeBu/J0IuIRYAvpN7VdNtmj/vVhNe+9NgviD0n6\nK01Bq2vGiwg/pvEBdAA/Ai4sWOdY4AWgfZRlZwI3FWy7Bjgie34j8ClgYd06vwP8F3B4A+V9HBgg\nHSE+QWri92TLbgDOr1l3T2AwX569dwZwffb8WuAjBfm8eZxyLwWC1JW3BBgC+mqWfwb4Wvb8POBH\nNcuWAZsKPud5pJ3N2izdl4Djapb/E/DXdds8SNoBj/n/GiWftwF3jPG5zwO+OcZ2FwJfHK/u6tOp\nqbP9a5b3ARuAfbPXnwa+mj1/F+kIvD7vTzb4/d4HGAZeU/M//9IY9fw88GPgyB38DV0HfKjuvadq\n/181778q+6zHAZ2k1scw8OfZ8v2B/UgH0a8G7s+X7cwPtyimkVIf/iWkH8q5Ne//QNWBtHeTdoJP\nRGoqj5fmR7OunH5Ja0kthYXZ4rNIR1kPZN1Lb83ev4T0A75M0tOSPiepoyCbt0XE/IjYNyJ+PyJq\nWx+rap7vSwqEz+RHgaSdzB7Z8iXAI+N9poJy19obeDki1te89wTpaD73bM3zjUC3pHZJ766p7x/U\nrHN5RMwnBbx7Gdk1uC/w0doj3Ozz7E3B/0vSHpIuy7rh1gHfpPr/2RGN1t1YXvk/ZXX2H8Dp2Vun\nk7raIH3Oo+s+57tJ3UONeC/wi4i4M3t9KfDbdd+vy7Pv0x4R8d8i4vYd/CwDpBZprV2A9fUrRsQD\npBbOl4FnSHV/P6klSkQ8GhGPReoSvofUXfaOHSxPy3GTappIEnAxaSd0cqT+WQAi4qS6dY8F9pHU\nXhQslMYj/gw4HrgvIoYlrSFrakfEL4EzsgD1duBKSbtFxAbSEfunlAbUryEdHV88gY9WOx3xKlKL\nYuEY5V5F6koqTnCMctet9jSwq6S+mmCxD+nIcrz0L6W6Yxxt+YuSPgjcJulbEfFMVvZPR8Sn69cf\n5//1GVIdHR4RL0l6Gw12gdUpqrsNwJya16Pt1Ounjf428ElJNwI9pMH7PJ+fRMRvTqCMkLrs9pGU\nB+l2UlfdSaTuvzFl3+cfFKxyUkT8lNRle0TNdvsDXcBDo20UEVcCV2brzie1qG8bI49gZFfVTskt\niunzT6Q+4lPqjshHcyvp6OfvJM1VGnx+/Sjr9ZHGA14A2iV9gpojLUnvkbR7RAyTmvoAQ5LeJOnV\nWd/6OmArqbtlUrId6v8DviBpF0ltSoO5+cDhvwJ/IulIJQdK2rc+nbHKXZfXKlL32Wey+jmc1BIZ\nMwDs4Gd5gNTq+tPsrX8BPiTp6KzscyX992wAtej/1UfWdSdpEWmMYSIuBj4g6fisXhdJelW27E7g\ndEkdSgP2jRwRX0NqPZxPOgtpOHv/auBgSe/N0uuQ9GuSDh0vwSxgHgAcRRo3ew1pLOBbFAxq5yLi\npzHy7KP6x0+zVS8FTpH0G9mYyvnAd+tal7XlOlJSRenEhAtJA+EPZMtOUnYiR1affwX8+3hlbXUO\nFNMg2xl+kPTDebaum2k7ka4TOIU0IPwkqZn8rlFWvZZ0BPYQqdtlMyO7gk4E7pM0QBogPj0iNpOO\nOK8kBYlfAD8hdYk0w/tIfcH3k8ZLrgT2yj7XFaT+8G+RugmuIg3C1xur3PXOIPXBPw18j9SP/sMm\nfQ6AzwPnSNojIlYCZ5NaA2uAh0njReP9vz5FOq24n9Td892JFCQibgU+AHwxS+snpB09pJ3bAVm5\nPkWq3/HSG8zK8uba9bOd7Qmk7qinSd13nyUdsZN12416EgYpGPx7RNwTEc/mD9L/8K0quNZhR0TE\nfaQTMC4ljXP0Ab+fL8+6cv93zSZfIh1wPJj9Pbtm2fHA3ZI2kILnd4G/bUY5ZzNF+MZFZmY2Nrco\nzMyskAOFmZkVcqAwM7NCDhRmZlZo1l1HsXDhwli6dOl0F8PMbFa5/fbbX4yI3cdfc3uzLlAsXbqU\nlStXTncxzMxmFUlPTHRbdz2ZmVkhBwozMyvkQGFmZoUcKMzMrJADhZmZFXKgMDOzQqUFCklflfS8\npHvHWC5J/6B0b+O7NcbtEc3MbHqV2aL4Gml66LGcBByUPc4h3Z/BzMxmmNICRUTcCLxcsMppwDci\nuQWYL2mv8dLdsKFZJTQzs0ZM5xjFIkbeVGc1I+9v/ApJ50haKWnliy+uYXBw9AS3boWBAdiypell\nNTPbaU3nFB6j3Yd21LsoRcRFwEUABxywPCJgaAiefRaGsxs2Pv88bN4M69dDezu86U1QqZRVdDOz\nncd0BorVwJKa14tJt1psyMAAPPAAPPVUer11KwwOpr8RcMwx0Ns7cpuhIdi4MQWTSgV23x3afN6X\nmVmh6QwUK4BzJV0GHA30R8QzjWy4YQNs2wabNqVgsOee6f32dlizBn75S3jySfiVX4GeHnjuOXjp\npdTi2LAhbbd+PRx5JBw67i3izcx2bqUFCknfBo4DFkpaDXwS6ACIiH8m3bj8ZNJN6TeSbhQ/rgi4\n5ZYUFDZtSoGgu7u6vLs7tRJuvjk9X7IE1q1L3VQbN8L8+dDZCatXpxbG4CAsWpSWbdiQurI2boSl\nS2GPPbbPf8uWlG93dwpW27alYFXUzTU0lLYbHEzPN2zglXGW9etTefL01qxJy6RU1q6u9HxgAPbb\nDxYsSK/HMjyc8ujoSM8jUtmGhtLytrZquYeGqo/89eBgqtM8nfyxaVP6u9deI/MZGkrP87wiqunn\ndTI0lIL0vHnVVl6+7mjP169P9TF37si8Xn45BX8pvV63DhYurK5T/xgcTP/DvL7yfPLHli2pnjo6\nRtbpaM/r3+voKF7frJWUFigi4oxxlgfwBzuabltb2pG3taUf64IFI5f39cHy5WmH++ijaWfS1weL\nF6d127NPvMsucM89aWfS1ZV2YmvWpB2ilJbtt1/aMUppxzcwAP39aac3PJy227IF5sxJgWXhwvT+\nwEDKY3AwPV+7NnWJ5a2ZPGAMD6d0851OR0fagQ0MVAOQlPIYGoL77kvlWbAglaOvD3bbLa27cWPa\nbv36tH5tl1pbWzW/SiW9zgPeaDvY9vbqjh9SGTZvTtv19KT38sATNaNK9Tvitra0bb5T7uwcGcBq\nA1m+fV5WqfqA6rqdndVtIW2brzM0VM07/6xdXSPLl283PFwtX333Y55ee3u13kbT11fdfmgo/Y8h\n/U/23LOabv65hoer34NKZWQQe+ml9J3s6kqPnp6RQSqvp66ukZ+jNmCZlWXW3Y+iUoF99qnu8EfT\n3p7GH/KjzdGO8vbcM+1wn38eXngh/Xj33jsdxa5fD3fckXZujz1Wbb1UKulHmu8U5sxJ28+dC48/\nnn7Evb3VLq7aH3FESqejI7UUdtklbd/entbt7Ezb1+5c8rGUnh545hm4995U1gULUlDLW1O1O2tI\nO53u7pRmf3/acQ0NpYCV1wmknY2U8pg7d2SAyI+0OzpSubq74cUX01H90FBav6sr5ZG3iDo6Uhq1\nR/D5Tm54OAXufGcvVXfiW7dWg+TQUFo+MJACaqVSLceWLamu8nrMA3b+vejoSOVob0/1uHr19nWT\nfx/y4Jd/X2rfh2owzcuUB6BKJf1furqqLb/8s+YB8YUX0vchz3toaGQwGhhI6eQt4drWXqUyMvjV\nBwsp/d/zFmB3Nxx0UPWAZe7c6ufI/x+dndVAtW1bSi/Puz64Dw9Xg1Rt3nlQtZ2TYrRf0gx2yCHL\n4xvfWFkYKJpheDi1Rvr7qzv32qO8Wps3p53Ds8+mH2tvL+y6a3XHVYaIFBDmzKl2t9nMMVprqfa7\ns3VrCtyVSrUl19+fgsjWrdsfCA0OVg9W5s1L6a1dm4JAT08KFO3tI7v7OjqqLcjavOvLMpq8xZ4f\n7OQt3/b2FIzy7ta8VbV2bTVI77prNY3u7pGtw7xMNvUk3R4Ryyey7axrUUyVtrYUHObPH3/dfCxk\nyZLx120WaWTrwGaWsbqzch0dqdVba7QxsUbk40Jr1lRbBkND6eBl48ZqfpDWGx5OASbftrZ8/f0p\nELW1pQOgvOtz3br0W+jvr7YIpfTdz1sqtS2iPBjUB4VKJaWzcOHIsbO8hVnbTbd5c0q/tzct27Yt\ntcRr63LbtnSwVNuCsuZztZrNcvlOsj7QLBr18tXmybuq2tqq43HPPVc9+WDr1pHjVHmr6emnRx6A\n5S2cPEBI1ZNKKpVqV2Jb28iuzfyz177u66sGjq6ukeNGXV1p+82b01hfZ2da3te3fbo2kgOFmU1I\nPr4F6ci/u7v5rdytW9PYWH9/9SQQqAaewcG0wx8cTI+NG0eOreRnGc6dWx0Ly8tb29rp7EyBNg8Y\n+YkpfX2phdXTs3O3Vnbij25mM11HRzr632vcWeCK1bZonn02BZX8TMY8YPT3p6BSO5bS2VkNHm1t\n6fTs3t7qSR75tVyLFqVg0qrjLw4UZtby8tbPrrtWB9uLDAykMw3Xrq1eoNvRkbrWOjpGnlqdn1km\npec9PSl49PbCEUeUd0LLVHKgMDOr09ubTjseTT5+MjycAskLL6RHHjQGB9Pzzk546CE47LDUrTVv\nXho/mY1nKDpQmJntgLa26gwDu+ySrusaza23wqpV1etu8tOXFy9O2+2/fwomc+ZMXdknyoHCzKwE\nRx2V/r78cpp7bmAgdWPlF8Tee2/1rK0DD0wBZI89qqcuzyQOFGZmJaofF4mAu+8eeabW2rUpcHR1\nwdFHw8EHT195R+NAYWY2haQ0yF1r7Vp48MF0KvANN6Ruq7e8JZ2mOxOu73CgMDObZvPnp5bE0BBc\ne20aw7j66nSG1bJl8OpXT+8EkA4UZmYzRKUCJ58MjzySWhj5tRr33gunnLL9bNlTxYHCzGyGOeCA\n9IhIXVFDQ/C976VbPO+339SXZxae0WtmtnOQUnA45ph0AeCPfzw95XCgMDOb4Xp70ym0AwPpHjlT\nzYHCzGwWWLIkXbx3001Tn7cDhZnZLNDVla4C7++HJ56Y2rwdKMzMZonFi1OguOWWqc3XgcLMbJbo\n6xs5GeFUcaAwM5sl2trgDW9IV3Bfc80U5jt1WZmZ2WT19qY79q1fDz//+dTk6UBhZjbLHHtsGqu4\n666pyc+Bwsxslmlvh0MOSTPPrllTfn4OFGZms9C8eWma8scfLz8vBwozs1moqwu2boUNG8rPy4HC\nzGwWqlTSXFDr15eflwOFmdkslN+He+3a8vNyoDAzm4WkdJpsf3+6CK9MDhRmZrPUgQfCunXws5+V\nm48DhZnZLNXXB1u2lN/9VGqgkHSipAclPSzp46Ms30fS9ZLukHS3pJPLLI+ZWSvp6kp3wdu4sdx8\nSgsUkirABcBJwDLgDEnL6lb7S+DyiHgtcDrwlbLKY2bWaqR0a9RNm8rNp8wWxVHAwxHxaERsAS4D\nTqtbJ4BdsufzgKdLLI+ZWcupVMq/lqLMQLEIWFXzenX2Xq3zgPdIWg1cA/zhaAlJOkfSSkkr+/un\ncG5dM7MZrrs7jVOUGSzKDBQa5b2oe30G8LWIWAycDFwiabsyRcRFEbE8IpbPm7d7CUU1M5ud5sxJ\ngeLWW8vLo8xAsRpYUvN6Mdt3LZ0FXA4QETcD3cDCEstkZtZS5s9PU3mUeeZTmYHiNuAgSftJ6iQN\nVq+oW+dJ4HgASYeSAoX7lszMGtTenk6TLXMW2dICRURsA84FrgV+QTq76T5J50s6NVvto8DZku4C\nvg2cGRH13VNmZlZg/vx0imxZe8/2cpJNIuIa0iB17XufqHl+P/D6MstgZtbqFixIt0d94AE49NDm\np+8rs83MZrl582DzZnjuuXLSd6AwM5vlurvTxIAvvVRO+g4UZmazXHs77LJLuuNdGRwozMxawIIF\n8PLL5QxoO1CYmbWAhQvTOMVVVzU/bQcKM7MWsOeeaXLAF0q4Es2BwsysRSxbBgMDzb/4zoHCzKxF\n9PWl6TyaffaTA4WZWYvo6IChIXjyyeam60BhZtYi5sxJgaK/v7npOlCYmbWI9vYULAYGmpuuA4WZ\nWQvZd980mD083Lw0HSjMzFpIZyds2waPPtq8NB0ozMxaSF9fmnK8mQPaDhRmZi2kuzv93by5eWk6\nUJiZtZDOzjSgvX5989J0oDAzazE9PbBhQ/PSc6AwM2sxCxakAe2hoeak50BhZtZienpSkGjWKbIO\nFGZmLWjbtual5UBhZtaCtmyBdeuak5YDhZlZi+nuTi2K555rTnoOFGZmLaazEyRoa9Ie3oHCzKzF\ntLenrqf7729Oeg4UZmYtZs6c9HdwsDnpOVCYmbWgAw/0dRRmZlZASrdFbQYHCjOzFjQ0lMYpmtGq\ncKAwM2tBvb2waVNzWhUOFGZmLUhKrYlmXHTnQGFm1oJ6elLXUzOmG29vdEVJi4B9a7eJiBsnXwQz\nM2s2KXU/VSqTT6uhQCHps8C7gPuBfGgkgMJAIelE4EtABfjXiPi7UdZ5J3Belt5dEfHbjRbezMxG\n192dup3uvhuWLp1cWo22KN4GHBIRDV++IakCXAD8JrAauE3Sioi4v2adg4A/B14fEWsk7dF40c3M\nbCzd3enRjIvuGh2jeBTo2MG0jwIejohHI2ILcBlwWt06ZwMXRMQagIh4fgfzMDOzMey2W3PudNdo\ni2IjcKek64BX4lNEfLhgm0XAqprXq4Gj69Y5GEDSz0jdU+dFxH82WCYzMxvH5s0QMbk0Gg0UK7LH\njtAo79UXtx04CDgOWAz8VNJhEbF2RELSOcA5AHvuuc8OFsPMbOfU25taFJO96K6hQBERX5fUSdYC\nAB6MiPEu41gNLKl5vRh4epR1bsnSekzSg6TAcVtd/hcBFwEccsjyScZGM7OdQ09PGqOY7DhFQ2MU\nko4DfkkanP4K8JCkN4yz2W3AQZL2y4LM6WzfKrkKeFOWx0JSIHq04dKbmdmYhoZSt9Nkxyka7Xr6\nAnBCRDwIIOlg4NvAkWNtEBHbJJ0LXEsaf/hqRNwn6XxgZUSsyJadICk/7fZjEfHSxD+OmZnlurrS\nFB6TvX92o4GiIw8SABHxkKRxz4KKiGuAa+re+0TN8wD+OHuYmVkT5RfddXZOLp1GA8VKSRcDl2Sv\n3w3cPrmszcxsKkzVWU+/B/wB8GHS2Uw3ksYqzMxshtJo555OQKNnPQ0Cf589zMxsFim1RSHp8oh4\np6R72P4aCCLi8Mllb2ZmZclbFGV3PX0k+/vWyWVjZmZTbXg4TTO+Zs3k0im8jiIinsmevgisiogn\ngC7gCLa/eM7MzGaQrq706NjRmfrqNDop4I1Ad3ZPiuuADwBfm1zWZmZWtoh0A6PJaDRQKCI2Am8H\n/jEi/gewbHJZm5lZ2bZtm8JAIelY0vUT/5G91/Dd8czMbOpJMHcutE9yb91ooPgj0g2GvpdNw7E/\ncP3ksjYzszJVKtDfD6tWjb9ukUavo/gJ8JOa14+SLr4zM7MZqqMjPTZtmlw6411H8X8j4o8kfZ/R\nr6M4dXLZm5lZWSqVNNV42ZMC5nM7/Z/JZWNmZtOhtxc2bpxcGoWBIiLyif9WApsiYhhAUoV0PYWZ\nmc1gbW2Tn/Op0cHs64A5Na97gB9NLmszMytbV1e6QnsyGg0U3RExkL/Ins8pWN/MzGaAjg7YvHly\naTQaKDZIel3+QtKRwCTH0c3MbCqke2a3Nbq/306jl2H8EXCFpHx+p72Ad000UzMzmxq9vXmgqJQb\nKCLiNkmvAg4h3bjogYjYOtFMzcxs6lQqMMoVDg1rKMJImgP8GfCRiLgHWCrJU4+bmc1wlUo+11Nl\nwuc+NdoU+TdgC3Bs9no18DcTzdTMzKZGpZK6n2Bowk2KRgPFARHxOWArQERsInVBmZnZDNaM+2Y3\nGii2SOoh6+SSdAAwOPnszcysTBKsWwfQUZloGo2e9fRJ4D+BJZIuBV4PnDnRTM3MbOqkE2Mn3rYY\nN1BIEvAA6aZFx5C6nD4SES9ONFMzM5sa7e0wZw7A8ITHKMYNFBERkq6KiCOp3rTIzMxmiXTjoih9\nMPsWSb820UzMzGz2anSM4k3AhyQ9DmwgdT9FRBxeVsHMzGxmaDRQnFRqKczMbMYa7w533cCHgAOB\ne4CLI2KS90oyM7PZZLwxiq8Dy0lB4iTgC6WXyMzMZpTxup6WRcSrASRdDNxafpHMzKxZpuLK7Fdm\niHWXk5nZzmm8QHGEpHXZYz1weP5c0rrxEpd0oqQHJT0s6eMF671DUkhavqMfwMzMylXY9RQRE54b\nRFIFuAD4TdJss7dJWhER99et1wd8GPj5RPMyM7PyTPiORw04Cng4Ih6NiC3AZcBpo6z318DngEne\n1dXMzMpQZqBYBKyqeb06e+8Vkl4LLImIq4sSknSOpJWSVvb3v9D8kpqZtaipnGZ8IkYr3itzjUhq\nA74IfHS8hCLioohYHhHL583bvYlFNDOz8ZQZKFYDS2peLwaernndBxwG3JBNDXIMsMID2mZmzbV5\nM0D7hMecywwUtwEHSdpPUidwOrAiXxgR/RGxMCKWRsRS4Bbg1IhYWWKZzMx2Op2dk9u+tECRXXdx\nLnAt8Avg8oi4T9L5kk4tK18zM6uS8mnGh4cnmkajkwJOSERcA1xT994nxlj3uDLLYma2s4rsJtYT\n3b7MriczM5tmlQps3QozdYzCzMxmgJ4egKEJdz05UJiZtbCZfh2FmZm1AAcKMzMr5EBhZmaFHCjM\nzKyQA4WZWQvzYLaZmZXOgcLMzAo5UJiZWSEHCjMzK+RAYWZmhRwozMxaXJoUsOJJAc3MbHTd3TCZ\n+1E4UJiZWSEHCjMzK+RAYWZmhRwozMxa3GSn8XCgMDNrceme2RPnQGFm1uJ6e8Gnx5qZ2Zj23Rcm\n065woDAzs0IOFGZmVsiBwszMCjlQmJlZIQcKMzMr5EBhZmaFHCjMzKyQA4WZmRVyoDAzs0IOFGZm\nVqjUQCHpREkPSnpY0sdHWf7Hku6XdLek6yTtW2Z5zMxsx5UWKCRVgAuAk4BlwBmSltWtdgewPCIO\nB64EPldWeczMbGLKbFEcBTwcEY9GxBbgMuC02hUi4vqI2Ji9vAVYXGJ5zMxsAsoMFIuAVTWvV2fv\njeUs4AejLZB0jqSVklb297/QxCKamdl4ygwUo91TadRpbiW9B1gOfH605RFxUUQsj4jl8+bt3sQi\nmpnZeNpLTHs1sKTm9WLg6fqVJL0Z+AvgjRExWGJ5zMxsAspsUdwGHCRpP0mdwOnAitoVJL0WuBA4\nNSKeL7EsZmY2QaUFiojYBpwLXAv8Arg8Iu6TdL6kU7PVPg/0AldIulPSijGSMzOzaVJm1xMRcQ1w\nTd17n6h5/uYy8zczs8nzldlmZlbIgcLMzAo5UJiZWSEHCjMzK+RAYWZmhRwozMyskAOFmZkVcqAw\nM7NCDhRmZlbIgcLMzAo5UJiZWSEHCjMzK+RAYWZmhRwozMyskAOFmZkVcqAwM7NCDhRmZlbIgcLM\nzAo5UJiZWSEHCjMzK+RAYWZmhRwozMyskAOFmZkVcqAwM7NCDhRmZlbIgcLMzAo5UJiZWSEHCjMz\nK+RAYWZmhRwozMyskAOFmZkVcqAwM7NCDhRmZlao1EAh6URJD0p6WNLHR1neJek72fKfS1paZnnM\nzGzHlRYoJFWAC4CTgGXAGZIVUxzZAAAF5klEQVSW1a12FrAmIg4Evgh8tqzymJnZxLSXmPZRwMMR\n8SiApMuA04D7a9Y5DTgve34l8GVJiogoSnhwELZta36Bzcxa0ZYtAJrw9mUGikXAqprXq4Gjx1on\nIrZJ6gd2A16sXUnSOcA52astb3xj3yPlFHm22boAOtZMdylmBtdFleuiynWRSDCwz0S3LjNQjBa+\n6lsKjaxDRFwEXAQgaWXE+uWTL97sl+pis+sC10Ut10WV66JK0sqJblvmYPZqYEnN68XA02OtI6kd\nmAe8XGKZzMxsB5UZKG4DDpK0n6RO4HRgRd06K4D3Z8/fAfx4vPEJMzObWqV1PWVjDucC1wIV4KsR\ncZ+k84GVEbECuBi4RNLDpJbE6Q0kfVFZZZ6FXBdVrosq10WV66JqwnUhH8CbmVkRX5ltZmaFHCjM\nzKzQjA0Unv6jqoG6+GNJ90u6W9J1kvadjnJOhfHqoma9d0gKSS17amQjdSHpndl34z5J35rqMk6V\nBn4j+0i6XtId2e/k5OkoZ9kkfVXS85LuHWO5JP1DVk93S3pdQwlHxIx7kAa/HwH2BzqBu4Bldev8\nPvDP2fPTge9Md7mnsS7eBMzJnv/ezlwX2Xp9wI3ALcDy6S73NH4vDgLuABZkr/eY7nJPY11cBPxe\n9nwZ8Ph0l7ukungD8Drg3jGWnwz8gHQN2zHAzxtJd6a2KF6Z/iMitgD59B+1TgO+nj2/Ejhe0sSv\nUZ+5xq2LiLg+IjZmL28hXbPSihr5XgD8NfA5YPNUFm6KNVIXZwMXRMQagIh4forLOFUaqYsAdsme\nz2P7a7paQkTcSPG1aKcB34jkFmC+pL3GS3emBorRpv9YNNY6EbENyKf/aDWN1EWts0hHDK1o3LqQ\n9FpgSURcPZUFmwaNfC8OBg6W9DNJt0g6ccpKN7UaqYvzgPdIWg1cA/zh1BRtxtnR/QlQ7hQek9G0\n6T9aQMOfU9J7gOXAG0st0fQprAtJbaRZiM+cqgJNo0a+F+2k7qfjSK3Mn0o6LCLWlly2qdZIXZwB\nfC0iviDpWNL1W4dFxHD5xZtRJrTfnKktCk//UdVIXSDpzcBfAKdGxOAUlW2qjVcXfcBhwA2SHif1\nwa5o0QHtRn8j/x4RWyPiMeBBUuBoNY3UxVnA5QARcTPQDSycktLNLA3tT+rN1EDh6T+qxq2LrLvl\nQlKQaNV+aBinLiKiPyIWRsTSiFhKGq85NSImPBnaDNbIb+Qq0okOSFpI6op6dEpLOTUaqYsngeMB\nJB1KChQvTGkpZ4YVwPuys5+OAfoj4pnxNpqRXU9R3vQfs06DdfF5oBe4IhvPfzIiTp22QpekwbrY\nKTRYF9cCJ0i6HxgCPhYRL01fqcvRYF18FPgXSf+L1NVyZiseWEr6NqmrcWE2HvNJoAMgIv6ZND5z\nMvAwsBH4QEPptmBdmZlZE83UriczM5shHCjMzKyQA4WZmRVyoDAzs0IOFGZmVsiBwqyOpCFJd0q6\nV9L3Jc1vcvpnSvpy9vw8SX/SzPTNms2Bwmx7myLiNRFxGOkanT+Y7gKZTScHCrNiN1MzaZqkj0m6\nLZvL/1M1778ve+8uSZdk752S3SvlDkk/krTnNJTfbNJm5JXZZjOBpApp2oeLs9cnkOZKOoo0udoK\nSW8AXiLNs/X6iHhR0q5ZEjcBx0RESPpd4E9JVwibzSoOFGbb65F0J7AUuB34Yfb+Cdnjjux1Lylw\nHAFcGREvAkREPjnlYuA72Xz/ncBjU1J6syZz15PZ9jZFxGuAfUk7+HyMQsBnsvGL10TEgRFxcfb+\naHPh/CPw5Yh4NfBB0kR0ZrOOA4XZGCKiH/gw8CeSOkiTzv2OpF4ASYsk7QFcB7xT0m7Z+3nX0zzg\nqez5+zGbpdz1ZFYgIu6QdBdwekRckk1RfXM2S+8A8J5sptJPAz+RNETqmjqTdFe1KyQ9RZryfL/p\n+Axmk+XZY83MrJC7nszMrJADhZmZFXKgMDOzQg4UZmZWyIHCzMwKOVCYmVkhBwozMyv0/wH4Jelm\nxfhDFQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1904986e2e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import average_precision_score\n",
    "\n",
    "average_precision = average_precision_score(df['y_true'], df['prob1'])\n",
    "\n",
    "precision, recall, _ = precision_recall_curve(df['y_true'], df['prob1'])\n",
    "\n",
    "plt.step(recall, precision, color='b', alpha=0.2,\n",
    "         where='post')\n",
    "plt.fill_between(recall, precision, step='post', alpha=0.2,\n",
    "                 color='b')\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(\n",
    "          average_precision))\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Edit Metadata",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}