DFS（枝刈り込み）による統計的識別手法の最良組み合わせ抽出

statistical_identify_dfs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn.metrics as metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 1, ..., 0, 1, 1],\n",
       "       [1, 1, 1, ..., 1, 1, 1],\n",
       "       [1, 1, 0, ..., 0, 1, 1],\n",
       "       ..., \n",
       "       [0, 1, 1, ..., 1, 0, 0],\n",
       "       [0, 0, 0, ..., 1, 1, 1],\n",
       "       [0, 0, 0, ..., 0, 1, 1]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u = np.random.randint(2, size=(100000, 70))\n",
    "y = u[:, 0]\n",
    "X = u[:, 1:]\n",
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def auc(y, X):\n",
    "    pred = np.mean(X, axis = 1)\n",
    "    fpr, tpr, thresholds = metrics.roc_curve(y, pred)\n",
    "    return metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4966415065603767"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_auc = auc(y, X)\n",
    "best_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dfs(idx, first_cmb, first_auc):\n",
    "    best_cmb = first_cmb\n",
    "    best_auc = first_auc\n",
    "    if len(first_cmb) == 0:\n",
    "        return [best_cmb, best_auc]\n",
    "    for i in range(idx, len(first_cmb)):\n",
    "        better_cmb = np.delete(first_cmb, i)\n",
    "        better_X = np.take(X, better_cmb, axis = 1)\n",
    "        better_auc = auc(y, better_X)\n",
    "        if better_auc >= best_auc:\n",
    "            best_cmb, best_auc = dfs(i, better_cmb, better_auc)\n",
    "    return [best_cmb, best_auc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 4s, sys: 43.2 s, total: 1min 48s\n",
      "Wall time: 1min 48s\n"
     ]
    }
   ],
   "source": [
    "%time best_cmb, best_auc = dfs(0, list(range(X.shape[1])), 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 1,  4,  6,  9, 10, 13, 14, 16, 17, 18, 26, 29, 30, 33, 35, 36, 42,\n",
       "        44, 45, 46, 51, 52, 53, 59, 60, 62, 67]), 0.50798529079632249)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_cmb, best_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcTeUfwPHP1zD2LSRLRZF9bUIp6RfRhjYpKRIpSkrR\nolLSniKRpD0qJS3Kln0fkWWEsW8xSLKbme/vj+fMdI1ZLubOvXfm+3697mvuPee593zPvXfO9z7P\nc87ziKpijDHGAOQKdgDGGGNChyUFY4wxySwpGGOMSWZJwRhjTDJLCsYYY5JZUjDGGJPMkoLxm4i0\nF5FJwY4jlIjIARG5IAjbrSAiKiK5s3rbgSAiK0Wk6Wk8z76TmcySQpgSkY0ictg7KP0lIh+LSKFA\nblNVv1DVawK5DV8icpmI/CYi/4rIPyLyo4hUz6rtpxLPdBG5z3eZqhZS1fUB2t5FIvKNiOz29n+Z\niDwqIhGB2N7p8pJTpTN5DVWtoarTM9jOSYkwq7+TOYElhfB2o6oWAuoC9YAngxzPaUnt166IXApM\nAsYDZYGKwB/AnED8Mg+1X9wiciGwANgC1FLVosBtwMVA4UzeVtD2PdTedwOoqt3C8AZsBJr5PH4N\n+NnncV7gDWAzsBMYDuT3Wd8aWArsB9YBLb3lRYEPgR3ANmAAEOGt6wjM9u4PA95IEdN44FHvflng\nWyAO2AA87FPueWAs8Lm3/ftS2b9ZwHupLP8F+NS73xTYCjwF7Pbek/b+vAc+z+0D/AV8BhQHfvJi\n/tu7X94r/xKQABwBDgDvessVqOTd/xgYCvwM/Is7qF/oE881wGrgH+A9YEZq++6V/dz380xlfQVv\n2/d4+7cbeNpnfQNgHrDP+yzfBSJ91ivQHVgLbPCWvYNLQvuBxcAVPuUjvPd5nbdvi4FzgZneax30\n3pfbvfI34L5f+4C5QO0U390+wDLgKJAbn++zF3u0F8dO4C1v+WZvWwe826X4fCe9MjWAycBe77lP\nBft/NdxuQQ/Abqf5wZ34T1QeWA6847N+EPADcBbul+WPwMveugbegak5rrZYDqjqrRsHvA8UBM4G\nFgL3e+uS/wGBJt4BRLzHxYHDuGSQyztoPAtEAhcA64EWXtnngeNAG69s/hT7VgB3AL4qlf3uBOzw\n7jcF4oG3cAngSu/gVMWP9yDpua96z80PlABu8bZfGPgG+N5n29NJcRDn5KSwx3t/cwNfAGO8dSW9\ng9zN3rqe3nuQVlL4C+iUzudfwdv2B17sdXAH2Gre+ouBRt62KgCrgEdSxD3Ze2+SEuVd3nuQG3jM\niyGft+5x3HesCiDe9kqkfA+8x/WAXUBDXDK5B/d9zevz3V2KSyr5fZYlfZ/nAR28+4WARin2ObfP\ntjry33eyMC4BPgbk8x43DPb/arjdgh6A3U7zg3P/RAdwv9oUmAoU89YJ7uDo+yv1Uv77Rfg+MCiV\n1yztHVh8axR3ANO8+77/gIL75dbEe9wF+M273xDYnOK1nwQ+8u4/D8xMZ9/Ke/tUNZV1LYHj3v2m\nuAN7QZ/1XwP9/HgPmgLHkg56acRRF/jb5/F0Mk4KI33WXQf86d2/G5jns05wSTWtpHAcr/aWxvqk\nA2R5n2ULgXZplH8EGJci7v9l8B37G6jj3V8NtE6jXMqkMAx4MUWZ1cCVPt/de1P5PiclhZlAf6Bk\nGvucVlK4A1gSyP+7nHCz9rzw1kZVp4jIlcCXuF+j+4BSuF+7i0UkqazgfrWB+4U2IZXXOx/IA+zw\neV4u3MHrBKqqIjIG9484E7gT1+SR9DplRWSfz1MicE1CSU56TR9/A4lAGeDPFOvK4JpKksuq6kGf\nx5twtZWM3gOAOFU9krxSpACudtESV/MBKCwiEaqakE68vv7yuX8I90sXL6bkffbev63pvM4e3L6e\n1vZE5CJcDSoK9z7kxtXefJ3wGYhIb6CzF6sCRXDfKXDfmXV+xAPu879HRB7yWRbpvW6q206hM/AC\n8KeIbAD6q+pPfmz3VGI0abCO5mxAVWfgfqW+4S3ajWvKqaGqxbxbUXWd0uD+IS9M5aW24GoKJX2e\nV0RVa6Sx6dHArSJyPq528K3P62zweY1iqlpYVa/zDTud/TmIa0K4LZXVbXG1oiTFRaSgz+PzgO1+\nvAepxfAYrnmkoaoWwTWRgUsm6cbshx24GpB7QZepyqddnCm4pqzTNQyXUCt7+/IU/+1HkuT9EZEr\ngCdw729xVS2Ga2JMek5a35nUbAFeSvH5F1DV0altOyVVXauqd+CaL18FxnqfcUbv/xZcU6U5A5YU\nso+3geYiUkdVE3FtzYNE5GwAESknIi28sh8CnUTkahHJ5a2rqqo7cGf8vCkiRbx1F3o1kZOo6hLc\nwXckMFFVk2oGC4F/RaSPiOQXkQgRqSkil5zC/vTF/dp8WEQKi0hxERmAawLqn6JsfxGJ9A5sNwDf\n+PEepKYwLpHsE5GzgOdSrN/J6R90fgZqiUgb74yb7sA56ZR/DrhMRF4XkXO8+CuJyOciUsyP7RXG\n9WEcEJGqwAN+lI/HdbLnFpFncTWFJCOBF0Wksji1RaSEty7l+/IB0E1EGnplC4rI9SLi11lTInKX\niJTyPsOk71SiF1siaX8GPwFlROQREcnrfW8a+rNN8x9LCtmEqsYBn+I6d8Gd3RELzBeR/bhfnlW8\nsgtxHbaDcL8GZ+Cq/ODaviOBGFwzzljSb8b4Emjm/U2KJQF3cK6LO/MoKXEUPYX9mQ20wHXM7sA1\nC9UDLlfVtT5F//Li3I7r2O2mqklNTmm+B2l4G9dpuxuYD/yaYv07uJrR3yIy2N998fZnN67m8xqu\naag67gybo2mUX4dLgBWAlSLyD64mFo3rR8pIb1yT3r+4g/RXGZSfiNvfNbj3+ggnNvG8heuvmYRL\nNh/i3itwfUSfiMg+EWmrqtG4PqZ3cZ9NLK7t318tcft8APeet1PVw6p6CHcW2BxvW418n6Sq/+JO\nnrgR971YC1x1Cts1/HfmiDFhx7sC9nNVTa8ZJiSJSC7cKbHtVXVasOMxJonVFIzJIiLSQkSKiUhe\n/mvjnx/ksIw5gSUFY7LOpbizY3bjmjjaqOrh4IZkzIms+cgYY0wyqykYY4xJFnYXr5UsWVIrVKgQ\n7DCMMSasLF68eLeqlsqoXNglhQoVKhAdHR3sMIwxJqyIyCZ/ylnzkTHGmGSWFIwxxiSzpGCMMSaZ\nJQVjjDHJLCkYY4xJZknBGGNMMksKxhhjkllSMMYYk8ySgjHGmGSWFIwxxiSzpGCMMSaZJQVjjDHJ\nLCkYY4xJFrCkICKjRGSXiKxIY72IyGARiRWRZSJSP1CxGGOM8U8gawofAy3TWX8tUNm7dQWGBTAW\nY4wxfghYUlDVmcDedIq0Bj5VZz5QTETKBCoeY4wJFdu3w8yZwY4idcHsUygHbPF5vNVbdhIR6Soi\n0SISHRcXlyXBGWNMplNlTr9fqV1Lad8ejh0LdkAnC4uOZlUdoapRqhpVqlSGs8kZY0zIiVu0kT/O\naUHjAddyR7FfmDwZIiODHdXJgpkUtgHn+jwu7y0zxphsQ+MTmH/nYPI3qMkFu+bxa6v3eCumJVWr\nBjuy1AUzKfwA3O2dhdQI+EdVdwQxHmOMyVTr1sGssrfTaHRPVhRvwq7fVtJy/APkyRu6jTS5A/XC\nIjIaaAqUFJGtwHNAHgBVHQ5MAK4DYoFDQKdAxWKMMVkp/vBxBg+GZ/rn4Xo6Ed+pDU0/aE+uCAl2\naBkKWFJQ1TsyWK9A90Bt3xhjgmHFJ4uJ7HYvu4/czjWtn+LtoddTLtVTaEJT6NZhjDEmjOzbcZhJ\n9ftQtWNDihyL44Y+NRk3jrBKCGBJwRhjzogqTBqwgD3n1uGaJa+xqEYnCm6M4bJXWiGh31p0koA1\nHxljTHYXGwvdu8PeSbkYmxfWvjuFS7tdHeywzoglBWOMOUVHj8J3XSaw/ssFzCvQn4FDLqF811VE\nREYEO7QzZknBGGNOwZzxu9lzdy/u2P85W4rU4N7f+1DmwgJA+CcEsD4FY4zxy187lCFNvqZym+pc\nu38MsXc+y7m7FnsJIfuwmoIxxqQjPh6GDoUhz+xi6YF72VemGvHjplCpYe1ghxYQVlMwxpg0zJ6l\n9Kw8gUceUSo1Ls3ecTMpv3ke+bNpQgBLCsYYc5KdO+HxW9ZztEkzhm68njlP/8Ivv8B5bepD7uzd\nwJK9984YY05BfDy8/14C2/oMpv+Rp8kVmZujrw/nsh4tIQyvOTgdVlMwxhhg7ly45BIo0/M2Bh55\nlMQr/0e+dTHkffh+yJVzDpU5Z0+NMSYVu3ZBl3uOcWXj4+zeDaWfug/9/AsKTfsRypcPdnhZzpqP\njDE5UkICvP8+jO2ziMEH7uXWK9rReMLTFCp0XbBDCyqrKRhjcpz586FJ1CEOde/N5AONqFL6b1r0\nqUuhQsGOLPispmCMyTHi4uDJJ2Hlh/P4MqIDFVmHdr2fiNdehaJFgx1eSLCkYIzJ9hIS4IMP4Kmn\n4N9/4c27cnPegtzw/m/IVVcFO7yQYknBGJOtxcRA585w1vyfefv8BUTNfoHq1S+BhJUQkT3GK8pM\n1qdgjMmWjh2DF16A5nXjeOz39vzMDXQo8j3VKxxyBSwhpMqSgjEm21m0CC6ur8Q8N4ZVuapzi34D\n/fsj0dFQIHsNYJfZLCkYY7KNQ4egd29o1Agi9uzi83z3UaTOBcjvv8Ozz0JkZLBDDHmWFIwx2cJv\nv0HtmomsfvNHutynzPizNLnnzXaXKtesGezwwoZ1NBtjwtq+fa52MP3DWD7P34VGTIfWP0PR66Bu\n3WCHF3aspmCMCVvjxkGtavGcNeoNVuWuRcPIJTByJFx7bbBDC1tWUzDGhJ2//oKHHoKxY2Fqkdv4\nn34P17eG996DsmWDHV5Ys6RgjAkbqvDJJ9C311EOHMrFwIF5aFKzKxxqB23bguSQ8a0DyJKCMSYs\nbNwIXbvCP5MXMLdAZwp3a0epJ58BrKkoM1mfgjEmpCUkwDvvwCXVD9Jq+qPMl0upeNY/lGpRP9ih\nZUtWUzDGhKyVK90QFbkWzGVZ/g6UOb4eHngAXnkFihQJdnjZUkBrCiLSUkRWi0isiPRNZX1REflR\nRP4QkZUi0imQ8RhjwkPSEBX16kFsLPR7IZJzzo+E6dNdZ7IlhIAJWE1BRCKAoUBzYCuwSER+UNUY\nn2LdgRhVvVFESgGrReQLVT0WqLiMMaFt4UJXO6i44ge+rr6AxtNfolSpKHhqhY1XlAUCWVNoAMSq\n6nrvID8GaJ2ijAKFRUSAQsBeID6AMRljQtTBg/Doo9Cq0S4GrGvHD7SmTZ6fKVXQBrDLSoFMCuWA\nLT6Pt3rLfL0LVAO2A8uBnqqamPKFRKSriESLSHRcXFyg4jXGBMnUqVCrprJr0OfERlajVcI4GDDA\njWxnA9hlqWCffdQCWAqUBeoC74rISY2FqjpCVaNUNapUqVJZHaMxJkD+/ts1FTVrBufITj7O/wCF\n6ldBli6Fp5+GPHmCHWKOE8iksA041+dxeW+Zr07Ad+rEAhuAqgGMyRgTAlTh66+hRrVE9n48nj5P\nKFNXnkPu+XNg1iyoVi3YIeZYgTwldRFQWUQq4pJBO+DOFGU2A1cDs0SkNFAFWB/AmIwxQbZpE3Tv\nDmt+XsOPhbpwceJMuPJnyH8d1K4d7PByvIDVFFQ1HugBTARWAV+r6koR6SYi3bxiLwKXichyYCrQ\nR1V3ByomY0zwxMfDW2+5AezqTnqNmNx1qJ97GYwaZQPYhZCAXrymqhOACSmWDfe5vx24JpAxGGOC\nb/FiN0TF77/DvNK30GjnD3DTTTB0KJQpE+zwjA+7otkYEzAHDkC/fjD8naOUOlv4+utIGhZ6AA52\ngFtusQHsQpAlBWNMQPz0k+s7KLd5LuuLdaZY5zvJf1s/oGWwQzPpCPYpqcaYbGbHDrjtNmh34wEG\nHOjJHLmcMkUOkb9Jg2CHZvxgScEYkykSE2H4cKhaFfaMn83WYjXpsHcw0r07rFgBLVoEO0TjB7+S\ngohEikilQAdjjAlPK1bAFVe4AUyjouDjr/JTrFwhd83BkCFQuHCwQzR+yjApiMj1uCEoJnuP64rI\nuEAHZowJfYcPuwuP69WDC5eNY9kNTzFlCpx308WwbBlcfnmwQzSnyJ+awgtAQ2AfgKouBazWYEwO\nN3Wqu9bsw4F/MafcbXx64GZqbfsVOXLYFchlrdPhyJ9P7biq7kuxTAMRjDEm9O3eDffcA82aKW32\nf8qWwtVp8NePMHAgLFgA+fMHO0RzBvxJCqtEpC2QS0QqisggYH6A4zLGhBhV+OQT15H85Zfwcs+d\nvHaoO3lqV4elS+HJJ20Au2zAn6TQA7gYSAS+A44CPQMZlDEmtKxd60Yy7dQxka4lv2PpEqXv2+cg\nc+fCzJkuU5hswZ+k0EJV+6hqPe/WF7CBSozJAY4dg5deglq1YP/CP9l2YRMGrr6FGlt+dQVq1bK+\ng2zGn0/zmVSWPZ3ZgRhjQsucOVC/Pjz/zHE+qjyQhcfqUGZvjGtDamlXJWdXaQ5zISItcNejlxOR\nt3xWFcE1JRljsqF9+1z3wPDhcN55sLXBLZRe+CPceiu8+y6ULh3sEE0ApTf20S5gBXAEWOmz/F+g\nbyCDMsZkPVUYOxYefhj+2XmE3g/n4rmXIik0twcc6Ag33xzsEE0WSDMpqOoSYImIfKGqR7IwJmNM\nFkua+Obnn6HTRXN4L29n8pW4Ewo9C9fY6PY5iT99CuVEZIyILBORNUm3gEdmjAm4pIlvqleH6Gn/\n8vvlD/Hh2ivIJ0fh0kuDHZ4JAn+SwsfAR4Dgzjr6GvgqgDEZY7LA4sXQsCE89hj0qDOLrcVrUm/O\nUOShh2D5cmjePNghmiDwJykUUNWJAKq6TlWfwU5JNSZsHTgAjz4KDRrA9u3wzTfwypCC5C5eBGbP\nhnfegUKFgh2mCRJ/Jtk5KiK5gHXe3MrbABvy0JgwlDTxzebNyvvNv+XuaovId+urQH344w+75sD4\nVVPoBRQEHgYaA12AewMZlDEmc+3YAW3bwo03QsV8O9jd5Ba6Tr6NfHOmuqFOwRKCAfxICqq6QFX/\nVdXNqtpBVVsBGwMfmjHmTCUmwogRUK0a/DBe+fGmUUzbVZ0SC3+B116D+fNtADtzgnSTgohcIiJt\nRKSk97iGiHwKLMiS6Iwxp+3PP6FpU7j/fndlcsy0ndww5RGkdm3XVPT445Dbpmk3J0ozKYjIy8AX\nQHvgVxF5HpgG/AFclCXRGWNO2dGj0L8/1KkDMcsTmPrgWKZOUS647ByYNw+mTYOL7F/YpC69nwmt\ngTqqelhEzgK2ALVUdX3WhGaMOVWzZ0PXrrBqFfS+fhUDd3Ymz3vz4IYJcO21UKNGsEM0IS695qMj\nqnoYQFX3AmssIRgTmvbtg27d3DzJxw4eZ3WHAbw+uS55NqyBzz+3AeyM39KrKVwgIt959wWo6PMY\nVbWBUIwJMlX49lt46CHYtctdiPbKypvI/dnPcPvtMHgwnH12sMM0YSS9pHBLisfvBjIQY8yp2bIF\nevSAH36ARnUO8/O4COo3ioRJD0O3LtC6dbBDNGEovQHxpp7pi4tIS+AdIAIYqaqvpFKmKfA2kAfY\nrapXnul2jcnOEhLgvffgqafcKadjHphB28n3IRPvgkbP2QB25owE7GoVEYkAhuKGxKgO3CEi1VOU\nKQa8B7RS1RrAbYGKx5jsYPlyaNzYDW/dvOF+drR5gNuHNUUSE12HgjFnKJCXMDYAYlV1vaoeA8bg\nzmjydSfwnapuBlDVXQGMx5iwdfiwqxnUrw/r18OkZ2by7eoaFBkzwg1ktGwZ/O9/wQ7TZAN+JwUR\nyXuKr10Odxprkq3eMl8XAcVFZLqILBaRu9PYdlcRiRaR6Li4uFMMw5jw9ttvULs2vPwy3HWXO920\n+c2FkRIlYO5cePNNKFgw2GGabCLDpCAiDURkObDWe1xHRIZk0vZzAxcD1wMtgH4ictJVNao6QlWj\nVDWqVKlSmbRpY0Lbnj3QqRNcfTWgyvJnvuKjUk9QogRQrx4sWeLGvjYmE/lTUxgM3ADsAVDVP4Cr\n/HjeNuBcn8flvWW+tgITVfWgqu4GZgJ1/HhtY7ItVfjiC6ha1V1iMLDHdv6s2oaaA9rB9On/DWAn\nEtQ4TfbkT1LIpaqbUixL8ON5i4DKIlJRRCKBdsAPKcqMBy4XkdwiUgBoCKzy47WNyZY2bHAXHt91\nF1xQUdnw9Eie/Kw6EVMnwRtvuOYiG8DOBJA/o2FtEZEGgHpnFD0EZDgdp6rGi0gPYCLulNRRqrrS\nm5MBVR2uqqtE5FdgGZCIO211xenujDHhKj7ezW3z7LNuBOvBg+HBm3cSUe1R17s8ciRUqhTsME0O\nIKqafgGRs3FNSM28RVOAHl5zT5aLiorS6OjoYGzamIBYvBi6dHFdBK1vSGBky7GUfLCtax5atQqq\nVLG5DswZE5HFqhqVUTl/vmnxqtpOVUt6t3bBSgjGZCcHD7phKRo0cJPgTHxzBeN2XUbJHu3g119d\noWrVLCGYLOVP89EiEVkNfIW7puDfAMdkTLb3yy/wwAOwaRM8eN8x3ij5Mvn7vgRFi8KXX9oAdiZo\n/Jl57UJgAO7U0eUi8r2ItAt4ZMZkQzt3wp13wnXXuf7imTNh6PabyP/K83DbbRATA3fcYWcWmaDx\nq16qqnNV9WGgPrAfN/mOMcZPqvDRR6416NtvYcBTh1i64KgbmaJXL/jxR3ceql2HY4LMn4vXColI\nexH5EVgIxAGXBTwyY7KJtWvdBWj33uvmuFkzYjpPf1WbvG8OdAWaNYMbbghukMZ4/KkprAAaAa+p\naiVVfUxVbY5mYzJw7BgMHAi1asHvv8Oot/9hZrX7Ob+jd+3nVf5cA2pM1vKno/kCVU0MeCTGZCPz\n57vTTFesgFtvheHtplOi513uNKPevd0kygUKBDtMY06SZlIQkTdV9THgWxE56WIGm3nNmJPt3+9G\nM33vPShXDsaPh1atgKXF3Axo48bBJZcEO0xj0pReTeEr76/NuGaMH8aPh+7dYft26NFdebXuaPLP\nWAyt3oS6dd1VanZWkQlxafYpqOpC7241VZ3qewOqZU14xoS+7dvhllugTRs46yxY/P0WBm+4kfz3\ntXdjFdkAdiaM+NPRfG8qyzpndiDGhBtV+OADd5rpzz/DwAGJLOn2PvXuqgHTpsGgQTB7tg1gZ8JK\nen0Kt+NGNq0oIt/5rCoM7At0YMaEsk2b4L77YMoUaNoURoyAyoV3wUWPu3ErRoyACy4IdpjGnLL0\n+hQW4uZQKI+baznJv8CSQAZlTKhSdcf73r3d/WFD4ula/BtyVWoHcg4sWgQXXWRNRSZspZkUVHUD\nsAE3KqoxOd7Gja52MHWqmw75097LKPdsZ4iOhhLF3XhFVaoEO0xjzkiafQoiMsP7+7eI7PW5/S0i\ne7MuRGOCKzERhg93F6EtWAAjhhxlSuPnKNfqYteO9NVX0KJFsMM0JlOk13yUdLllyawIxJhQtGGD\nqx389psbqmLkSKjwQBs3tPVdd8Hbb+MmTTYme0jvlNSkq5jPBSJUNQG4FLgfKJgFsRkTNImJ7gK0\nWrVg4UL4cPBBJv90lAoVcB0KP/8Mn31mCcFkO/6ckvo9birOC4GPgMrAlwGNypgg2rDBjVHXvTtc\ndhnEvj+VewfVQga+5ApcfbUb+9qYbMifpJCoqseBm4EhqtoLKBfYsIzJeomJMHSoqx1ER8PHb+9j\n4rn3Ubp9M8idG5o3D3aIxgScPwPixYvIbUAHoI23LE/gQjIm661f74a2njEDrrkGPu00jdKPtodd\nu6BvX3j2WbsIzeQI/l7RfBVu6Oz1IlIRGB3YsIzJGomJ8O67rnawZInrSP71Vyhd7Sw3ot2CBfDy\ny5YQTI6RYU1BVVeIyMNAJRGpCsSq6kuBD82YwFq3ztUOZs6EFtcoX1z7OSVWLgF5C+rUcT3MdhGa\nyWH8mXntCiAW+BAYBawRkcaBDsyYQElMhMGDoXZtWLoUxry2mV8irqdEr7tdzcAGsDM5mD99CoOA\n61Q1BkBEqgGfAVGBDMyYQIiNdbWDWbPg2haJfNHkfYq/8IQbs2LwYHjwQYiICHaYxgSNP0khMikh\nAKjqKhGJDGBMxmS6xEQYMgSefBIiI2HUKOjYchdStS9ceqkb0KhChWCHaUzQ+ZMUfheR4cDn3uP2\n2IB4JoysXetqB7Nnww0t4/nk2jGc1bH9fwPYVa5sTUXGePw5+6gbsB54wrutx13VbExIS0hwo1DU\nqQPLl8P4/kv5YVdDzurZASZOdIVsRFNjTpBuUhCRWkBLYJyqtvJur6vqEX9eXERaishqEYkVkb7p\nlLtEROJF5NZTC9+Y1K1ZA1deCb16QYsrj7Dl7qdp9UIUsm0bjB3rRjQ1xpwkvVFSn8INcdEemCwi\nqc3AliYRicDNw3AtUB24Q0Sqp1HuVWDSqby+MalJSIC3vDNKV66ETz6B77QNhYcMdAPYxcS4uTON\nMalKr0+hPVBbVQ+KSClgAu6UVH81wF3TsB5ARMYArYGYFOUeAr4FLjmF1zbmJKtXu76DuXPh1pYH\neOe9PJStmBfO7eNVGWx4a2Mykl7z0VFVPQigqnEZlE1NOWCLz+OtpBgzSUTKATcBw9J7IRHpKiLR\nIhIdFxd3imGY7C4hAd58E+rWdRWByY9P4utVNSk7aoArcNVVlhCM8VN6NYULfOZmFuBC37maVfXm\nTNj+20AfVU2UdDr7VHUEMAIgKipKM2G7Jpv480/o1Anmz4d2Lf7mw6KPUuD1j90MaJYIjDll6SWF\nlA2v757ia2/DzcWQpLy3zFcUMMZLCCWB60QkXlW/P8VtmRwmIQEGDYJnnoECBWBy36lc/fFdSFyc\nuxjh2WchX75gh2lM2ElvjuapZ/jai4DK3gB624B2wJ0ptlEx6b6IfAz8ZAnBZMS3dtCqlZsqs8yu\nkvDbeTBhAtSrF+wQjQlb/ly8dlpUNV5EegATgQhglKquFJFu3vrhgdq2yZ6S+g6efRYKFlDm3v8p\njSJ/R8rfGhyaAAAXhUlEQVS8A2XquCxh1xwYc0YClhQAVHUC7qwl32WpJgNV7RjIWEx4W7XK1Q4W\nLIAuzTcy5Nj95H1/Elx+uRvALn9+SwjGZAK/zygSkbyBDMSY1MTHw6uvuhah2DWJLLpnCO/PrUne\nxXPdRAgzZthcB8ZkIn+Gzm4gIsuBtd7jOiIyJOCRmRwvJgYaN3YTn113HayasYuocc8gV1wBK1a4\nSZRzneqZ0saY9PjzHzUYuAHYA6Cqf+BmYjMmIOLj4ZVXXO1gU+xx5j/4Kd+OVUrVOgcWL3adyeef\nH+wwjcmW/EkKuVR1U4plCYEIxpgVK9xI1k8+CQ9f/jtbyjag4Xv3IFMmuwKVKlnfgTEB5E9S2CIi\nDQAVkQgReQRYE+C4TA4THw8DB8LFF8NfGw4T0+ZJXp/RgDy7/4LvvoNrrgl2iMbkCP6cffQArgnp\nPGAnMMVbZkymWL7cnVm0eDG0bQuf7WpD5PeT3EBGb7wBxYsHO0RjcowMk4Kq7sJdeGZMpjp+3J1Z\n9MILUK7Iv3z7RR5uvjMfTH8SnuoNzZsHO0RjcpwMk4KIfACcNN6QqnYNSEQmR1i2DDp2hCVL4OUm\nv/D4uvuJiLkbGABNmwY5OmNyLn/6FKYAU73bHOBs4GgggzLZ1/HjrmZw8cVwaMseNl55N31nXkdE\nkUJw/fXBDs+YHM+f5qOvfB+LyGfA7IBFZLKtpUtd38HSpfDS/6bQd3l7cs3ZC/36wdNPQ167PtKY\nYDudYS4qAqUzOxCTfR075s4seuklKFECxo2DNheWhvsvhGGT3DRpxpiQ4E+fwt/816eQC9gLpDnf\nsjG+lixxfQfLlikjGn1Eh5pLyNdmCFAL5syxaw6MCTHpJgVxEx3U4b95EBJV1Sa5MRk6dgwGDICX\nX4Z6xTawq05XSs2fApFNbAA7Y0JYuh3NXgKYoKoJ3s0SgsnQ4sUQFQUvvZjAR/XeYcGhmpRavwCG\nDYNp02wAO2NCmD9nHy0VEZu1xGTo6FE3E1rDhrBnD/z6aRx3rXkOufJKWLkSunWzAeyMCXFpNh+J\nSG5VjQfqAYtEZB1wEDdfs6pq/SyK0YSB6GjXd7B65XGGN/6CW8bfTfES50Dj36FiRWsqMiZMpNen\nsBCoD7TKolhMGDp6FPr3h9deg2vOimZuxc4UmbMMFpd14xVdcEGwQzTGnIL0koIAqOq6LIrFhJmF\nC911BxtiDvFjredpufJNJM858P33NoCdMWEqvaRQSkQeTWulqr4VgHhMGDhyBJ5/Hl5/HcqWhc31\n2lByyWTo0sVVGYoVC3aIxpjTlF5SiAAK4dUYjAE3R3LHjrDtz/107RjJK2/no+jSpyGhL/zvf8EO\nzxhzhtJLCjtU9YUsi8SEtMOH4bnn4M03ocNZPzOsZDfyl+kARQfClVcGOzxjTCbJsE/BmHnzXN/B\n7tW7mVvpERrGfgE1akDr1sEOzRiTydI7afzqLIvChKTDh6F3b2jcGKL2TmJ70Wo03PS1qzL8/ru7\nIMEYk62kWVNQ1b1ZGYgJLXPnutrBmjXumrM37ilDZO8q7qrkWrWCHZ4xJkDs8lJzgkOH4NFH4fLG\nSpvdI9ncugfDhkHBRrVg1ixLCMZkc6czdLbJpmbPdtMiJ6xdx6pyXaiybRr809Sdg5ovn12VbEwO\nYDUFw6FD8Mgj0PSKBO7Z/SZr8taiyr+LYcQImDrVJQRjTI4Q0KQgIi1FZLWIxIrISXMwiEh7EVkm\nIstFZK6I2GwrWWzWLKhdG955B57oFMdTiS8ScU0ziIlxF6PZAHbG5CgB+48XkQhgKHAtUB24Q0Sq\npyi2AbhSVWsBLwIjAhWPOdHBg9CzJzRrcoxb//mQaVMTGTjqHGTpUhg/HsqVC3aIxpggCGSfQgMg\nVlXXA4jIGKA1EJNUQFXn+pSfD5QPYDzGM3Om6zs4a91CNpToTNndKyDhPKA5VKgQ7PCMMUEUyLaB\ncsAWn8dbvWVp6Qz8ktoKEekqItEiEh0XF5eJIeYsBw/Cww9DyysP0Xd3bxbkupSy+f6Gn36C5s2D\nHZ4xJgSExNlHInIVLilcntp6VR2B17QUFRVls7+dhunToXNnWL8e/jy3NVW2TIH774dXX4WiRYMd\nnjEmRAQyKWwDzvV5XJ7/5npOJiK1gZHAtaq6J4Dx5EgHDkDfvvDZ0H8oVzEvM2bkowr9IPFpaNo0\n2OEZY0JMIJPCIqCyiFTEJYN2wJ2+BUTkPOA7oIOqrglgLDnStGmu76DWxh/ZVLAbBW+5hzxNBgJN\ngh2aMSZEBaxPwZvKswcwEVgFfK2qK0Wkm4h084o9C5QA3hORpSISHah4cpIDB+DBB6Ht/+IYsvsO\nfqAVxS4oQZ7bbw52aMaYEBfQPgVVnQBMSLFsuM/9+4D7AhlDTvPbb67voMrGiWzM154CR/fDCy9A\nnz4QGRns8IwxIc6uTMom/v0XHngArr7aHftf+awcBRvUcNcd9OtnCcEY45eQOPvInJmpU+G+exNp\nuXkEc+r8Qb15w8ifvybcNSPYoRljwowlhTC2fz888QRMfX8tY/J3oSEzoOTVIEcAG6/IGHPqrPko\nTE2eDHVrxlNkxOvE5K5Ng8il8OGHboUNYGeMOU2WFMLM/v3QtStccw2UidzDwEIDyXN9CyQmxp1/\nasNbG2POgCWFMDJxItSvcRQZ+QF9Hk9kyvLS5F6+FMaNg7Jlgx2eMSYbsD6FMPDPP/DYY7Diw/n8\nGtmZShoDzStA/uZw/vnBDs8Yk41YTSHETZkCDWocpOaHvZjLZVx49r8wYYINYGeMCQirKYSoAwfc\nmUXDhsGcAq25jKnuMuWXX4YiRYIdnjEmm7KkEIJmzYKHOuxjzaa8PPpofi6+9jmIfBaa2JhFxpjA\nsqQQQg4fhqefhg2DvmdixIMcu/Nuzn3zFeCKYIdmjMkhrE8hRCxYAM1q7aTRoLaM4yZK1ijNuY+1\nDXZYxpgcxmoKQXb0KPTvD3+88gs/yV0UzXMAnn+JiMcfhzx5gh2eMSaHsaQQREuXwt13w/Ll8Eyb\ncym8uza5PhgGVasGOzRjTA5lSSEIjh+HVwYmsuuF4Tye9w+K//g+N9xQE5gW7NCMMTmcJYUsFhMD\n/dqupufKLjRhFscaNieymQ1gZ4wJDZYUskhCAgx6PZ59z7zBFwnPk6tgfnj3IyLvucfGKzLGhAxL\nCllg7Vro2BFi5+5hXZ5XocV1RI4cCmXKBDs0Y4w5gSWFAEpMhPffOcKqPh+zqkBXBn9WmoJNliHn\nnRvs0IwxJlWWFAJk61Z446Y5dIvuzAOs5tnhlSjZrhlgCcEYE7rs4rUA+OajA/x84cO8FX0F5Uoc\nQX+d6CUEY4wJbZYUMtHevdCuHZx1b2u6HHuX/R16UHjjCqTFNcEOzRhj/GLNR5nkt7F76fJwfjbH\n5ee6ri+QeCcUu7JxsMMyxphTYknhDB08CF/e8i03TuzOUyXuod6CV6lf35KBMSY8WVI4A79P+Itd\nt/egy4Fv2Xp2Pe4a34689YMdlTHGnD7rUzgNR4/Cp+0mUOH66jQ98BPr7nuZ8lsXkLdRvWCHZowx\nZ8RqCqdo8WJ3IVrCigpcXKY+5/0wlAujqgQ7LGNC0vHjx9m6dStHjhwJdig5Rr58+Shfvjx5TnOU\nZUsKfjp6OJHJbYby1+Rl7C3zASN+qk6N66cEOyxjQtrWrVspXLgwFSpUQGw4l4BTVfbs2cPWrVup\nWLHiab1GQJuPRKSliKwWkVgR6ZvKehGRwd76ZSISki3yMd+uIqbkFdww6WEalN3KiugjXH99sKMy\nJvQdOXKEEiVKWELIIiJCiRIlzqhmFrCkICIRwFDgWqA6cIeIVE9R7FqgsnfrCgwLVDyn4/ih4/zW\nbCAX3lqX8w//ydJHP6X2lgkUL2MjmhrjL0sIWetM3+9A1hQaALGqul5VjwFjgNYpyrQGPlVnPlBM\nREJmlLjON+2l7tQ3WHJeGyQmhrpvdrARTY0x2Vogk0I5YIvP463eslMtg4h0FZFoEYmOi4vL9EDT\n0qlvaRZ9uJxGm76ieNXSWbZdY0zm+v777xER/vzzz+Rl06dP54YbbjihXMeOHRk7dizgOsn79u1L\n5cqVqV+/Ppdeeim//PLLGcfy8ssvU6lSJapUqcLEiRNTLfP8889Trlw56tatS926dZkwYQIAe/bs\n4aqrrqJQoUL06NHjjGNJTVh0NKvqCGAEQFRUlGbVdq+6CrjqpBxljAkzo0eP5vLLL2f06NH079/f\nr+f069ePHTt2sGLFCvLmzcvOnTuZMWPGGcURExPDmDFjWLlyJdu3b6dZs2asWbOGiIiIk8r26tWL\n3r17n7AsX758vPjii6xYsYIVK1acUSxpCWRS2MaJQ4KW95adahljTDbwyCNuXvLMVLcuvP12+mUO\nHDjA7NmzmTZtGjfeeKNfSeHQoUN88MEHbNiwgbx58wJQunRp2rZte0bxjh8/nnbt2pE3b14qVqxI\npUqVWLhwIZdeeqlfzy9YsCCXX345sbGxZxRHegLZfLQIqCwiFUUkEmgH/JCizA/A3d5ZSI2Af1R1\nRwBjMsbkMOPHj6dly5ZcdNFFlChRgsWLF2f4nNjYWM477zyKFCmSYdlevXolN/P43l555ZWTym7b\nto1zz/3vd3D58uXZti3138FDhgyhdu3a3Hvvvfz9998ZxpFZAlZTUNV4EekBTAQigFGqulJEunnr\nhwMTgOuAWOAQ0ClQ8RhjgiujX/SBMnr0aHr27AlAu3btGD16NBdffHGaZ+mc6tk7gwYNOuMYU3rg\ngQfo168fIkK/fv147LHHGDVqVKZvJzUB7VNQ1Qm4A7/vsuE+9xXoHsgYjDE51969e/ntt99Yvnw5\nIkJCQgIiwuuvv06JEiVO+gW+d+9eSpYsSaVKldi8eTP79+/PsLbQq1cvpk2bdtLydu3a0bfviZdn\nlStXji1b/ju3ZuvWrZQrd3K/ZenS/53Y0qVLl5M6xAPJxj4yxmRbY8eOpUOHDmzatImNGzeyZcsW\nKlasyKxZs6hcuTLbt29n1apVAGzatIk//viDunXrUqBAATp37kzPnj05duwYAHFxcXzzzTcnbWPQ\noEEsXbr0pFvKhADQqlUrxowZw9GjR9mwYQNr166lQYMGJ5XbseO/VvRx48ZRs2bNzHpLMhQWZx8Z\nY8zpGD16NH369Dlh2S233MLo0aNp0qQJn3/+OZ06deLIkSPkyZOHkSNHUrRoUQAGDBjAM888Q/Xq\n1cmXLx8FCxbkhRdeOKN4atSoQdu2balevTq5c+dm6NChyWce3XfffXTr1o2oqCieeOIJli5diohQ\noUIF3n///eTXqFChAvv37+fYsWN8//33TJo0ierVU14XfPrEteCEj6ioKI2Ojg52GMYYP6xatYpq\n1aoFO4wcJ7X3XUQWq2pURs+15iNjjDHJLCkYY4xJZknBGBNQ4dZEHe7O9P22pGCMCZh8+fKxZ88e\nSwxZJGk+hXz5Tn8kZzv7yBgTMOXLl2fr1q1k5UCWOV3SzGuny5KCMSZg8uTJc9ozgJngsOYjY4wx\nySwpGGOMSWZJwRhjTLKwu6JZROKATVm4yZLA7izcXlaz/Qtv2Xn/svO+Qdbv3/mqWiqjQmGXFLKa\niET7c2l4uLL9C2/Zef+y875B6O6fNR8ZY4xJZknBGGNMMksKGRsR7AACzPYvvGXn/cvO+wYhun/W\np2CMMSaZ1RSMMcYks6RgjDEmmSUFj4i0FJHVIhIrIidNrirOYG/9MhGpH4w4T5cf+9fe26/lIjJX\nROoEI87TkdG++ZS7RETiReTWrIzvTPmzfyLSVESWishKEZmR1TGeCT++m0VF5EcR+cPbv07BiPN0\niMgoEdklIivSWB96xxVVzfE3IAJYB1wARAJ/ANVTlLkO+AUQoBGwINhxZ/L+XQYU9+5fGy7758++\n+ZT7DZgA3BrsuDP5sysGxADneY/PDnbcmbx/TwGvevdLAXuByGDH7uf+NQHqAyvSWB9yxxWrKTgN\ngFhVXa+qx4AxQOsUZVoDn6ozHygmImWyOtDTlOH+qepcVf3bezgfOP2xd7OWP58dwEPAt8CurAwu\nE/izf3cC36nqZgBVDad99Gf/FCgsIgIUwiWF+KwN8/So6kxcvGkJueOKJQWnHLDF5/FWb9mplglV\npxp7Z9yvl3CQ4b6JSDngJmBYFsaVWfz57C4CiovIdBFZLCJ3Z1l0Z86f/XsXqAZsB5YDPVU1MWvC\nC7iQO67YfArmBCJyFS4pXB7sWDLR20AfVU10PzazndzAxcDVQH5gnojMV9U1wQ0r07QAlgL/Ay4E\nJovILFXdH9ywsidLCs424Fyfx+W9ZadaJlT5FbuI1AZGAteq6p4siu1M+bNvUcAYLyGUBK4TkXhV\n/T5rQjwj/uzfVmCPqh4EDorITKAOEA5JwZ/96wS8oq4RPlZENgBVgYVZE2JAhdxxxZqPnEVAZRGp\nKCKRQDvghxRlfgDu9s4WaAT8o6o7sjrQ05Th/onIecB3QIcw+4WZ4b6pakVVraCqFYCxwINhkhDA\nv+/meOByEcktIgWAhsCqLI7zdPmzf5txtSBEpDRQBVifpVEGTsgdV6ymAKhqvIj0ACbizoYYpaor\nRaSbt3447qyV64BY4BDu10tY8HP/ngVKAO95v6jjNQRHcEzJz30LW/7sn6quEpFfgWVAIjBSVVM9\nBTLU+Pn5vQh8LCLLcWfp9FHVsBhSW0RGA02BkiKyFXgOyAOhe1yxYS6MMcYks+YjY4wxySwpGGOM\nSWZJwRhjTDJLCsYYY5JZUjDGGJPMkoIJOSKS4I34mXSrkE7ZCmmNQHmK25zujdT5h4jMEZEqp/Ea\n3ZKGmBCRjiJS1mfdSBGpnslxLhKRun485xHv+gVjMmRJwYSiw6pa1+e2MYu2215V6wCfAK+f6pO9\nawY+9R52BMr6rLtPVWMyJcr/4nwP/+J8BLCkYPxiScGEBa9GMEtEfvdul6VSpoaILPRqF8tEpLK3\n/C6f5e+LSEQGm5sJVPKee7WILBE3z8QoEcnrLX9FRGK87bzhLXteRHqLm68hCvjC22Z+7xd+lFeb\nSD6QezWKd08zznn4DJ4mIsNEJFrcnAP9vWUP45LTNBGZ5i27RkTmee/jNyJSKIPtmBzEkoIJRfl9\nmo7Gect2Ac1VtT5wOzA4led1A95R1bq4g/JWEanmlW/sLU8A2mew/RuB5SKSD/gYuF1Va+FGAHhA\nRErgRl2toaq1gQG+T1bVsUA07hd9XVU97LP6W++5SW7Hjct0OnG2BHyH63jauwq9NnCliNRW1cG4\n0UWvUtWrRKQk8AzQzHsvo4FHM9iOyUFsmAsTig57B0ZfeYB3vTb0BNxw0SnNA54WkfK4+QXWisjV\nuBFEF3nDd+Qn7TkVvhCRw8BG3PwLVYANPmNBfQJ0xw3lfAT4UER+An7yd8dUNU5E1nvj3KzFDew2\nx3vdU4kzEje3gO/71FZEuuL+r8sA1XFDX/hq5C2f420nEve+GQNYUjDhoxewEzf6Zy7cQfkEqvql\niCwArgcmiMj9uLFyPlHVJ/3YRntVjU56ICJnpVbIG6+nAW6QtluBHrhhnf01BmgL/AmMU1UVd4T2\nO05gMa4/YQhws4hUBHoDl6jq3yLyMZAvlecKMFlV7ziFeE0OYs1HJlwUBXZ4k6t0wA2edgIRuQBY\n7zWZjMc1o0wFbhWRs70yZ4nI+X5uczVQQUQqeY87ADO8NviiqjoBl6xSm8/6X6BwGq87Djfj1h24\nBMGpxukNI90PaCQiVYEiwEHgH3EjiV6bRizzgcZJ+yQiBUUktVqXyaEsKZhw8R5wj4j8gWtyOZhK\nmbbAChFZCtTETXMYg2tDnyQiy4DJuKaVDKnqEdyold94I3QmAsNxB9ifvNebTept8h8Dw5M6mlO8\n7t+4oa3PV9WF3rJTjtPrq3gTeFxV/wCW4GofX+KapJKMAH4VkWmqGoc7M2q0t515uPfTGMBGSTXG\nGOPDagrGGGOSWVIwxhiTzJKCMcaYZJYUjDHGJLOkYIwxJpklBWOMMcksKRhjjEn2f3KK8kXMBvD1\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd1544bc208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred = np.sum(np.take(X, best_cmb, axis = 1), axis = 1) / X.shape[1]\n",
    "fpr, tpr, thresholds = metrics.roc_curve(y, pred)\n",
    "plt.title('Receiver Operating Characteristic')\n",
    "plt.plot(fpr, tpr, 'b', label='AUC = %0.2f'% best_auc)\n",
    "plt.legend(loc='lower right')\n",
    "plt.plot([0,1],[0,1],'r--')\n",
    "plt.xlim([-0.1,1.1])\n",
    "plt.ylim([-0.1,1.1])\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

Timer unit: 1e-06 s

Total time: 0.007946 s
File:
Function: auc at line 1

Line # Hits Time Per Hit % Time Line Contents

 1                                           def auc(y, X):
 2         1         3283   3283.0     41.3      pred = np.mean(X, axis = 1)
 3         1         4498   4498.0     56.6      fpr, tpr, thresholds = metrics.roc_curve(y, pred)
 4         1          165    165.0      2.1      return metrics.auc(fpr, tpr)

To reduce total processing time by using multiprocessing

import multiprocessing as mp

def roc(v):
    """ calculate one pair, return (index, auc) """
    i, true, pred = v
    fpr, tpr, thresholds = metrics.roc_curve(true, pred, drop_intermediate=True)
    auc = metrics.auc(fpr, tpr)
    return i, auc

pool = mp.Pool(3) 
result = pool.map_async(roc, ((i, true[i], pred[i]) for i in range(2)))
pool.close()
pool.join()
print result.get()

Python is not supported tail call optimization, therforer, recursive call function should transform loop function.
my god!