Skip to content

Instantly share code, notes, and snippets.

@justheuristic

justheuristic/l2r - mq2007

Created July 2, 2015 13:56
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save justheuristic/06995a42ff5d6e9d51f4 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save justheuristic/06995a42ff5d6e9d51f4 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
l2r - mq2007
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"%pylab inline\n",
"from time import time\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import pandas\n",
"import h5py\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.externals import joblib\n",
"from sklearn.metrics import mean_squared_error\n",
"from sklearn.cross_validation import train_test_split"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Load the data. One fold for now"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def save_as_h5(path_to_txt,output_name=\"mslr\"):\n",
" \n",
" print \"opening \"+path_to_txt\n",
" f = open(path_to_txt)\n",
" labels = []\n",
" features = []\n",
" print \"extracting...\"\n",
" for line in f:\n",
" line = line[:line.find('#') - 1]#удалить комменты из конца линии\n",
" ls = line.split()\n",
" labels.append(int(ls[0]))\n",
" features.append([float(x[x.find(':') + 1:]) for x in ls[1:]])\n",
" f.close()\n",
" print \"converting...\"\n",
" labels = np.asarray(labels, dtype=np.int32)\n",
" features = np.asarray(features)\n",
" query = features[:, 0].astype(int)\n",
" features = features[:, 1:]\n",
" print \"saving...\"\n",
" h5f = h5py.File(output_name, 'w')\n",
" h5f.create_dataset('qids', data=query)\n",
" h5f.create_dataset('labels', data=labels)\n",
" h5f.create_dataset('features', data=features)\n",
" h5f.close()\n",
" print \"done\"\n",
" return features,query,labels"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def load_h5(name):\n",
" print \"reading from\",name\n",
" h5f = h5py.File(name,'r')\n",
" labels = h5f['labels'][:]\n",
" qids = h5f['qids'][:]\n",
" features = h5f['features'][:]\n",
" h5f.close()\n",
" print \"done\"\n",
" return features,qids,labels"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"opening MQ2007/Fold1/train.txt\n",
"extracting...\n",
"converting...\n",
"saving...\n",
"done\n",
"opening MQ2007/Fold1/test.txt\n",
"extracting...\n",
"converting...\n",
"saving...\n",
"done\n",
"opening MQ2007/Fold1/vali.txt\n",
"extracting...\n",
"converting...\n",
"saving...\n",
"done\n",
"Wall time: 4.08 s\n"
]
}
],
"source": [
"%%time\n",
"##warning! this can take a long time. no need to rerun that code if u have CSV files created once.\n",
"save_as_h5(\"MQ2007/Fold1/train.txt\",\"mq2007_train\")\n",
"save_as_h5(\"MQ2007/Fold1/test.txt\",\"mq2007_test\")\n",
"save_as_h5(\"MQ2007/Fold1/vali.txt\",\"mq2007_vali\")\n",
"#print \"converted that\""
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"reading from mq2007_train\n",
"done\n",
"reading from mq2007_vali\n",
"done\n"
]
}
],
"source": [
"Xtr,Qtr,Ytr = load_h5(\"mq2007_train\")#smallest one 4 speed purpose\n",
"Xts,Qts,Yts = load_h5(\"mq2007_vali\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train: (42158L, 46L) qids: 1017\n",
"test: (13813L, 46L) qids: 339\n",
"qid intersection: 0 (must be 0)\n"
]
}
],
"source": [
"print \"train: \",Xtr.shape,\"qids:\",len(set(Qtr))\n",
"print \"test: \",Xts.shape,\"qids:\",len(set(Qts))\n",
"print \"qid intersection:\",len(set.intersection(set(Qtr),set(Qts))),\"(must be 0)\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Metrics & scoring"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from metrics import roc_auc_score, mean_ndcg #def mean_ndcg(y_true, y_pred, query_ids, rank=None)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def score(model, X, Y, qid):\n",
" Ypred = model.predict(X)\n",
"\n",
" print \"NDCG5 :\", mean_ndcg(Y, Ypred, qid, rank=5)\n",
" print \"NDCG10 :\", mean_ndcg(Y, Ypred, qid, rank=10)\n",
" print \"NDCGfull:\", mean_ndcg(Y, Ypred, qid, rank=None)\n",
" print \"rank AUC:\", roc_auc_score(Y, Ypred)\n",
" \n",
" print \"MSE :\", mean_squared_error(Y,Ypred)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def learning_curve_ndcg(model, X, y, qid, rank=10):\n",
" \"\"\"stolen from some GIT repo\"\"\"\n",
" max_n_trees = len(model.estimators_)\n",
" scores = []\n",
" \n",
" if hasattr(model, 'staged_predict'):\n",
" #for boosting ensembles\n",
" n_trees = np.arange(max_n_trees) + 1\n",
" for y_predicted in model.staged_predict(X):\n",
" scores.append(mean_ndcg(y, y_predicted, qid, rank=10))\n",
" else:\n",
" y_predicted= np.zeros(len(y)).astype(float)\n",
" n_trees = np.arange(max_n_trees) + 1\n",
" \n",
" for i in xrange(max_n_trees):\n",
" y_predicted += model.estimators_[i].predict(X)\n",
" scores.append(mean_ndcg(y, y_predicted, qid, rank=rank))\n",
" \n",
" plt.plot(n_trees, scores)\n",
" plt.xlabel(\"Number of trees\")\n",
" plt.ylabel(\"NDCG-\"+ str(rank))\n",
" _ = plt.title(\"NDCG learning curve\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Model training & evaluation"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Random Forest (mse)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 1min 49s\n"
]
}
],
"source": [
"%%time\n",
"from sklearn.ensemble import RandomForestRegressor\n",
"rfr = RandomForestRegressor(n_estimators=300, min_samples_split=5, random_state=1, n_jobs=-1)\n",
"rfr.fit(Xtr, Ytr)#only 2 cores and the unlimited tree growth. Don't take it for a benchmark time"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train:\n",
"NDCG5 : 0.854486132106\n",
"NDCG10 : 0.854737617893\n",
"NDCGfull: 0.854734843908\n",
"rank AUC: 0.999791041674\n",
"MSE : 0.0425252525027\n",
"test\n",
"NDCG5 : 0.416685583502\n",
"NDCG10 : 0.444312479701\n",
"NDCGfull: 0.610025074284\n",
"rank AUC: 0.667734621756\n",
"MSE : 0.306501984652\n",
"Wall time: 1.74 s\n"
]
}
],
"source": [
"%%time\n",
"print \"train:\"\n",
"score(rfr,Xtr,Ytr,Qtr)\n",
"print \"test\"\n",
"score(rfr,Xts,Yts,Qts)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 9.08 s\n"
]
},
{
"data": {
"image/png": [
"iVBORw0KGgoAAAANSUhEUgAAAY4AAAEZCAYAAACAZ8KHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
"AAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYJEWd//H3h5nhvkEQBpT70AUEZMBF1hEQ8RwFFPBA\n",
"xQNdAVFRYT2Y9Wa98CfKwyWHB+iCcrgIgjqK6yo3Ioycolxy34jOOJ/fHxGNRU13dVd3V1V39ef1\n",
"PPVMZ2ZkZGRnT34rIjIiZZuIiIiRWqLXBYiIiMklgSMiItqSwBEREW1J4IiIiLYkcERERFsSOCIi\n",
"oi0JHDGlSZot6bYeHfsYSR/rxbEjxiKBIzpO0q2S7pa0bMO6d0j6ecPyIkmPSXpU0n2SLpL0+kHy\n",
"eqmkX0p6RNI9kuZJelXD9rUkHS/pjprXzZJOkrRp58+0PbbfY/vTvS5HRLsSOKJblgDeN0yaLW2v\n",
"AGwCnAwcLekTAxsl7QV8v26baXsN4BPAq+r21YBfA0sDL6x5bQP8AnjJeJ7McCRN7+bxOqEfziE6\n",
"I4EjusHAF4FDJa00bGL7AdvfBt4DHC5pFUkCvgx80vY3bT9a0/7S9rvqru8HHrL9Ztt/rNsftn2y\n",
"7aNHUlBJa0s6s9ZmbpF0UMO2WZL+T9KDku6U9DVJMxq2L5L075JuBK6X9CJJt0v6QK1x3SnprQ3p\n",
"T5b0qfrz7GHSribpXEkPS7pE0qclXdziPF4o6de1rH+WtF9dP0/S2xvSvbUxn4ZzuAG4QdI3JH2h\n",
"Ke+zJb1/uN9X9K8EjuiWy4B5wKFt7HMOMB2YBWwKrAOc0SL9rsAPR1k+JC0BnAtcCawN7AIcImm3\n",
"mmQhpda0GvCCuv3fm7KZA2wHPAcQsCawYs3v7cDXG4Kn62dAq7RfBx6tad4C7Ne0b+N5PBs4D/gq\n",
"sDrwPODqIY45mDmU3/nmwGnA3g15r0KpvZ02gt9X9KkEjugWU5qVDpK0+oh2sBcA9wGrUm7WAHe1\n",
"2GU14C8DC5JeXb9xPyLpghEccjtgdduftr2w1lpOAPap5bnC9iW2F9n+E3Ac8KKmPD5n+yHbf6vL\n",
"Cyi1pH/Y/jHwGCUIPlXMhp8HTStpGrAHcITtJ23PB05p2rfRG4ALbX+v5vWA7auHSDuYxnP4FWBJ\n",
"O9VtewG/tv2X4X5f0b/ShhldY/taST8CDgPmD5e+NgM9A3gAuL+uXgv40xC73E/55jtwvHOAVWrT\n",
"zJtGUMRnA2tLerBh3TTgl7U8m1Cay7YFlqX8/7msKY/mJ7Tut72oYfkJYPmhyj9E2mfUYzXmfXuL\n",
"81gHuKXF9uE8dRzblnQ6sC9wMSUonVo3t/x9Rf9KjSO67QjgncDMEaSdQ2keugS4nnJD26tF+p8C\n",
"r6n9IY2G+mbe7Dbgj7ZXafisaPuVdfsxwHXARrZXAj7K4v+H2p1ueiTp76X8HtZtWLfuEGmhnMeG\n",
"Q2x7HFiuYfmZIyjTacBetQlsFnBmXf9nWv++ok8lcERX2b4Z+B6DP2ElAEmrSnojcDTwedsPusz/\n",
"/wHg47VDd0VJS9RO4GPr/l8GVgG+JWkDFStQ2vhHcoO+BHhU0oclLSNpmqR/kfT8un15Sj/DE5I2\n",
"o3Tej4UYQVCz/Q/gB8DcWq7NgDcz9Dl9B9hV0uskTa8d61vVbVcBe9R8NqL0pQx3/KsoTYYnAOfb\n",
"fqRuGu73FX0qgSN64ZOUpp7mG9/Vkh4FbgT2Bw6xPXdgo+0zKR21+wN3UPozPgmcVbffD+wAPElp\n",
"m3+E0nG7HK1v8q77/wN4JSXQ3EL5pn8cpcMaSsf+G2q+xwGnN53DYDfyVgGruaO6VdoDgZUo53wK\n",
"pRbw90EztW8DXg58kNJ8dyWwZd38lbrf3cBJwLdHWIbvAjvXfweOs4jWv6/oU+rki5wk7Q4cRWn3\n",
"PMH2kUOk2w74P+D1tn9Q161M+YbzXMof8/62f9OxwkZMIpKOBNaw/bZelyWmno7VOOqTIEcDu1Me\n",
"TdxX0uZDpDsSOJ+nV9u/Cpxne3PKt6VhO1Mj+pWkTSVtWZvfZlFqXaN+9DhiLDrZVDULuMn2rfWx\n",
"ytMpnZ3NDqI8m3/vwIr67PpOtr8JUB/1e7iDZY2Y6FagdEo/Rvm/9MX61FhE13XycdyZLP744PaN\n",
"CSTNpASTnSnPhA+0m60P3CvpJGAr4HLgfbaf6GB5IyYs25cBG/e6HBHQ2RrHSDpPjgIOq0/MND5h\n",
"Mp0yx9A3bG9DeYTwsI6UMiIi2tLJGscdLP7cefOgpW2B0+tj96sDL5O0APgtcLvtS2u6MxgkcEjq\n",
"XM9+REQfsz3S8U2L6WTguAzYWNJ6wJ2Uxyj3bUxge4OBn2uz1LkD7baSbpO0ie0bKHMQXTvYQcZy\n",
"8hOdpLmNj6P2m5zf5NbP59fP5wZj/9LdscBhe6GkA4ELKI/jnmh7vqQD6vZjW2ZQOs2/I2lJ4GYg\n",
"jx1GREwAHZ2rqk7U9uOmdYMGjObn0eukbNt1rnQRETEaGTk+sc3rdQE6bF6vC9Bh83pdgA6b1+sC\n",
"dNC8XhdgIuvoyPFOk+R+7uOIiOiEsd47U+OIiIi2JHBERERbEjgiIqItCRwREdGWBI6IiGhLAkdE\n",
"RLQlgSMiItqSwBEREW1J4IiIiLYkcERERFsSOCIioi0JHBER0ZYEjoiIaEsCR0REtCWBIyIi2pLA\n",
"ERERbUngiIiItiRwRMSkILGRxKR74+dQZZaYIfE1ieMnQnnakcARMYVJrCfxZYkPSywhcbDE0g3b\n",
"3ynxJonVJDbrYTlXAOYD8yRWG2S7JNbufsmeVobF7qcSbwf+LLGHxP+rv+dlJd4A/AFYD9hVYpeB\n",
"PCTmSiw3zLGWrPntMsKyPaOhfIe0c16D5pd3jkdMDhI7AQ/ZXNPmfgJeCvzUZkFd91lgBWBn4GfA\n",
"fsAewEXAecDZwA3AmcDfKF8yHwCeazPoTUPiIGALm3e1f3bDnsOuwCeBK4BnA69uLIfEx4EPA+va\n",
"PNS071LAvwPTbb4wyuOvAjxss2iI7csBlwFLA+cAvwM2o/xejwQOAE4HtgVWBTYAXgv8hvJ7/7DN\n",
"9hL7At8FDrb5WtMxpgGrU67DMcBGwLOAa4BNgYeA44EdgTWB99j8WWJ5SpD6GiX4HgNaeyz3zgSO\n",
"iAlMYmmbJ+vPvwKWA7Yd6gY2RB7bAJcAf6LcQI4ALgR+DtwJHATMo9zQzgUErE25sX22rlsTOAHY\n",
"y+aKhnz/DDxZ972i7rujzR9GWLYZA8GsLi8DPNkcnCSOAJYBPgH8Avg1cCgwAzgMeBflZv1zys3z\n",
"E8AtwFbAXjX99sALgL8C3wZOBU62WSSxFvAi4PnAfTafbzj2F4D3AXNtPjvIOYhyU14Z+Fw93rOA\n",
"24DTbK5vSLtk/T2eYvPTum465Tq8iPK7PrGez47136Uo1+4ASoB4FLiqHmdBXX8zJaDuCPwfsDzl\n",
"Oi+iBLRFwHMBA68C/WZM907bHfsAu1P+UG8EPtIi3XbAQmCPpvXTgCuBc4fYz50sfz759PIDXgv8\n",
"EPiF4NXBD4N/C35Tm/l8FHwUeDvwXPDj4O82pfkA2OAtG9YtDVbD8ifBX6k/rwJ+BPwo+Anwk+DP\n",
"g/8D/L0Rluu94IXgn4LXBa8Evh78ubp9Bnjl+vOF4FfVn1cFXwb+Ifg34PPAzwJvXct0L/h48Hfr\n",
"+a5V9/tY/f1dBf4a+BLwj8DvAd8F/kE9x4fAK9Z9Xgq+CbwN+D7wRg3lF3gf8KXgy8Grj+FafwN8\n",
"C/iHdfnE+rv9PvgL4JPAe9Rr8pwR5rkU+NngY8Dr1OvzrrINj+lvs3N/9EwDbqK04c2oEXLzIdL9\n",
"DPgRsGfTtg8A3wHOGeIYYzr5fPKZaB/wtuBTwNPrze2qepN8e72xvQF8Zpt5Xgx+af1Z4C+Bn9eU\n",
"5tngsxoDxSD5bFhvyiuBDwKfVm/uGtgPvFy90c6ped4FPh28A/gZ4OXBLwOfD74VvCn4w+A7wdeB\n",
"vw2+u97076+B4Kx6E129oSxLgw8HHwxeomH9puD1hij/0uB31kChWvbDwacO/H5qurPAb6s33mvB\n",
"c+r6Q8E/Aa9Yb8QH1e17NJZhlNd9R/AD4LUb1g15Lcb+d4bHsn/HmqokvQA4wvbudfmwWtrPN6U7\n",
"BPg7pdbxI9tn1vXrACcDnwE+YPtVgxzDTlPVhFCr+n+xB2//jsFJrAQ8BswFHgG2BF5BaYveGPgX\n",
"4BuUdvD9gV8CF9vMHEHey1KaXn4ErGnz13Eo7ym1vDsD77X52SBpXkjpI7kX+B7wIPBeYBVKU9sf\n",
"KU1gP7B5ou6zNbAapRnqxZT+gR9QmmVeBaxh8+Wxln8kJPagXI9fA2sBr7GxxAzgUkqTEJTmn+1t\n",
"bhqn4y478PvotLHeO6ePZ2GazKS08Q24ndLG+BRJM4E5lD/C7eBpN52vAB8CVuxgGWMc1M6364F9\n",
"KB2rfa3e5G62eWSM+SwJXE6pdS+g3DinAbMogeJkm3uAveqTRA9RblZLSqxjc3uLvA8EvghcR+l4\n",
"HXPQqD5JCUQ/oPSLLMbmV9JT5/Blm38AR9VyqaZx0z5XNiz+pH4GfHucyj5SZ1P6Et4A7DpQVpsF\n",
"EntS+jJuBtYer6BR8+9K0BgPnQwcI/nmeRRwmG1LEqVjDUmvBO6xfaWk2a0ykDS3YXGe7XmjK26M\n",
"weso1/sA+jBw1EdBv0/p+LyZ8q34dBjZ00P1ZvkBSsfsW20eq5veXvP7CqWGMQd4js0NwNNq5jb3\n",
"N+T3G2AH4IwhjrcKpWP0eR5hJ/VI2dwMbD7CdIs9wTQZaqQ10H2ufpq33dyw+FDz9omq3kdnj1t+\n",
"HWyq2gGY29BUdTiwyPaRDWlugacGo6wOPEH5z7g98GZKh/nSlFrHmbb3azpGmqomAIn/Bb4OHA3M\n",
"Gs9vYRNBbbr4DOXvcAblccl9gJc1fVMeeJb/E8DRNvfVdYcCbwR+T6lZf5PypekWSjPIZW2W52PA\n",
"3pSnbVaiPNn0Y8qTNK+gNK981+bdoznf6H9jvXd2MnBMpzRf7EJ51OwSYF/b84dIfxLl6akfNK1/\n",
"EXBo+jgmJonnUJ79fxbwTuDjwG42v+9pwcaRxInA1ZRn5/8NuJhSu9qF0hyzpP/5yOyBwH9RnuX/\n",
"UM3iSkpt4ybKI5fHUZ7fn2mPbABXU3k2Al5fj3E/sAklYDxAuRa3Afe6jUd2Y2qZsH0cthdKOhC4\n",
"gNJue6Lt+ZIOqNuPbSe7TpSx2ySeDawDXGrz916XZ5y8HTjJZiFwjMSKwAeBt/W2WGNXB45tBbwc\n",
"+JzLeIOBZ+9PAD5K6az+B/AiifUonar/SnlW/xJKv8XxNjfWbOdJfAg4i9Lp27Zao2scT3AXpfks\n",
"oisyALBLajv3rykDq35u89aGbR8GvuKGgVB1/TqU5pCuzmUzUrVz93bgBQNtvxLPpLTXH0QZsHUx\n",
"8FWbl0qok23cEu8Gnm/zjnHK71DK4LIbbV4wxPG2AnYD3gH8B3CRTUNzLMsAC2pgHVgnYE/KU0Wp\n",
"FUTXjfne2annhLvxYQKM4wCvB166ad20xufC67qXg38PXoMywGhD8OvB64MN3hX8VfB2Dfv8J/gf\n",
"4H/p9XkOce4vBl8yyPr/qc/f3wn+dD2/jcE3grcax+O/u44JEHi1OsbgYfAz28hD4Fk8faDb9PrM\n",
"/5/Azx9BHgeAF4HPBk/v9XXJJ5/hPmO9d6bGMeYy8FPgVJtTGtY9nzLsf1WbR+u684Bv23xX4kxK\n",
"W/mKwOGUJ2guoLRTn2yzf/1Weh1l4OQawMtt/tahc3gv8KDNd9vc73PAQpuPN63fjNIMegrlefw7\n",
"KB3DuwO/tNl9HMr8b8BplDEFX6M8PinKwxSbUH63A2MFFg6Rx1I1j1cDB9kcU9efSOn8vspmxxGU\n",
"ZQng2TZ/HOt5RXRDahw9L4P/BP5607r967fsV9flpeo38FXq8mzKaNpzwQ9SphQwZXqBB8HLgLeo\n",
"eS8FPoMyGvcGRjjdQBvlX40yQvcv4F0H2b4x+IPg1QbZdjl4pxZ5fxD85/qN3OBDwDeDd2ixzzKU\n",
"qTGWb5FmCfAV4L3Bz6s1jXsp03I8lzLyek/KdBR3DZQRvEKtJa1Xl4+gTFfxHMpo5UPAW9W8tgDP\n",
"7PXfVz75dOIz1ntnz0+glyc/tmN7BfCStYnikqZtR4FvGwgoNVD8dpA89qk31PXBZ1KmS7iQMiXE\n",
"peAP1XRL1BveR8E/Y5ymIqhNPeeBj6VMafG9pu2bU+bnuQL8mbpOtTxr1GahGS3yX7beiDegzEn0\n",
"zHqz/soQ6VcG30Np0nsE/N4h0r0SfCX/nObiGPC7h0i7Ww0EnwL/EfzrGiSPqOvXrek2okyDsQB8\n",
"eK//tvPJp5OfBI6eHNfT6rf0Xeo32sfBSzZs/1n9tn0P+BeU+W0+M0g+y4H/s2ndsyhzEn2tOUBQ\n",
"2t6vGAgo43Aed1P6INak9Lnc0XAzXoIyR9K76rb7KHP0nEyZy+dT4JPaONY69d/n1KD6rYEbdD2v\n",
"jSk1k+/XdRtQJrx72yB5XQzep41jb1aD4951+RWUSfAW67+g9G2Mad6hfPKZ6J+x3jvTxzGq47IJ\n",
"ZYzKScC6lCmn97e5rPZN3AtsQRnEeCflEc032fxmHI69LvC/wNtcp2Ue4X77Aru4PnFUpwm5F1jO\n",
"ZVppUR7r3MHm1jpl9unApjaW+DrwJso7GpamvE/gXzyKdn2JaynjDzagDPjckvI7uoPS13BeTfd8\n",
"yliF4+uxb6NMiPl6yqjoQfsuIqK1CTsAsBt6GDj2Av6bMindaZT5g9ajvB/gScrgrjXtzjx6KvEu\n",
"yiC7vYZJtybwMGVStl9RRhpvAXyJMufQ6TbPaUh/BmWQ2k2UYPgM+59vC5NYH7iPcsPfwaadsTiN\n",
"5doIuLuW5Wx4arT1/pRJ4xofXT2lHu89lPcPvAN4oc2tozl2RCRw9CpwfIoy78sLKW8d+zpl0NsB\n",
"lAnQjrY5rIPHX4nyYpcNXecwqk8ZvXzguPVJn6so4yjWoQwQexllbMWOlNdHvsTmlQ35HkKZWuOv\n",
"lKeVDrC5oFPnUY85G9jE5rghti9BbVEdWHbGPkSMSQJHbwLHOZRXan4TeJ3ND+p6ATPchVHhEqcC\n",
"GwL/SXn72pcoTUj/anO9xGsptYq1KLWgjSiB7SjKtNZ3AFfaHNyQ5zKUOcNeAXyZ8jjxk50+l4jo\n",
"rgk75Uif25Iy2+n5lNdVAlC/FXdrKpEDKLOpnkqZeG9XylxR+0psD+xE6QtYBfirzZO1KWoDyqye\n",
"H6cEv6e4TL19m8TxlHc+JGhExGJS42j7mKxM6aRdaSI0mdRJBjeyOUdiF8p7DM6idJ4P+r4IiTk1\n",
"zattzu1eaSNiIhjrvXOJ8SxMP5CQxKyBF84MYkvg9xMhaADYXGdzTl38JfAt4F1DBY3q8vrvLR0t\n",
"XET0pQSOBhLbUt5q9ltgmyGSbUmZYnvCsVlg81Y3vPRnCHdQ3qp28zDpIiIWk8BRSbyC8jKcb1Pa\n",
"/jcbIulWNPRrTEZ1DM+b04cREaORwMFTA/pOoryN7XjgWmBTiS9J7FPTDDRdbcUErXFERHTDlH+q\n",
"qr5P+izgoza/rquvpzyxtC0lgKwMzJF4DfBc4JqeFDYiYgKY8k9VSbwDeKXNaxrWbUuZ6mJ5ylTd\n",
"D1BGYE8DZPPcsRwzIqKXMo5j7LYHLmxadz3lTX0/pjTnrQK8hPIOjf/raukiIiaYBA6YBU+f7sLm\n",
"MYk7KK96vZAy5cUjwI96UL6IiAllSgeOOkPsRgze2X0+cL7NZd0tVUTExDalAwdlrMY1g80tNTD9\n",
"eEREPN1Ufxx3FnBJrwsRETGZTPXAsT1llHhERIxQxwOHpN0l/UHSjZI+0iLddpIWStqjLq8r6eeS\n",
"rpX0e0kHD7Vv+2VipsRSpMYREdG2jvZxSJoGHE2Z8vsO4FJJ59ieP0i6Iykd0gPPFi8A3m/7KknL\n",
"A5dLurB531H6OuWVritQ3nYXEREj1OkaxyzgJtu32l5AeYf1nEHSHQScQXkHNgC2/2L7qvrzY8B8\n",
"ytiK8TCT8irSSzv1eteIiH7V6cAxk/LuigG313VPkTSTEkyOqasWu5FLWg/YmvHrj1iL8ghuBvNF\n",
"RLSp04/jjuTb/FHAYbYtSfD092DUZqozgPfVmgdN2+c2LM6zPa/Vweo7rNegTFb4+AjKFxExqUma\n",
"Dcwet/w6OVeVpB2AubZ3r8uHA4tsH9mQ5hb+GSxWB54A3mn7HEkzKKO1f2z7qEHyb3u+FYk1gOts\n",
"Vh/VSUVETHITfa6qy4CNa1PTncDewL6NCWxvMPCzpJOAc2vQEHAicN1gQWMMngn8ZRzzi4iYUjra\n",
"x2F7IXAgcAFwHfA92/MlHSDpgGF23xF4E/BiSVfWz+7jUKy1gLvGIZ+IiClpyk2rLvFWYGeb/TpT\n",
"qoiIiW2sTVVTceR4ahwREWMwVQNH+jgiIkZpKgaOZ5IaR0TEqE3FwJGmqoiIMZiKgWMm5dHgiIgY\n",
"hSkVOCREme/qjl6XJSJisppSgQNYDXjC5oleFyQiYrKaaoFjJqltRESMyVQLHOuQwBERMSZTLXCk\n",
"xhERMUYJHBER0ZapGDhu73UhIiIms6kWONLHERExRlMtcKSpKiJijKZM4KivjF0P+FOPixIRMalN\n",
"mcABbAzcZ/NgrwsSETGZTaXAsQ1wRa8LEREx2SVwREREW6ZS4NiaBI6IiDGbEoFDYmlS44iIGBd9\n",
"Hzhq0DgHON/m7l6XJyJisuvrwFHfv3Ec8DDw5h4XJyKiL0xvtVHSysDulIFzUKbruMD2Q50u2Dj5\n",
"1/rZyuYfvS5MREQ/GLLGIWk/4HJgNrBM/ewMXCHpLSPJXNLukv4g6UZJH2mRbjtJCyXt2e6+w3gW\n",
"cLnN46PcPyIimrSqcXwM2La5diFpFeAS4JRWGUuaBhwN7EqZ5uNSSefYnj9IuiOB89vddwTWAO5p\n",
"c5+IiGhhNH0cHmG6WcBNtm+1vQA4HZgzSLqDgDOAe0ex73DWhHSIR0SMp1Y1js8Al0v6Cf+cinxd\n",
"YDfgUyPIeyZwW8Py7cD2jQkkzaQEhJ2B7fhnUBp23xFag1I7ioiIcTJk4LB9iqRzgZcCa9fV84DD\n",
"bY9kvqeR1EyOAg6zbUkC1Ma+AEia27A4z/a8huXUOCJiypM0m9JfPS5aPlVl+wHgtFHmfQelhjJg\n",
"XRZ/idK2wOklZrA68DJJC0a470AZ57YoQ/o4ImLKq1+o5w0sSzpiLPm1DBxDkXSN7S2GSXYZsLGk\n",
"9YA7gb2BfRsT2N6gIc+TgHNtnyNp+nD7jlBqHBER42zIwNH4aGwDU5qT1houY9sLJR0IXABMA060\n",
"PV/SAXX7se3uO9wxn15+RGocERHjTvbg3Qm1yei7wKLmTcBetpfvcNmGJcm2Nfg2VqDUNpazR95n\n",
"EhHR71rdO0eiVVPVNcAXbV8zyEF3Ge0Bu2gN4O4EjYiI8dVqHMchwCNDbNujA2UZb2uSZqqIiHHX\n",
"6nHcX7bYdmlnijOu1iAd4xER466tkeOSJtP7LJ4B3NfrQkRE9Jt2pxwZdWdKD6wK3N/rQkRE9Jt2\n",
"A8d5HSlFZ6wKPNDrQkRE9Ju2Aoftj3aqIB2QwBER0QGt3sfxDkkfbli+Q9Kjkh6T9J7uFK99EstK\n",
"zCCBIyKiI1rVON4NfLNh+R7bK1DmlBrN9B/d8hVgfxI4IiI6otUAQNlufCrpvwFsPylpmc4Wa0zW\n",
"owSMBI6IiA5oFThWalyw/VkASUsAq3WyUGM0kzJ+I4EjIqIDWjVVXSjp040r6jszPgX8pKOlGpuZ\n",
"9ZPAERHRAa0mOVweOIHyZr6r6+qtKNOlv8P2o10pYQvNE3VJLAs8DtwKrAMsmbmqIiKebqyTHA4Z\n",
"OBoOsCHwXMqU6vNt3zTag423QQLHRsBvKbWNe2zW7FnhIiImqI7Njitpd2AF2/8N3Nywfi/gYdsX\n",
"jvagHTQTmA9sTZqpIiI6olUfxyeAXwyy/heUfo6JaCbltbN3kMAREdERrZ6qWsr2YtOS275X0nId\n",
"LNNYrE0JGmsAPe+DiYjoR60CxwqSZthe0LhS0gxg6c4Wa9QGahxrAguGSRsREaPQqqnqB8Bx9ekq\n",
"ACStABxbt01EaaqKiOiwVoHj45SBdLdKuqK+i+OPwL3Ax7pRuFFYhTKV+inAqT0uS0REXxrJ47jL\n",
"AhvWxZts/7XjpRqhQR7H/RVwuM3FPSxWRMSE1rHHcWvmqwNvADarq66TdJrtifqCpGWACRPYIiL6\n",
"Uatp1TcHrgG2Ba4HbgRmAb+XtNlQ+/XYMsCTvS5EREQ/a1Xj+DTwPtvfb1wpaU/gM8Cew2VeBxEe\n",
"BUwDTrB9ZNP2OcAngUX18yHbP6vbDgfeVNdfA7zN9t+GOeTSpMYREdFRreaqusH2Ju1ua0gzjVJT\n",
"2ZXylNOlwL625zekWc724/XnLYAf2t5I0nrAz4DNbf9N0veA82yf0nSM5j6Ou4Btbe4c5rwjIqas\n",
"sfZxtHqq6vFRbhswi9KZfmsdC3I6MKcxwUDQqJYHBt7/8QhlHMaykqYDy1KCz3DSxxER0WGtmqqe\n",
"IekDwGBR6RkjyHsmcFvD8u3A9s2JJL0G+BywFrAbgO0HJH0J+DMlEFxg+6IRHDN9HBERHdYqcJwA\n",
"rDDIegHHjyDvEU1nbvss4CxJOwHfAjatM/IeQnmb38PAf0t6o+3vLFYYae7AT/CzJWF2AkdERANJ\n",
"s4HZ45bfcOM4Rp2xtAMw1/budflwYFFzB3nTPjdTaiW7AC+x/Y66/s3ADrbf25T+qXa6+i6O+20m\n",
"8mttIyJ6rpPTqh8xxCYD2P7kMHlfBmxcO7rvBPYG9m06xobALbYtaZua732Srgc+Xt9t/iSlg/2S\n",
"YY6X/o2IiC5o1VT1OIs3Ny0HvB1YnfIY7ZBsL5R0IHAB5XHcE23Pl3RA3X4s5ZHe/SQtAB4D9qnb\n",
"rpJ0KiX4LAKuAI4b5lzSvxER0QUjaqqStCJwMCVofB/40mBTrndbU1PVRsAF9lPTo0RExCA6PeXI\n",
"asD7gTdSJg3cxvaDoz1Yh6WpKiKiC1r1cXwReC2liWhL2xP9xUgJHBERXdBq5Pgi4O8M/kIk216x\n",
"kwUbiaahZ9SOAAANwUlEQVSmqhcBn7bZqcfFioiY0DrWVGW71ajyiSjzVEVEdMFkCw6tpKkqIqIL\n",
"EjgiIqIt/RY4Mo4jIqLD+ilwpI8jIqIL+ilwpKkqIqILEjgiIqIt/RY40scREdFh/RQ40scREdEF\n",
"/RQ40lQVEdEFCRwREdGWfgsc6eOIiOiwfgoc6eOIiOiCfgocaaqKiOiCBI6IiGhLvwWO9HFERHRY\n",
"vwWO1DgiIjqsnwLHasB9vS5ERES/64vAIbEEsDoJHBERHdcXgQNYBXjU5u+9LkhERL/raOCQtLuk\n",
"P0i6UdJHBtk+R9LVkq6UdLmknRu2rSzpDEnzJV0naYcWh1oDuKcT5xAREU83vVMZS5oGHA3sCtwB\n",
"XCrpHNvzG5JdZPvsmn4L4IfARnXbV4HzbO8laTqwXIvDrQncPd7nEBERi+tkjWMWcJPtW20vAE4H\n",
"5jQmsP14w+Ly1D4KSSsBO9n+Zk230PbDLY6VGkdERJd0MnDMBG5rWL69rnsaSa+RNB/4MXBwXb0+\n",
"cK+kkyRdIel4Scu2OFYCR0REl3SsqQrwiBLZZwFnSdoJ+BawaS3XNsCBti+VdBRwGPCJ5v0lzYXX\n",
"vRi8SDpjtu1543UCERH9QNJsYPZ45dfJwHEHsG7D8rqUWsegbF8sabqk1Wq6221fWjefQQkcg+03\n",
"V+KZwNU288al5BERfaR+oZ43sCzpiLHk18mmqsuAjSWtJ2lJYG/gnMYEkjaUpPrzNgC277f9F+A2\n",
"SZvUpLsC17Y4VpqqIiK6pGM1DtsLJR0IXABMA060PV/SAXX7scCewH6SFgCPAfs0ZHEQ8J0adG4G\n",
"3tbicGuQp6oiIrpC9oi6IiYkSbYtiRuAV9rc0OsyRURMdAP3ztHu3y8jx59BphuJiOiKfgkcSwNP\n",
"9LoQERFTQb8EjiWBBb0uRETEVDDpA4fENACbf/S6LBERU8GkDxzADMisuBER3dIPgWNJEjgiIrqm\n",
"XwJH+jciIrqkXwJHahwREV3SD4EjfRwREV3UD4EjNY6IiC7ql8CRPo6IiC7pl8CRGkdERJf0Q+BI\n",
"H0dERBf1Q+BIjSMioov6JXCkjyMiokv6JXCkxhER0SUJHBER0ZZ+CBwzSFNVRETX9EPgSI0jIqKL\n",
"EjgiIqItCRwREdGWfggc6eOIiOiifggcqXFERHRRRwOHpN0l/UHSjZI+Msj2OZKulnSlpMsl7dy0\n",
"fVrddm6LwyRwRER00fROZSxpGnA0sCtwB3CppHNsz29IdpHts2v6LYAfAhs1bH8fcB2wQotDJXBE\n",
"RHRRJ2scs4CbbN9qewFwOjCnMYHtxxsWlwfuG1iQtA7wcuAEQC2Okz6OiIgu6mTgmAnc1rB8e133\n",
"NJJeI2k+8GPg4IZNXwE+BCwa5jipcUREdFHHmqoAjyiRfRZwlqSdgG9J2gx4BXCP7SslzW6dwxtf\n",
"BI8/Kp29FDDP9rwxlToios/U++js8cqvk4HjDmDdhuV1KbWOQdm+WNJ0YDXgX4FXS3o5sDSwoqRT\n",
"be+3+J7fuRy4weZr41j2iIi+Ub9QzxtYlnTEWPLrZFPVZcDGktaTtCSwN3BOYwJJG0pS/XkbANv3\n",
"2f4P2+vaXh/YB/jZ4EEDyLTqERFd1bEah+2Fkg4ELgCmASfani/pgLr9WGBPYD9JC4DHKEFi0Oxa\n",
"HCpvAIyI6CLZI+qKmJAkGfwd4Hybb/e6PBERk4Ek2271tGpLGTkeERFt6ZfAkT6OiIgu6YfAkT6O\n",
"iIgu6ofAkaaqiIguSuCIiIi29EvgSB9HRESX9EPgSB9HREQX9UPgSFNVREQXJXBERERb+iVwpI8j\n",
"IqJL+iFwpI8jIqKL+iFwpKkqIqKLEjgiIqIt/RI40scREdEl/RI4UuOIiOiSfggc00mNIyKia/oh\n",
"cCywW74hMCIixlFfBI5eFyAiYirph8Bxdq8LEBExlUz6d46P5b25ERFTUd45HhERXZXAERERbel4\n",
"4JC0u6Q/SLpR0kcG2T5H0tWSrpR0uaSd6/p1Jf1c0rWSfi/p4E6XNSIihtfRwCFpGnA0sDvwHGBf\n",
"SZs3JbvI9la2twbeChxX1y8A3m/7ucAOwHsH2bevSZrd6zJ0Us5vcuvn8+vncxsPna5xzAJusn2r\n",
"7QXA6cCcxgS2H29YXB64r67/i+2r6s+PAfOBtTtc3olmdq8L0GGze12ADpvd6wJ02OxeF6CDZve6\n",
"ABNZpwPHTOC2huXb67qnkfQaSfOBHwOLNUlJWg/YGvhtR0oZEREj1unAMaJnfW2fZXtz4FXAtxq3\n",
"SVoeOAN4X615RERED3V0HIekHYC5tnevy4cDi2wf2WKfm4FZtu+XNAP4EfBj20cNknbyDkKJiOih\n",
"sYzjmD6eBRnEZcDGtanpTmBvYN/GBJI2BG6xbUnbANSgIeBE4LrBgkZNl8F/ERFd1tHAYXuhpAOB\n",
"C4BpwIm250s6oG4/FtgT2E/SAuAxYJ+6+47Am4DfSbqyrjvc9vmdLHNERLQ2qacciYiI7pu0I8eH\n",
"G1g4GUm6VdLv6mDIS+q6VSVdKOkGST+RtHKvyzkSkr4p6W5J1zSsG/JcJB1er+UfJO3Wm1KP3BDn\n",
"N1fS7fX6XSnpZQ3bJtv5DToAt1+uYYvzm/TXUNLSkn4r6SpJ10n6XF0/ftfO9qT7UJq9bgLWA2YA\n",
"VwGb97pc43BefwRWbVr3X8CH688fAT7f63KO8Fx2ojxCfc1w50IZHHpVvZbr1Wu7RK/PYRTndwTw\n",
"gUHSTsbzeybwvPrz8sD1wOb9cg1bnF9fXENg2frvdOA3wAvH89pN1hrHsAMLJ7HmDv9XA6fUn08B\n",
"XtPd4oyO7YuBB5tWD3Uuc4DTbC+wfSvlD3dWN8o5WkOcHyx+/WBynt9gA3Bn0ifXsMX5QR9cQ9tP\n",
"1B+XpHzRfpBxvHaTNXCMaGDhJGTgIkmXSXpnXbem7bvrz3cDa/amaONiqHNZm3INB0zm63lQnXvt\n",
"xIamgEl9fk0DcPvuGjac32/qqkl/DSUtIekqyjX6ue1rGcdrN1kDR7/26O/oMmfXyyhzc+3UuNGl\n",
"XtkX5z6Cc5mM53kMsD7wPOAu4Est0k6K86sDcM+kDMB9tHFbP1zDQQYY98U1tL3I9vOAdYB/k/Ti\n",
"pu1junaTNXDcAazbsLwuT4+Yk5Ltu+q/9wI/pFQX75b0TABJawH39K6EYzbUuTRfz3XquknF9j2u\n",
"gBP4Z3V/Up5fHYB7JvAt22fV1X1zDRvO79sD59dv19D2w8D/ANsyjtdusgaOpwYWSlqSMrDwnB6X\n",
"aUwkLStphfrzcsBuwDWU83pLTfYW4KzBc5gUhjqXc4B9JC0paX1gY+CSHpRvTOp/xgGvpVw/mITn\n",
"12IAbl9cw6HOrx+uoaTVB5rYJC0DvAS4kvG8dr3u/R/DUwMvozwJcRNlYGDPyzTG81mf8mTDVcDv\n",
"B84JWBW4CLgB+Amwcq/LOsLzOY0yW8DfKf1Rb2t1LsB/1Gv5B+ClvS7/KM5vf+BU4HfA1fU/5ZqT\n",
"+PxeCCyqf49X1s/u/XINhzi/l/XDNQS2AK6o5/Y74EN1/bhduwwAjIiItkzWpqqIiOiRBI6IiGhL\n",
"AkdERLQlgSMiItqSwBEREW1J4IiIiLYkcMSkJWmRpC82LB8q6YhxyvtkSXuOR17DHOd1derrnzat\n",
"f7akfYfaL6KXEjhiMvs78FpJq9Xl8RyUNOq8JLXzZs23A++wvUvT+vWBN4xD/hHjLoEjJrMFwHHA\n",
"+5s3NNcYJD1W/50t6ReSzpJ0s6TPS3qzpEtUXqK1QUM2u0q6VNL1kl5R958m6Qs1/dWS3tWQ78WS\n",
"zgauHaQ8+9b8r5H0+bruE5RXJH9T0n817fJ5YKf6MqFDJL1F0jm1ZnJhnaLmm/WFPVdIevUw5VtL\n",
"0i9rftdIeuEof+cRnX3neEQXfIPyXvrmG29zjaFxeUtgM8o7Cv4IHG97lspb4A6iBCIBz7a9naSN\n",
"gJ/Xf98CPFTTLwX8StJPar5bA8+1/afGA0tamxIItgEeAn4iaY7tT9ZZSz9o+4qm8n4EONT2q2oe\n",
"b635b2H7IUmfBX5qe/86L9FvJV0EvGmI8u0BnG/7s3WepuWG/9VGDC6BIyY1249KOhU4GPjrCHe7\n",
"1PW9BJJuAi6o638PDEw/beD79Rg3SbqFEmx2A7aQtFdNtyKwEbAQuKQ5aFTbUd6JcH895neAfwPO\n",
"rtsHe3FQ8zoDF9p+qC7vBrxK0qF1eSngWS3KdymlZjMDOMv21YP+ZiJGIIEj+sFRlEndTmpYt5Da\n",
"FCtpCcqb0Ab8reHnRQ3Li2j9f2Kg1nKg7QsbN0iaDTzeYr/GQCCeXgMaaX9Kc/572L6xqRyDlq9u\n",
"2wl4JXCypC/b/tYIjxvxNOnjiEnP9oOU2sHb+edN+FbKOwigvDJzRpvZCnidig2BDSgzh14A/PtA\n",
"B7WkTSQtO0xelwIvkrSapGnAPsAvhtnnEWCFpvI0uoBSy6KWY+uG9YuVT9KzgHttn0B5z8TWRIxS\n",
"ahwxmTV+U/8ScGDD8vHA2SqvzzwfeGyI/Zrzc8PPf6a8l2BF4ADbf5d0ArAecEXtK7iH8t6GId+o\n",
"ZvsuSYcBP6cEgB/ZPneYc/sd8I9a/pMp/TGN+X8KOErS7yhfAG+hBMihyjcb+JCkBcCjwH7DHD9i\n",
"SJlWPSIi2pKmqoiIaEsCR0REtCWBIyIi2pLAERERbUngiIiItiRwREREWxI4IiKiLQkcERHRlv8P\n",
"hT8XXtGsQiUAAAAASUVORK5CYII=\n"
],
"text/plain": [
"<matplotlib.figure.Figure at 0x2278ba8>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%time\n",
"learning_curve_ndcg(rfr,Xts,Yts,Qts,rank = 10)"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"Gradient Boost (mse)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 1min 39s\n"
]
}
],
"source": [
"%%time\n",
"from sklearn.ensemble import GradientBoostingRegressor\n",
"gb = GradientBoostingRegressor(learning_rate=0.06 ,loss='ls',n_estimators=150, max_depth=6,\n",
" verbose=0, subsample=0.5, random_state=42)#verbose >= 2 to show OOB scores\n",
"gb.fit(Xtr, Ytr)\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train:\n",
"NDCG5 : 0.622219101762\n",
"NDCG10 : 0.631239990375\n",
"NDCGfull: 0.716050528964\n",
"rank AUC: 0.859756636547\n",
"MSE : 0.200612338366\n",
"test\n",
"NDCG5 : 0.423640126452\n",
"NDCG10 : 0.451001178021\n",
"NDCGfull: 0.61496848539\n",
"rank AUC: 0.686001882485\n",
"MSE : 0.300900600008\n",
"Wall time: 1.42 s\n"
]
}
],
"source": [
"%%time\n",
"print \"train:\"\n",
"score(gb,Xtr,Ytr,Qtr)\n",
"print \"test\"\n",
"score(gb,Xts,Yts,Qts)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 3.83 s\n"
]
},
{
"data": {
"image/png": [
"iVBORw0KGgoAAAANSUhEUgAAAZQAAAEZCAYAAACw69OmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
"AAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYXFWd//H3h7DvSxRCCJuAoIIsJqMiY2TRiEIQUIgi\n",
"qKiRGSLiqARxJOMeF8CRkR8KAuICCAgBlRCRMDgiiwQIkJCwREmAsAgCIpKQz++Pcyq5qa6qru6u\n",
"W1Xp/r6ep5/Uvffce091p+63zi7bhBBCCAO1WqczEEIIYXCIgBJCCKElIqCEEEJoiQgoIYQQWiIC\n",
"SgghhJaIgBJCCKElIqCEUIOksZIe7tC9z5L0hU7cO4SBiIASOkbSAkmLJa1b2PdRSdcXtpdJel7S\n",
"c5KelPRbSe+rca13SPpfSc9KelzSTEkHFY6PkPRDSYvytR6QdJ6kV5f/TvvG9nG2v9LpfITQVxFQ\n",
"QqetBpzQS5rdbG8A7AScD5wp6YuVg5IOBy7Jx0bafiXwReCgfHwz4A/A2sBb8rX2BG4ADmjlm+mN\n",
"pNXbeb8yDIb3EMoRASV0koFvA5+RtFGvie2/2v4JcBxwsqRNJAk4DfiS7R/Zfi6n/V/bH8+nngg8\n",
"Y/uDth/Kx/9m+3zbZzaTUUlbSrosl34elDSpcGyMpJskPS3pEUnfk7RG4fgySf8maT5wn6S3Sloo\n",
"6dO5hPaIpA8V0p8v6cv59dhe0m4m6SpJf5N0i6SvSLqxwft4i6Q/5Lz+RdLRef9MSccW0n2oeJ3C\n",
"e5gHzJP0fUnfqrr2lZJO7O33FQavCCih024DZgKf6cM504DVgTHAq4GtgEsbpN8f+GU/84ek1YCr\n",
"gFnAlsB+wKckvT0nWUoqZW0GvCkf/7eqy4wHRgOvAQRsDmyYr3cs8D+FoOr8U9Eo7f8Az+U0xwBH\n",
"V51bfB/bAL8GvgsMB3YH7qxzz1rGk37nuwA/B44oXHsTUmnv5038vsIgFQEldJpJ1VOTJA1v6gR7\n",
"CfAksCnpIQ7waINTNgMeq2xIOjh/Q39W0vQmbjkaGG77K7aX5lLOOcCROT+3277F9jLbfwZ+ALy1\n",
"6hpft/2M7X/m7SWkUtXLtn8DPE8KjsuzWXhdM62kYcChwKm2X7Q9B7ig6tyi9wMzbF+cr/VX23fW\n",
"SVtL8T38HrCkffKxw4E/2H6st99XGLyiLjR0nO17JF0NTAbm9JY+Vye9Avgr8FTePQL4c51TniJ9\n",
"U67cbxqwSa7iOaqJLG4DbCnp6cK+YcD/5vzsRKp22wtYl/S5uq3qGtU9xp6yvayw/QKwfr3810n7\n",
"inyv4rUXNngfWwEPNjjem+X3sW1JFwETgBtJwerH+XDD31cYvKKEErrFqcDHgJFNpB1Pqma6BbiP\n",
"9KA7vEH664BDcntLUb1v8tUeBh6yvUnhZ0Pb787HzwLuBXawvRFwCj0/W32d1ruZ9E+Qfg+jCvtG\n",
"1UkL6X28qs6xvwPrFba3aCJPPwcOz1VpY4DL8v6/0Pj3FQapCCihK9h+ALiY2j2+BCBpU0kfAM4E\n",
"vmH7aaf1Fz4N/GduSN5Q0mq58fnsfP5pwCbAhZK2V7IBqQ2hmQf3LcBzkj4naR1JwyS9TtIb8vH1\n",
"Se0YL0jamdRpYCBEE8HO9svA5cCUnK+dgQ9S/z39FNhf0nslrZ4b9F+fj90BHJqvswOpraa3+99B\n",
"qno8B7jG9rP5UG+/rzBIRUAJ3eRLpCqj6gfinZKeA+YDHwE+ZXtK5aDty0gNxB8BFpHaS74EXJGP\n",
"PwW8EXiRVPf/LKnBeD0aP/ydz38ZeDcpAD1IKhn8gNRQDqlDwfvzdX8AXFT1Hmo94BsFsuoG8kZp\n",
"jwc2Ir3nC0ilhpdqXtR+GDgQ+A9SNeAsYLd8+PR83mLgPOAnTebhZ8C++d/KfZbR+PcVBimVucCW\n",
"pHHAGaT603NsT62TbjRwE3BEfjggaQHpA/oysMT2mLx/CvBR0n9SgJNtX1PamwhhFSJpKvBK2x/u\n",
"dF7C0FNao3zugXImqcvmIuBWSdNyT5TqdFOB6qBgYKztv9bYf5rt08rJeQirDqWR/msBs0m9qz5C\n",
"E9VVIZShzCqvMcD9thfkbp4XkRpTq00ijSF4osaxevXIzTamhjDYbUBqDH+e9Bn7du7FFkLblRlQ\n",
"RtKzO+NKPXgkjSQFmbPyruo6299Kuk3Sx6quPUnSnZLOlbRxi/MdwirD9m22d7S9nu3t61Urh9AO\n",
"ZQaUZhpnzgAm55461T1b9ra9B/BO4N8LA6jOArYjNfg9CnyndVkOIYTQX2UObFxEz/7x1YOu9gIu\n",
"ysMDhgPvlLTE9jTbjwLYfkLSL0lVaDfafrxysqRzSFM89CCpvN4GIYQwSNnuf5OC7VJ+SMHqAWBb\n",
"YE1SP/ddGqQ/Dzg0v14X2CC/Xg/4P+DteXtE4ZwTgZ/VuZ7Lem897+UDwM+BP9nH39GUduVxgH/L\n",
"yGfks2t/Ip8tzaMHcn5pJRTbSyUdD0wndRs+1/YcSRPz8bMbnL4FcHkuuawO/NT2tfnYVEmVAWkP\n",
"ARPLeg/NkDgM+D6pY8HuncxLCCF0UqlzeTlNZPebqn01A4kL/eZtP0idh7Pto1uZxxY4gTQu5kng\n",
"ex3OSwghdExMDjlww0klpQXAayRWt1na5Lkzy8pUi83sdAaaNLPTGWjSzE5noEkzO52BJs3sdAaa\n",
"NLPTGShbqSPlO0mSPZDGpabvw2Jgd5tHJeYD423uLfu+IYTQagN9bsZcXgMgIdKaHJUp1O9ixdxI\n",
"IYQwpERAGZgNgRft5ZPx3Qm8vkH6EEIYtCKgDMxmrCidQASUEMIQFgFlYDYj9e6qiCqvEMKQFQFl\n",
"YIazcgllAbCBtHyd864jsYfEGp3ORwhh8ImAMjArVXnZmFRK6Xi1l1R3ffJLgLFtzEoIYYiIgDIw\n",
"1W0o0AXVXhJbAk9LXC2xT2H/+qQ1xbfrWOZCGEQkJDFJ4tQGaYZJ/Ls0+J+3g/4NlqxWQJkH7NCB\n",
"vBS9BbgOmAZcLbF53v9a0ozO23cqYyEMFrnq+P+Rpn86UWKTOkknkRYb3KNdeeuUCCgDUyugPEjn\n",
"H9h7A7+z+QFpaeU35v27Ac8xxEsoUswQEVriC6TP+puAXwMfrE4gsU1O9yvg7W3NXQdEQBmY6l5e\n",
"kGZY7nRAeQvw+/z6j6wcUH5D5/PXMVJaRiF/0EPoF4k1gY8Dn7R5DjgbmJgHO1fSiDRx7OmkdZwi\n",
"oISGapVQFgDbSgxrf3ZAYgPg1cCf8q6bSN+gAHYFrmSIllDyB/xMYH3gbR3OTli1HQbcazMnb/8v\n",
"6Xm6N4DE2sCFwCbAt4AbgNENOssMChFQBqZHQLF5AfgrsGVHcgT/Asyy+WfevgXYK1fz7EZqW1lH\n",
"YsMO5a+T3gPsBJxC9HQLfSTxNYkv58b140lfToDlPTx/AHxP4jTgemAtYH+bl2yeB24F/rWf915T\n",
"Yq0Bv4mSRUAZmOpxKBU92lEktpY4oVgkLkmxugubp4GHgXHASzaLSbMjD6lSisRw4L9J1RTTiYBS\n",
"k8TRUR3YU67i+gRwIHANaQXa6tVizyJVby0CfgQckb9gVlxLH6u9JN4k8TDwPDBXYtPCseF9fR9l\n",
"i4AyMLWqvKB2O8rnSEXfT5ecp5UCSvZHUk+Uu/J2N3QcaJtc/fhT4Gc2NwL3AWtJbNvRjHWZXE3z\n",
"feBSibUkVpM4VeLwTuetC+wP3Ev6fD0BfKt6mQqbF21+bPMdmx/aLKu6Ro+AInFAL4HhKOAc0iq2\n",
"VwA/zF2VJwI3d1uppdSAImmcpLmS5ks6qUG60ZKWSjqssG+BpLskzZJ0S2H/ppJmSJon6VpJG5f5\n",
"HurnmbWBNUjfHKo9SBrvUUm7EfB+YB9S98JSPqC5WmsMqd2k6CbSN6tKQFllSygSo/Lvvi/+k1T9\n",
"8HlYXj0xky5uR8kPjXZ3M90PmEUq0Z5OWpb7IOD7EmPanJdu8z7gEpt/2HzA7tdierOA4RK7wPLn\n",
"wqXAoQ3OeTtwRQ5ek0lfBK8FPgO8o1C13RVKCyiShpHqGMcBrwEmSNqlTrqppGJkkYGxtvewXfzP\n",
"PBmYYXsnUnvA5DLy34TNgCfzw6ladQngI8A1NjcD44GzchF6QCTeLbFI4qsSHySVRG62+WtV0j+S\n",
"/taz6+SveE0NtFouf+saOZBr1Lnuq4E7SN/amj1nV1JVxZFV3yhn0t3VXnsDt7T5G+ghwC9J/1/H\n",
"Aa8A3kpakfTyPGB2yMl/g4OBywZynVxi+Q7w5bzrE7C8bbPWfbcHNiB/EczB40hSCenNNvcPJD9l\n",
"KLOEMga43/YC20uAi0gP02qTSFH6iRrHaj3YDgYuyK8vIH0IOqFedRcUqrxydcvxpPp7bP5EqnJp\n",
"RRfCNwC/BdYjPWS/BLyjRrp7gWdproTycWBGf8dq5AbLc4H39uf8BtfdnNTXfz7Q44tJId2wwkBO\n",
"SN/kzrB5rCrpTGBsG9q0+utjpIfNzu24Wf67HQRcafMMaQnug2z+bjONNIDvkiE6D9wBwN02i1pw\n",
"rf8G3ijxVtLy4f9J6n1Zy9uBa4tfWm3us3m/XfN52XFlBpSRpKJzxcK8bzlJI8nf2POu4rd9A7+V\n",
"dJukjxX2b257cX69GFZ6eJRO4qe5YaxRQClWeb0rp7u5cPynwAdakJ1XAzNsPmXzDptpNeptsXmZ\n",
"1HX4jkL+6rWh7E96mNSdSqIX+5AaLPfq5/k95If+xcBPgK+T3nc9HwPulhgpsRXpIXl2jXTzSA/s\n",
"EyTWbVVeWyGPuB5PKoG3axqffwGesHkAwObZ/P+m4mukQbFfrnXyYJUD7ceAX7Tiejb/IH22riR9\n",
"Hn8C7Frni807SB1IVhlljhhuZm3hM4DJti1JrFwi2dv2o5JeAcyQNNf2jSvdIJ1X9z6SphQ2Z9qe\n",
"2Xz2a12PNYAJpFGvL1E/oCwG1s1dcz8LfKeqauwXwNcl1s/dCftrZ1IRuldVyxIvII2VWa0YgPJ/\n",
"6jeTHsKXStxoc20f83QU8HNgzz6e18ihwEakEtgONP7WfixwG6lEfBtwQe7pthIbSxwEfBH4vMTT\n",
"pC8nH7b5ZQvz3pDEFqSS5q8K/0c+QKoCvpP2BZRDSA+5mmyWSRwN3C5xg81v2pSvjsml9HNIq7Ke\n",
"18JLX0D6nHzF5nGJJaRhBstLQPlZ8zZSZ5rSSBpLK6t+bZfyQxqdfU1h+2TgpKo0D5KqXx4ifftZ\n",
"DBxc41qnAp/Or+cCW+TXI4C5de7v1r8nbwM2+MfgieAfNEg7G3wc+EHw6jWOXw0+agB5WQ38d/CG\n",
"/Tx/MXjLqn1b5/0CHwS+vY/XXBv8FHh78Avg9VrwO18L/AB437y9BvhF8Fo10u4KXpjTXANeAt6m\n",
"iXtsB94F/GXwt1v9/6aXe381/66uBY8Bbwm+E7wf+F3g6W3Kx1zw6CbSHQb+XTt/R/18P28Ev3YA\n",
"5wv80/x3GfD/417u9VvwuKp9b+nr5681ecEDOb/MKq/bgB0lbStpTeAI0mSFy9ne3vZ2trcjtaMc\n",
"Z3uapHUlbQAgaT1SXeLd+bRpwDH59TGkrnTtMopUdfcO4JXUL6FACpZfAU5zVffCbKDVXlsBz9g8\n",
"28/zH6LQEy17E/AHG5PaK7aUeqRp5EDgLpsHSe02rZjG/99JI5J/B2CzhFTCqjUB54eB83Oao4AP\n",
"2Py5txvYPOQ04vkm2jhTdC4RvpdUzTgdOJ80w8HzpIFxbZm5WmJn0uwBf+otLankNDr3UOpm3yR1\n",
"9umvk0ifj4Nt/t6aLNU1m0I7Sv5/8TlaVM3WTqUFFNtLSY3R00kPl4ttz5E0UVJvxbgtgBsl3UFq\n",
"e7jadqXq5RvAAZLmAfvm7XYZBfwf8DTpIVA9j1fRg6Rqv3pF5atJjcL9/Ru8mtS431+Xkfq0F9tS\n",
"3kzucuxUf345fWtcn0AKlAC3M4BqL4mNJE4n9eL7bNXhuVRVe+Vec0eRHsrYPGlzSR9v27IHuMQ+\n",
"TUyz8TpSd+abnMYuvMZmhM3eTlWRC4G1JV7Zijw1MB5qt79Vyw/X/yM1VHcliRGkB/SbK70NJdZv\n",
"9OUoj0S/X+J7EscAnwQOt3mxDVleKaCQJpnchiars7tJqeNQbP/G9qtt72D763nf2bZ7NJLa/rDt\n",
"y/PrB23vnn9eVzk3H/ur7f1t72T77bafKfM9VNma1NHgN6QpFBqVUKYDn6/37cZpQrmnSEGqP3Ym\n",
"PVj7xeZbpG7dv5eWlyTeBPyhkOwSUv/7Zu1JmtMI0rfdfgWU/A3tVmBD4LV2j/d5HzmgSOwrcQ1w\n",
"IzDHA+tKuQhYs6qXWJ/lRv5fA/+vl15khwGX5RJhD3n/XdTvBdQqh9C3kv6vSKXRbvUeUh5/AXwo\n",
"7zsHuCcHjE1rnLMzqQ33eVIHhAk2C9uQVygElBwAvw0cY/NSm+7fMjFSvm9GkQJKZcxM3YBic43T\n",
"9PGNzAd27GdeBlpCweZMUrfaX+YxBq9l5WqPG0nVXr3mMTcibkWqjoJUQulvT6+tgA1tjnXt7pFz\n",
"WdHT6z9JE++dTHpA91vhAV5vXMBmeVxLb95FGsS2O6karniNYVoxcejhpKreRkqt9srf5ncmdaNu\n",
"1q+BAwdQui7b4aRgcg7wEYn3kf4WO5C62Nf6ne8G3Gpzss1Imxvallu4B3i1xHqkziT/bS/vkblK\n",
"6db/EN2qElBuAP5B4xJKMwYSUAZUQqmw+Rmp+u06Ul/7fxSOvUz68DVT7bUN8EjhW9VsYMd+jGqH\n",
"9OFv9IG6D9g5dwvejdRO9Tu7YRVks2r2rJJ4B+k9NdPbZwKp6u19wNQ8ILPif4D5El8kzUT7x16u\n",
"VXY7ykGkQbdNfxt26lr8DE2UQCXWlfi1xIfaMeYnVw/uSRpNfhvwd9K8Wh/KJY7jSW1AG1Sd+nrS\n",
"377tnOb7WkSaG+xxUglplRQBpW9GAQ/netVjGfh/wD4FlDwCff+8OeASSsFnSAMfq+cAg9TW0szg\n",
"0R1gRXVT/h3No3/VNc0ElFeTHty/dGunn+jxAJf4AOnb7gmkIFn3wZgbq/cDLnfqqn06qRSFxCtI\n",
"nVM+RapePKeJdouyA8p4GnQXbuBXpJLYSiQ2kbhPWj6r7unAy6R2sJ9IvKbkwHII8BunKVIMfBX4\n",
"kp0Cd35430bPWX93Y8XA3064C9gYOLqZtqyu1e5uaatK97fa1/QT4Fe28HqHgK9qMq3Ad+Q8bJe7\n",
"mq7WwrysB16/xv51cvfkdXo5/3jwWVX7fgT+RD/ychn4yF7SPA7+M/htLf4bjwbfUdjeAPwIeEze\n",
"fhK8eYPzPwS+orC9YT5nB/Ap4HP78Xd5oVbX8xa8143BfwNv1I9z3wq+s8b+o0hd5p8Anwaen3+H\n",
"64K/BX4o/z73bvX7yff/JXhCL2lOAZ9Wte9R8Kgy8tRkvrcFD+/U/VfkAw/k/CihNEliHVLXylZO\n",
"eXA/zZdQ9iI1Ul9A6n013y38JuM0xUaPQZZOVWD3kAbfNbJSCSXrb0+v3kookEopq7OiE0CrVOqz\n",
"K1OMnARcZ1OZoLS3UuUE0sBOAJy6dX+ftAzsvwHf7UtmnDp1/JnUvjUgEjtLXFVowzmU9N7+1o/L\n",
"3QhsIvUoPY0HTiNNkXQIqXH7OZsXbD5rsx2pSqesWbffwMqzUtRyHakUCSyvJlsL2tYI34PNArem\n",
"yrajIqA0bxSwyG5qBoBmPUAasd5jxgKJzSXOl5Z3z/wYaY6sL5IC24DbT/qguOpjPa8ivZ+iPvf0\n",
"yrMLbEF6cDdyL3CRV54eZMCcqkT+TAoq2wPHkRr8K+aRFunqQeK1pGBYvU7Gd0kNxffZ/apWuRUY\n",
"3Y/zqn2FNKbr3Xn7/cDP+nOh/GXmpxTWUc/tZQcAV9vcZLO9zW01Tv8xsJ9avJ5H7p23HmmMVSO3\n",
"AdsUumPvRho/1crP9pAUAaV5lQb5lsnf/h8ndUcGQGJjiQ+R2mdeBn6qNHX4e4Hz8gPvUNK0Ne3S\n",
"TECpVUK5C3iN+jaz8m7A7CYCxWRScC3DXaRG9duBKV65++g86pdQTiF1ECguqoTNU8CJ+Xh/rBRQ\n",
"JM7u44BTJPYkjTP6BPDp3KtvL1JbSH9dCLy/UOLZl/S3a1iKz6W2q2jNfHZFewF/6i0wOA00voGU\n",
"X+h8+8mgEQGleS0PKNl8ckOvxOX5HkcC77E5llQ1cANwo80jADaz7R5rnpTpJuBN9RpT8wNlO9Jg\n",
"zuVydc1D9K26ppnqLmyednkjmM8iBZTt3XPdi/nUKKHk0eb7k6q3enBacKm/f7PlASVPHPlR0mSO\n",
"ffEVUgP1hcC2pMXernChV19fOXU6eIwVD+bxND+e5TxSl15JrN3P3oDV3kBzo/0hVXtVOrh0rIfX\n",
"YFPm5JCDTakBBVhKelC90it33f1J7uLY7AelDH8BlpEeRLWqE7YCnqr+Zp5Vqr1mNXmv3ense8Xm\n",
"etLUJ7XUq/I6Bfiu04DVVruD1E16bVKV1Wo0mMK/msS/kNYkeo/NUon/Jg2ea8Vo9wtJk2sOJ7Wb\n",
"7NPkeTNJbYLnk3qLLZWYCpzl/o9O3yvnpxm/Ar4o8StSCeXMXtKHJkQJpXmjSA/WVqsElEnA92p9\n",
"Y7Q5q9Ao3Ha5CqFRtder6FndVVF3gKPEeoXupRVNlVA66H7gVcVBfRJvJn3bLeWhlB+wc0m/mwNJ\n",
"DeJ1Z1yW+LjEtwu7JpEGy1W6V59D6txRL2j2xY9IY2neQ+qu29RMBbkN5gvAI6TS1wHAO0nrrvTX\n",
"XjT5ZcRpLM07SeuTvJbUGSMMVKe7qXVr97ca3fquAR/Y+nz64NzN8klKntV0gPn8DPh7hW2RZvcV\n",
"+OP1usOC/xX8xzrHvgb+BzjPHu0NchflHt2Xu+kHvAi8dSHPD4DHl3zPs8GfIs0GfQj47jrpNsld\n",
"qv8K3hG8Ofhp8Cad/r018R63ze9P/Th3C9JM1306N593XKffe7f8DPS5GSWU5pVZ5fU60rodZc9q\n",
"OhA3Ae+SOFBiB1Kj6ixSV9haDfIVs0gLCK1UvSoxirTWw9Ws6EL6edLcVgNZI6YditVepwEz7X4N\n",
"DuyLW0k9/Z4izRP3qlq9A0ndnK/M+TqV1N5ymWusCdNtbBaQlrF4XW9p1XPlyKYa5Gvc8zF7+QJ/\n",
"YYCiDaV5ZQWUB4EXqNOY20VuIo16PgXYgzSW4DOkDgPPkEeDV7N5TmIhqc5/duHQV0iN3z8A7pC4\n",
"hPTAbNv08QNQ6UixLmk8Qyum6e/NraR2kO/Y/EPiUVJHiOXdq/NUNJXf4bOkIP8OWrPcdLtUGstn\n",
"10sgsQcwU+K/gNNzEOlLg3woSQSUJuRpyIdBvwaANWTzT4mRTut4dy2nOu/vAd9TYaVHpVX8fk39\n",
"EgqsGOA4O5+zK+kht1MOOJcDvwOmOvdk63LzgLeQeje91+U0xFe7hzR/3K/zdmUK/+J4nVOAHzqv\n",
"fS7xdeBQu+kOEd3gt6R1jk6vdTB3d76S1P4yEdhJYgFwND2XOQhtFlVezdkCeLSvxelmdXswqebC\n",
"CH2b6aRvxI0eWjex8txJR5IWwqo8iL9Oatg9rcVZLcs80qDA8+2a85+1nNPYiQNh+Sy4cyj09FJa\n",
"SvgIVv4dfpcuXrekjuuBfaqrtHL34reQqkjPdurOvTewBunz+Vnau9heqCFKKM0ZQepvH2qwe+0h\n",
"Mw34gsQwpwGL7yaNQK+c/wCrVrXMbaS1Yk5t503tlaaYn8vKve4+CfzM5vFCekNLJ84snc2TEg+S\n",
"en79IXeZP4r0/2VNUq+ss3Lav5EmaQ1dotQSiqRxkuZKmi/ppAbpRktaKunQqv3DJM2SdFVh3xRJ\n",
"C/P+WZLGlfkesi2IgNJvubF1EbC3xNbAlvQ+31LXsnnE5gh3dgGkueQSSp7h+OOsgiv81fFb4DiJ\n",
"c0jT4OxHmqF5F5vvl1VTEAautBKKpGGkfvn7kx4mt0qaZntOjXRTSYtWVY/EPoE0Z1Nx7QIDp9lu\n",
"Z/VIBJSB+yVprMI80niFls7BNQTNIQ12FKmX3HS71zmsVhW/JJVELgZet4q0qwXKrfIaA9xvewGA\n",
"pItIUzPMqUo3ibSI00qT30nailRn/FV6zkxa+kI9VSKgDNwVpKqve0iD6sIA5Kqhl4Fvklaq3LeX\n",
"U1YZNn+g99mtQxcqs8prJCt3s12Y9y0naSQpyFT6gReLsqeTGtpqTdE+SdKdks6VtHHrslxXBJSB\n",
"m02a7HI/0mp6YeDmkKYt2SdXK4bQUWWWUJqp5zwDmGzbkkQueUh6N/C47VmSxladcxbwpfz6y6R6\n",
"45oNc5KmFDZn2p7ZdO5XNgJ4tJ/nBlIDscQvgT1XtV5tXewTwGP2gJeiDkNUfr6Obdn18nD7lpP0\n",
"RmCK7XF5+2Rgme2phTQPsqL6ajhpgN/HSTOpfpA0YeLapEnkLrN9dNU9tgWust1jmVlJtt2SqjGJ\n",
"24GP2TFwaiDyTLmbucn5nkII7TXQ52aZAWV10qp6+5EmgLsFmFDdKF9Ifx4pOFxetf+twGdsH5S3\n",
"R9h+NL8+ERht+/01rtfKgPJIuk8aMBZCCIPRQJ+bpVV52V4q6XjSvEPDgHNtz5E0MR8/uy+XK7ye\n",
"Kmn3vO8h0mjZ0uS1Pl4BK/r3hxBC6Km0EkqntaqEkpcJvcfmFS3IVgghdK2BPjdj6pXeRYN8CCE0\n",
"IQJK76LLcAghNCECSu8ioIQQQhMioPQuAkoIITQhAkrvIqCEEEITIqD0LqauDyGEJkRA6d0WRC+v\n",
"EELoVQSU3kWVVwghNCECSu8ioIQQQhMioDQgsQ6wDsTsuCGE0JsIKI3tACyIJUdDCKF3EVAaG0Oa\n",
"JTmEEEIvIqA0NpoIKCGE0JQIKI2NAW7tdCZCCGFVENPX1z2fdYCngE1tXmxdzkIIoTvF9PXl2QO4\n",
"N4JJCCE0p9SAImmcpLmS5ks6qUG60ZKWSjq0av8wSbMkXVXYt6mkGZLmSbpW0sYlZT8a5EMIoQ9K\n",
"CyiShgFnAuOA1wATJO1SJ91U4Bqguqh1AnAvKy8BPBmYYXsn4Lq8XYYIKCGE0AdlllDGAPfbXmB7\n",
"CXARML5GuknApcATxZ2StgIOBM5h5UBzMHBBfn0BcEiL810RASWEEPqgzIAyEni4sL0w71tO0khS\n",
"kDkr7yrXgI6CAAAaaElEQVSWRE4HPgssq7ru5rYX59eLgc1bleEV+WIz4JXAfa2+dgghDFarl3jt\n",
"ZrqPnQFMtm1JIpdEJL0beNz2LElj694gnVf3PpKmFDZn2p7ZTMaBXYB7bF5uMn0IIaxy8vN1bKuu\n",
"V2ZAWQSMKmyPIpVSivYCLkqxhOHAOyUtBf4FOFjSgcDawIaSfmz7aGCxpC1sPyZpBPB4vQzYntLP\n",
"vG8N/Lmf54YQwiohf8meWdmWdOpArldmlddtwI6StpW0JnAEMK2YwPb2trezvR2pHeU421fa/rzt\n",
"UXn/kcDvcjAhX+OY/PoY4IoS8r41K1fXhRBC6EVpAcX2UuB4YDqpp9bFtudImihpYl8vV3j9DeAA\n",
"SfOAffP2gEhsKq3UFjMK+MtArxtCCEPJkB4pL7EN8DXgMOAqm/fm/VcB59hcWX5OQwihO8RI+YGZ\n",
"BLwMvAvYubB/FFHlFUIIfTLUA8quwMXAzcAO0vLfx9ZElVcIIfTJUA8ouwF32TxPmghya4n1ST3L\n",
"nupozkIIYRVTZrfhribxClLgqHRlvg/YibTk78OxSmMIIfTNkA0opOqu2YXAMY8UUERUd4UQQp8N\n",
"5YCyG3BXYXse8GrgRSKghBBCnw3lNpTqgFKp8opBjSGE0A9DOaDsCswubFdKKNHDK4QQ+mFIBhSJ\n",
"YaQ1Wu4u7F4AbEEqpURACSGEPhpSbSgS7yV1B14EPGbzXOWYzVKJh4DRRJVXCCH02VAroUwAfkNa\n",
"5XF2jePzSEE2AkoIIfRRwxJKXq99HCsWxloITLf9TNkZK8m6wJeAE4Hv1zh+H/CkzQttzVUIIQwC\n",
"dUsoko4G/kRafGWd/LMvcLukY+qd1+XWBX4PvJ60ImS1eUTpJIQQ+qVRCeULwF7VpRFJm5DWWr+g\n",
"5lndbV3gBZtFdY7/DlijjfkJIYRBoz+N8qvylCTrQv3qLJsHWbG+fQghhD5o1Cj/VeBPks6SdEr+\n",
"+X/A7aQ1RHolaZykuZLmSzqpQbrRkpZKOixvry3pZkl3SLpX0tcLaadIWihpVv4Z19xbBVJA+Xsf\n",
"0ocQQmhSwwW2JG0KvAPYMu9aRGqUf7rXC0vDSI3c++fzbgUm2J5TI90MUsnhPNuX5f3r2n5B0uqk\n",
"do//sP1/ec3j52yf1sv9eywUI/E48Dq7/jr0IYQwVA10ga2GVV62/wr8vJ/XHgPcb3sBgKSLgPHA\n",
"nKp0k0jryY+uunelampNYBhQDGL9fcMNq7xCCCH0X7/GoUiqNYaj2khW7jG1kBXdjyvXGUkKMpV2\n",
"CxeOrSbpDmAxcL3tewunTpJ0p6Rzc9fmJvKMSAHlH82kDyGE0Dd1SyiV9owqJpUORjRx7WYa788A\n",
"Jtu2JFEoedheBuwuaSNguqSxtmeSgs+XcrIvA98Bjm3iXmsDL9m83ETaEEIIfdSoyusi4GfAsqr9\n",
"Ij2ce7OItDZ7xShWLGZVsRdwUYolDAfeKWmJ7WmVBLb/JulXwBuAmbaXt39IOge4ql4GJE1ZsbXb\n",
"7XBnVHeFEEImaSxprGFrrlevUV7S7cAxtntUb0l62PaoGqcV06xOapTfD3iENHalR6N8If15wFW2\n",
"L5c0HFhq+xlJ6wDTgf+yfZ2kEbYfzeecCIy2/f4a11upcUliFHCTzVaN8h1CCENVmY3ynwKerXPs\n",
"0N4ubHuppONJwWAYcK7tOZIm5uNnNzh9BHCBpNVI7TwX2r4uH5sqaXdSldpDwMTe8pJFl+EQQihR\n",
"w27Dq7IaJZQ9gB/Z7NHBbIUQQtcaaAmlT728cjXYqiq6DIcQQon62m2435GrC0RACSGEEvU1oPy6\n",
"lFy0RwSUEEIoUZ8Ciu1TyspIG6xHBJQQQihNo/VQPirpc4XtRZKek/S8pOPak72WihJKCCGUqFEJ\n",
"5RPAjwrbj9vegDQAcUKpuSpHdBsOIYQSNQoosv1kYfsXALZfJK3euKqJEkoIIZSoUUDZqLhh+2uQ\n",
"Jm0ENiszUyWJgBJCCCVqFFBmSPpKcUeewPHLwLWl5qocEVBCCKFEjaZe+SxwjqQHgDvzvtcDtwEf\n",
"LTtjJYiAEkIIJaobUGw/Dxwp6VXAa0lzZ82xfX+7MtdiEVBCCKFEjdZDGQdsYPsXwAOF/YcDf7M9\n",
"ow35a6UYhxJCCCVq1IbyReCGGvtvILWjrGqi23AIIZSoUUBZq7iYVYXtJ0jf9lc1UeUVQgglahRQ\n",
"NpC0RvXOvK+ZFRu7TQSUEEIoUaOAcjnwA0nrV3ZI2gA4Ox9b1URACSGEEjUKKP8JLAYWSLo9r4Xy\n",
"EPAE8IVmLi5pnKS5kuZLOqlButGSlko6LG+vLelmSXdIulfS1wtpN5U0Q9I8SddK2riZvBABJYQQ\n",
"StXrio2S1gVelTfvt/2Ppi4sDSOtKb8/sAi4lRpryud0M0gP+/NsX1a5r+0X8tr0vwf+w/b/Sfom\n",
"8KTtb+YgtYntyTXuX71i46PAnjaPNpP/EEIYaspcUx5Jw4H3AzvnXfdK+rntp5q49hhSAFqQr3UR\n",
"MB6YU5VuEnApMLq403alNLEmaU36p/P2wcBb8+sLgJlAj4BSQ5RQQgihRI2mr98FmA3sRSppzCcF\n",
"ibsl7VzvvIKRwMOF7YV5X/EeI0lB5qy8y4Vjq0m6g1Ttdr3te/OhzW0vzq8XA5s3kReIgBJCCKVq\n",
"VEL5CnCC7UuKO3M7x1eBw3q5duO6tOQMYLJt53nClhe1bC8Ddpe0ETBd0ljbM1e6QTqv1/tIrJmS\n",
"s6SJPIUQQuiHRgFlV9s9gobty4qN5A0sAkYVtkeRSilFewEXpVjCcOCdkpbYnla4398k/SqnnQks\n",
"lrSF7cckjQB6jJWpkDQlvdpobbjsJdiviWyHEMLQIGksMLZl16vXKC9plu09+nqskGZ1UlXZfsAj\n",
"wC3UaJQvpD8PuMr25bntZqntZyStA0wH/sv2dblR/inbUyVNBjburVFeYkvgNpstG+U5hBCGsjIb\n",
"5V8h6dMUqqGKx3q7sO2lko4nBYNhwLm250iamI+f3eD0EcAFee2V1YALbV+Xj30DuETSscAC4H29\n",
"5YVoPwkhhNI1KqFMoXY7iEjNF/9VYr4GrKqEshvwE5vdOpytEELoWqWVUGxP6e9Fu1CUUEIIoWSN\n",
"pq8/tc4hA9j+Uik5KkfMNBxCCCVr1Ibyd3pWea0HHEvqkbWqBZQooYQQQokaVXl9u/Ja0obAJ4EP\n",
"AxcB3yk/ay0Vi2uFEELJept6ZTPgROADwI+BPW0/3eicLhUllBBCKFmjNpRvA+8BfgDsZvu5tuWq\n",
"9SKghBBCyRp1G14GvAQ1pyux7Q3LzNhAVXUb/izwSpvPdjhbIYTQtcrsNtxorZRVTZRQQgihZIMp\n",
"aDQS3YZDCKFkQymgRAklhBBKFAElhBBCSwyVgBLjUEIIoWRDJaBECSWEEEo2lALKPzqdiRBCGMyG\n",
"SkBZC3ix05kIIYTBbCgFlH92OhMhhDCYlRpQJI2TNFfSfEknNUg3WtJSSYfm7VGSrpd0j6S7JX2y\n",
"kHaKpIWSZuWfcU1kJQJKCCGUrOHkkAMhaRhwJrA/sAi4VdK06jXlc7qpwDWsWG54CXCi7TskrQ/8\n",
"SdK1tueSptQ/zfZpfcjOmqRpZEIIIZSkzBLKGOB+2wtsLyFNez++RrpJwKXAE5Udth+zfUd+/Tww\n",
"BxhZOKevc81ECSWEEEpWZkAZCTxc2F7IykEBSSNJQeasvKvHTJWStgX2AG4u7J4k6U5J50rauIm8\n",
"REAJIYSSlVblRY3gUMMZwGTbliSqSh65uutS4IRcUoEUfCqrRX6ZtNjXsbUuLmlKevX5jeGaN8Cf\n",
"FvXxPYQQwqAlaSwwtmXXqzd9/YAvLL0RmGJ7XN4+GVhme2ohzYOsCCLDSYMPP2Z7mqQ1gKuB39g+\n",
"o849tgWusr1rjWPF6eufBbayebZV7y+EEAab0qavb4HbgB3zQ/8R4AhgQjGB7e0rryWdRwoO03Jp\n",
"5Vzg3upgImmE7Ufz5nuA2U3kJaq8QgihZKUFFNtLJR0PTAeGAefaniNpYj5+doPT9waOAu6SNCvv\n",
"O9n2NcBUSbuTqtQeAiY2yoeEgDWIXl4hhFCq0qq8Oq1SdJNYE3jeZs1O5ymEELrZQKu8hsJI+aju\n",
"CiGENhgKAWVNIqCEEELphkJAWYtoPwkhhNINlYASJZQQQihZBJQQQggtMRQCSkwMGUIIbTAUAkqU\n",
"UEIIoQ0ioIQQQmiJoRBQosorhBDaYCgElCihhBBCG0RACSGE0BJDIaBElVcIIbTBUAgoUUIJIYQ2\n",
"iIASQgihJYZCQIkqrxBCaIOhEFCihBJCCG1QakCRNE7SXEnzJZ3UIN1oSUslHZq3R0m6XtI9ku6W\n",
"9MlC2k0lzZA0T9K1kjbuJRsRUEIIoQ1KCyiShgFnAuOA1wATJO1SJ91U4BqgslLYEuBE268F3gj8\n",
"u6Sd87HJwAzbOwHX5e1GosorhBDaoMwSyhjgftsLbC8BLgLG10g3CbgUeKKyw/Zjtu/Ir58H5gAj\n",
"8+GDgQvy6wuAQ3rJR5RQQgihDcoMKCOBhwvbC1kRFACQNJIUZM7Ku3oscC9pW2AP4Oa8a3Pbi/Pr\n",
"xcDmveQjAkoIIbTB6iVeu0dwqOEMYLJtSxIrqrwAkLQ+qfRyQi6prHyDdF7d+0iaAkftA888JV19\n",
"m+2ZfXoHIYQwiEkaC4xt1fXKDCiLgFGF7VGkUkrRXsBFKZYwHHinpCW2p0laA7gM+IntKwrnLJa0\n",
"he3HJI0AHq+XAdtTJEYAt9vMHPhbCiGEwSN/yZ5Z2ZZ06kCuV2aV123AjpK2lbQmcAQwrZjA9va2\n",
"t7O9HakkclwOJgLOBe61fUbVdacBx+TXxwBX0FhUeYUQQhuUFlBsLwWOB6YD9wIX254jaaKkib2c\n",
"vjdwFPA2SbPyz7h87BvAAZLmAfvm7UbWJAJKCCGUTnYzTR2rHkm5iYXLgJ/ZXNbpPIUQQjerPDf7\n",
"e36MlA8hhNASQyGgRJVXCCG0wVAIKGsRI+VDCKF0QyWgRAklhBBKFgElhBBCSwyFgBKTQ4YQQhsM\n",
"hYASJZQQQmiDCCghhBBaYigElKjyCiGENhgKASVKKCGE0AYRUEIIIbTEUAgoUeUVQghtMKgDisTq\n",
"gG1e7nReQghhsBvUAYWo7gohhLYZ7AElqrtCCKFNBntAiRJKCCG0SakBRdI4SXMlzZd0UoN0oyUt\n",
"lXRYYd+PJC2WNLsq7RRJC2us5FhLBJQQQmiT0gKKpGHAmcA44DXABEm71Ek3Fbim6tB5+dxqBk6z\n",
"vUf+qT6vKNZCCSGENimzhDIGuN/2AttLgIuA8TXSTQIuBZ4o7rR9I/B0nWs3u0RlrIUSQghtUmZA\n",
"GQk8XNhemPctJ2kkKciclXc1u8D9JEl3SjpX0sYN0kWVVwghtEmZAaWZ4HAGMNm2SaWOZkoeZwHb\n",
"AbsDjwLfqZ/09cfBp16Z213GNnHtEEIYMiSNzc/HKZKmDPR6q7cgT/UsAkYVtkeRSilFewEXSQIY\n",
"DrxT0hLb0+pd1PbjldeSzgGuqp+FO38C7GCfMaWPeQ8hhEHP9kxgZmVb0qkDuV6ZAeU2YEdJ2wKP\n",
"AEcAE4oJbG9feS3pPOCqRsEkpxth+9G8+R5gdoPkUeUVQghtUlqVl+2lwPHAdOBe4GLbcyRNlDSx\n",
"t/Ml/Rz4A7CTpIclfTgfmirpLkl3Am8FTmxwmejlFUIIbaLUfDH4SDL4vcCRNod3Oj8hhNDtJNl2\n",
"s71oe4iR8iGEEFoiAkoIIYSWGOwBJSaHDCGENhnsASVKKCGE0CYRUEIIIbTEYA8oUeUVQghtMtgD\n",
"SpRQQgihTSKghBBCaInBHlCiyiuEENpksAeUKKGEEEKbREAJIYTQEoM9oESVVwghtMlgDyhRQgkh\n",
"hDaJgBJCCKElBntAifVQQgihTUoNKJLGSZorab6kkxqkGy1pqaTDCvt+JGmxpNlVaTeVNEPSPEnX\n",
"Stq4QRbWItpQQgihLUoLKJKGAWcC44DXABMk7VIn3VTgmqpD5+Vzq00GZtjeCbgub9fT1VVeksZ2\n",
"Og/NiHy2VuSztSKf3aPMEsoY4H7bC2wvAS4CxtdINwm4FHiiuNP2jcDTNdIfDFyQX18AHNIgD91e\n",
"5TW20xlo0thOZ6BJYzudgSaN7XQGmjS20xlo0thOZ6BJYzudgbKVGVBGAg8XthfmfctJGkkKMmfl\n",
"Xc2sR7y57cX59WJg8wZpo8orhBDapMyA0kxwOAOY7LSwvfJP8zdI5zW6zxdZOaiFEEIoidIzuYQL\n",
"S28Eptgel7dPBpbZnlpI8yArgshw4AXgY7an5ePbAlfZ3rVwzlxgrO3HJI0Arre9c437l/PGQghh\n",
"ELPdpy/2Rau3MiNVbgN2zEHhEeAIYEIxge3tK68lnUcKHtN6ue404BhSQ/4xwBW1Eg3klxJCCKHv\n",
"Sqvysr0UOB6YDtwLXGx7jqSJkib2dr6knwN/AHaS9LCkD+dD3wAOkDQP2DdvhxBC6LDSqrxCCCEM\n",
"LYNupHyzgynbTdIoSddLukfS3ZI+mff3ZaBmu/I6TNIsSVd1ax4BJG0s6VJJcyTdK+lfui2vkk7O\n",
"f/PZkn4maa1uyGOtgcON8pXfx/z82Xp7h/P5rfw3v1PS5ZI26sZ8Fo79h6Rlkjbt1nxKmpR/p3dL\n",
"KrZz9y2ftgfNDzAMuB/YFlgDuAPYpdP5ynnbAtg9v14fuA/YBfgm8Lm8/yTgG12Q108DPwWm5e2u\n",
"y2POywXAR/Lr1YGNuimv+f/hg8BaeftiUrtfx/MI7APsAcwu7KuZL9LA5DvyZ2rb/BlbrYP5PKBy\n",
"f1KVd1fmM+8fRRq0/RCwaTfmE3gbMANYI2+/or/5HGwllGYHU7ad7cds35FfPw/MIY3L6ctAzdJJ\n",
"2go4EDiHFT3wuiqPAPlb6T62fwSpzc723+iuvD4LLAHWlbQ6sC6pg0rH8+jaA4fr5Ws88HPbS2wv\n",
"ID1YxnQqn7Zn2F6WN28GturGfGanAZ+r2tdt+TwO+Hp+ZmK7Msi8z/kcbAGl18GU3SD3fNuD9GHo\n",
"y0DNdjgd+CywrLCv2/IIsB3whKTzJN0u6YeS1qOL8mr7r8B3gL+QAskztmfQRXmsUi9fW5I+SxXd\n",
"9Ln6CPDr/Lqr8ilpPLDQ9l1Vh7oqn8COwL9K+qOkmZLekPf3OZ+DLaB0fQ8DSesDlwEn2H6ueMyp\n",
"nNmx9yDp3cDjtmdRZ5Bpp/NYsDqwJ/B923sCf6dqXrdO51XSq4BPkaoLtgTWl3RUMU2n81hPE/nq\n",
"eJ4lnQK8ZPtnDZJ1JJ+S1gU+D5xa3N3glE7+PlcHNrH9RtKXyUsapG2Yz8EWUBaR6iwrRrFyhO0o\n",
"SWuQgsmFtivjZxZL2iIfHwE83qn8AW8GDpb0EPBzYF9JF3ZZHisWkr793Zq3LyUFmMe6KK9vAP5g\n",
"+ymnbvSXA2/qsjwW1fs7V3+utsr7OkbSh0hVsx8o7O6mfL6K9EXizvx52gr4k6TN6a58QvosXQ6Q\n",
"P0/LJA2nH/kcbAFl+WBKSWuSBlP2NlCyLSQJOBe41/YZhUOVgZrQYKBmO9j+vO1RtrcDjgR+Z/uD\n",
"3ZTHCtuPAQ9L2inv2h+4B7iK7snrXOCNktbJf//9SWOyuimPRfX+ztOAIyWtKWk7UhXJLR3IH5B6\n",
"cpK+SY+3/WLhUNfk0/Zs25vb3i5/nhYCe+Yqxa7JZ3YFaUwf+fO0pu0n6U8+29GzoJ0/wDtJPaju\n",
"B07udH4K+XoLqV3iDmBW/hkHbAr8FpgHXAts3Om85vy+lRW9vLo1j68HbgXuJH3D2qjb8kpqkL0H\n",
"mE1q6F6jG/JIKoE+Qpo89WHgw43yRaq+uZ8UJN/RwXx+BJgP/LnwOfp+F+Xzn5XfZ9XxB8m9vLot\n",
"n/n/5IX5/+ifSFNb9SufMbAxhBBCSwy2Kq8QQggdEgElhBBCS0RACSGE0BIRUEIIIbREBJQQQggt\n",
"EQElhBBCS0RACYNSni7824Xtz0g6tdE5fbj2+ZIOa8W1ernPe/O0/NdV7d9G0oR654XQKRFQwmD1\n",
"EvAeSZvl7VYOuOr3tfKsw806Fvio7f2q9m8HvL8F1w+hpSKghMFqCfAD4MTqA9UlDEnP53/HSrpB\n",
"0hWSHpD0DUkflHSLpLskbV+4zP6SbpV0n6R35fOH5cWfbsmLP328cN0bJV1JGjVfnZ8J+fqzJX0j\n",
"7/sisDfwI0nfrDrlG8A+SougfUrSMZKm5ZLMDEnr5oWUbs4zMR/cS/5GSPrffL3Zkt7Sz995GOLi\n",
"20wYzL4P3FXjgVxdwihu7wbsTFoz4iHgh7bHKK2wOYkUoARsY3u0pB2A6/O/x5CmqB8jaS3g95Ku\n",
"zdfdA3it7T8XbyxpS1KA2BN4BrhW0njbX5L0NuA/bN9eld+TgM/YPihf40P5+rvafkbS14DrbH9E\n",
"adXFmyX9FjiqTv4OBa6x/bU859h6vf9qQ+gpAkoYtGw/J+nHwCeBfzR52q3Oa4JIuh+YnvffTVrZ\n",
"DlIAuiTf435JD5KC0NuBXSUdntNtCOwALAVuqQ4m2WjgettP5Xv+FPhX4Mp8vNaU59X7DMyw/Uze\n",
"fjtwkKTP5O21gK0b5O9WUkloDeAK23fW/M2E0IsIKGGwOwO4HTivsG8pubpX0mrAmoVj/yy8XlbY\n",
"Xkbjz0ullHO80yJay0kaS1qvpd55xQAhVi4xNdteU339Q23Pr8pHzfzlY/sA7wbOl3Sa7QubvG8I\n",
"y0UbShjUbD9NKk0cy4qH8wJgr/z6YNJsq30h4L1KXgVsT5qNdTrwb5WGcUk7KS201MitwFslbSZp\n",
"GGnZgBt6OedZYIOq/BRNJ5XKyPnYo7C/R/4kbQ08Yfsc0tLPexBCP0QJJQxWxW/23wGOL2z/ELhS\n",
"0h3ANcDzdc6rvp4Lr/9CWhtiQ2Ci7ZcknUNaVOn23BbxOPCeqnNXvqj9qKTJwPWkwHC17at6eW93\n",
"AS/n/J9Pau8pXv/LwBmS7iJ9aXyQFDjr5W8s8FlJS4DngKN7uX8INcX09SGEEFoiqrxCCCG0RASU\n",
"EEIILREBJYQQQktEQAkhhNASEVBCCCG0RASUEEIILREBJYQQQktEQAkhhNAS/x9XMHBaa6Y8mgAA\n",
"AABJRU5ErkJggg==\n"
],
"text/plain": [
"<matplotlib.figure.Figure at 0x1fb93fd0>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%time\n",
"learning_curve_ndcg(gb,Xts,Yts,Qts,rank = 10)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# LambdaMart (ndcg)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 2min 44s\n"
]
}
],
"source": [
"%%time\n",
"from lambdamart import LambdaMART\n",
"\n",
"lmart= LambdaMART(n_estimators=150, max_depth=6,\n",
" learning_rate=0.1, max_rank=70,gain = \"exponential\")\n",
"lmart.fit(Xtr,Ytr,Q=Qtr)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train:\n",
"NDCG5 : 0.656670577472\n",
"NDCG10 : 0.642952406323\n",
"NDCGfull: 0.744143185806\n",
"rank AUC: 0.744558304194\n",
"MSE : 3.02827923988\n",
"test\n",
"NDCG5 : 0.440132596805\n",
"NDCG10 : 0.465539559628\n",
"NDCGfull: 0.62078060222\n",
"rank AUC: 0.669451820022\n",
"MSE : 3.07903462698\n",
"Wall time: 1.39 s\n"
]
}
],
"source": [
"%%time\n",
"print \"train:\"\n",
"score(lmart,Xtr,Ytr,Qtr)\n",
"print \"test\"\n",
"score(lmart,Xts,Yts,Qts)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 3.61 s\n"
]
},
{
"data": {
"image/png": [
"iVBORw0KGgoAAAANSUhEUgAAAY0AAAEZCAYAAABrUHmEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
"AAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xe4XFW5x/HvLwkgNVQpIVIEpEg3EQUuoYhBQRRQjIIg\n",
"IAgSBeRSvApcLICioiJcpCoqAQExWAhFAihITWgJkAChg6GDICbkvX+sdcLOZM6cOWVPy+/zPOdJ\n",
"dlv7nUnOvLPW2mstRQRmZmb1GNTsAMzMrH04aZiZWd2cNMzMrG5OGmZmVjcnDTMzq5uThpmZ1c1J\n",
"wxZokkZJeqJJ9z5T0jebcW+zvnLSsNJJmiHpOUmLFfYdIOn6wvYcSa9Lek3S85KulfSZKmV9VNKN\n",
"kl6V9E9JEyXtUji+sqSzJT2Vy3pY0vmS3lf+K+2diDg4Ir7T7DjMesNJwxplEPC1Hs7ZKCKWBNYB\n",
"LgBOl3Rc10FJewCX5GPDIuLdwHHALvn4csDNwLuArXJZmwE3AB8ZyBfTE0lDGnm/MnTCa7CB56Rh\n",
"jRDAqcCRkob2eHLEixHxa+Bg4FhJy0gS8CPgxIg4LyJey+feGBEH5ksPB16OiL0j4tF8/JWIuCAi\n",
"Tq8nUEmrSLos12IekTS2cGykpFskvSTpaUk/k7RQ4fgcSYdImgY8KGkbSU9KOiLXtJ6WtG/h/Ask\n",
"fTv/fVQP5y4n6UpJr0i6TdJ3JN1U43VsJenmHOvjkr6Q90+UtH/hvH2L5RRew0PAQ5LOkPSDirL/\n",
"IOnwnt4v60xOGtYodwATgSN7cc14YAgwEngfsCpwaY3zdwB+38f4kDQIuBKYBKwCbA8cJmnHfMps\n",
"Um1pOeBD+fghFcXsCowA1gcErAgslcvbH/h5IXFG/ulS69yfA6/lc/YBvlBxbfF1rAb8GfgJsDyw\n",
"CXB3N/esZlfSe74ecBGwZ6HsZUi1tovqeL+sAzlpWKMEqSlprKTl67ogYhbwPLAs6YMa4JkalywH\n",
"PNu1IekT+Zv2q5Im1HHLEcDyEfGdiJidayvnAJ/N8dwVEbdFxJyIeAz4BbBNRRknRcTLEfFW3p5F\n",
"qh29HRF/AV4nJcC5YRb+XvVcSYOB3YDjI+LfETEV+GXFtUWfA66JiItzWS9GxN3dnFtN8TX8DQhJ\n",
"W+djewA3R8SzPb1f1pncZmkNExH3S/ojcAwwtafzc9PPCsCLwAt598rAY91c8gLpG2/X/cYDy+Tm\n",
"mL3qCHE1YBVJLxX2DQZuzPGsQ2oi2xxYjPT7c0dFGZVPYr0QEXMK228AS3QXfzfnrpDvVSz7yRqv\n",
"Y1XgkRrHezL3PhERksYBY4CbSAnpV/lwzffLOpNrGtZoxwNfAobVce6upCah24AHSR9me9Q4/zrg\n",
"k7n/o6i7b+SVngAejYhlCj9LRcTO+fiZwBRgrYgYCvwP8/8O9Xba6HrOn0l6H4YX9g3v5lxIr+O9\n",
"3Rz7F7B4YXulOmK6CNgjN3uNBC7L+x+n9vtlHchJwxoqIh4GLqb6k1QCkLSspM8DpwMnR8RLkebw\n",
"PwL4Vu68XUrSoNzhe1a+/kfAMsCFktZUsiSpTb+eD+fbgNckHSVpUUmDJb1f0gfy8SVI/QpvSFqX\n",
"1FHfH6KOhBYRbwOXAyfkuNYF9qb71/QbYAdJn5Y0JHeib5yPTQZ2y+WsReo76en+k0nNhOcAV0XE\n",
"q/lQT++XdSAnDWuGE0nNO5UfendLeg2YBuwHHBYRJ3QdjIjLSJ2y+wFPkfovTgSuyMdfALYA/k1q\n",
"i3+V1Em7OLU/4CNf/zawMynJPEL6hv8LUuc0pE78z+VyfwGMq3gN1T7EayWryk7pWuceCgwlveZf\n",
"kr79/6dqoRFPAB8Dvk5qspsEbJQP/zhf9xxwPvDrOmP4LbBd/rPrPnOo/X5ZB1KZizBJGg2cRmrn\n",
"PCciTunmvBHALcBnIuLyPBBrXOGUNYFvRcRPSwvWrI1IOgV4d0R8sdmx2IKltKSRn/h4kPQY5FPA\n",
"7cCY/ORH5XnXkDr9zs/fJovHB+XrR+ZvUGYLnPxFahHgXtJTS38C9s+d/WYNU2bz1EhgekTMyI9O\n",
"jiN1bFYaS3r2fmY35ewAPOyEYQu4JUkd0K+TfpdOdcKwZijzkdthzP+I4AeLJ0gaRkok25G+PVWr\n",
"9nyWQjuq2YIoIu4A1m52HGZl1jTqafc6DTgmPxkz35MkkhYmzSv0u4EPz8zMeqvMmsZTzP9ceeWA\n",
"pM2Bcfmx+uWBnSTNKlS7dwLujIiqTVeSyuvFNzPrYBFR7/il+S4s5YeUkB4GVgcWJj0fvl6N888H\n",
"dqvYNw7Yp8Y1UVb8A/xenNDsGByn42znONshxjaLM/p6bWk1jYiYLelQYALpkdtzI2KqpIPy8bNq\n",
"XS9pcVIn+JfKitHMzHqn1LmnIk269peKfVWTRVQ8bx4R/yI1WZmZWYvwiPDGmNjsAOo0sdkB1Gli\n",
"swOo08RmB1Cnic0OoA4Tmx1AnSY2O4CylToivGySIvramWNmtoDqz2enaxpmZlY3Jw0zM6ubk4aZ\n",
"mdXNK/eZmTWYxMnAm8DZETzd7Hh6wzUNM7MqJJaS2ERilMTC/ShnZYm/SWmVRInhwIGkVRPvk9pr\n",
"TXUnDTNbYEnsIDFdmndMmISAm0mLVJ1LWtq3L+UPAi4kzYzRtVrlvsC4CA4mDWD+icSKfXoBTeCk\n",
"YWYLDIn1JcZKrCSxOWkG7YdJa9cXbUJaXXJD0hx4h0gs2YdbHg0sBGwLHCixNGmJ3XMAIriLNIXS\n",
"z/ryeprB4zTMrONJDAGOAg4HrgM+mg/tC/wdeADYKoIH8vknAYrgmLx9CXBrBD/sxT3fQ1pqd+MI\n",
"npQYR1oXZeUINiuctyhpbr4jIvhTv15o3bH1/bPTScPMOlqhiWhlYN8IHs+1hlUjmJrPORLYJoJd\n",
"ctPUw8DuEUzKxzcF/gisGcFbdd73eODdEXylUMZdwFciOKPi3C8CO0ew+wC85Dpic9IwM5tPTgA/\n",
"JC3ytmMEb3Zz3iLAncB5wE2kZqt1It5ZF0jiz6SF5Q7rrpzCuYOBR4BPdiWevP9g4DcRvFpx/rLA\n",
"DGCVCF4v7F8KWD+Cf9T9ouvgEeFmZtV9A9gR+EStD/pce9gJOAz4OXBxMWFkewFDgbsk1u/hvtsD\n",
"zxcTRr7PmZUJI+9/EbglxwCAxBKkteBb6ukqJw0z6zgSys1DewEfieClnq6J4AnSh/aqwEVVjr8Y\n",
"wWdJK4l+vYfi9ic9ddUblwJ7AEgsBlwJPAQc0ctySuXmKTNra/lx2d8D65IGzL0BzCYtOb1DBM/1\n",
"srxBEcypcXxj4JII3lfYtwxwJLAr8CKwEbB6BC/34r4rANOBDYBLgGnAfhG83Zv467tX3z87PSLc\n",
"zNqWxMrANaRO6t2BRUmPyi4GPBTBa70ts1bCyO4D3i2xYgTPSYwE/gxcDnwx3/ul3iSMfN+ZEncA\n",
"9wJnAd+oI5aGc03DzNpOfvrpS6RmojMi+G6D7/8n4LwILpM4F5jSm8dxa5T7EdJTXef3O8ia9/HT\n",
"U2a2AMhPQ+0LnAxcD3w/D5BrdBzHAiuSxn48A2yS+0TagpunzKzlSKwCPFPlKaSertsIOKgwvmEo\n",
"acT0EqRO6kHA6MonkxrsJuA00jQgU9spYfSXn54yswGX51J6FNizD5fvR5q2Y1TePoaUMH4N/C/w\n",
"wSYnDIDbSR3v+5M6rRcYbp4y61B5LMHuwCkR/KfOa1YC3g8sC0yI4JW8X6RpNebrmJVQZW1C4jjS\n",
"t/B1gA0jmFnn/QcBjwG/ArYjJZ1JwEYRPFVPGY0icRPwYWB4u01v7j4NM5tH/vC9CVgGeAXYM4LH\n",
"e7hmGdKTQQ+R5ki6LYJD8rGTgKW6mowK12wNnA2M7Bq0lqcRf4yUNPYDVorg8z3EOjiCWRIfJE3g\n",
"tyEpWQwCrojgm718C0on8T3gwxFza0RtwyPCzdqExOISIyXek6ea6Nr/PonDB/BWewELk8YL/AG4\n",
"Lk/aV8upwO8j2Bb4GPDZvBbEqsBBwJj8iGvRMaS+0e8X9n0GuD+C+4FvAR+QOLVrTYpcayH/fRHS\n",
"46rXSixEqhldlscmHAssV1F2K/kpcHCzg2g01zTMGkjit8BI0rP8T5GmuJgN3AqsCbwngn/28x4r\n",
"kr6lfzKC2/K+G0mPpo7r5podSCOY3981tkHip8BbpP6E10hjIF6L4Bv5+AakGWM3Bu4A9iE9SfRb\n",
"4LgIrsznLQdcQJow8ElgG9JYhONIa0y8nd+PB4BPArtFMDlfu0i9EwRa/dw8ZdYG8rf0KcAapCaj\n",
"H5Da7R8DXiDVDO6K4LQ+lr8nad6kRUgJ4ujCsY8B3wM2jSDyOIc3SLWEA0k1gn0i+EvhmuHAPcAc\n",
"Ut/EUsBtpJleX5M4D3gkgu9IfJxUo3kKGE+a1O/tQlki1X5mAzcCHyF1at8HfIqUmO4kJZC1e/vE\n",
"lfWOk4ZZG5A4AVgxr9jW9UH6PWAUaZGeLYFTI9i0j+WPBy4DflWlY1rA3aQV6D5Ims9oCPAf0niH\n",
"46o9kSTxE+DpCE7J2xeTPvgnkyYDXCuCF/KxocCr9X7g5+ay6EouueayWgR/7u1rt95x0jBrccXO\n",
"4dzWXzym/O1/EGl67J0juKdKGd8gjUJ+tsoxATNJC/5UfcpI4nOkp5KuIU138QKpc/uFXryO1YEv\n",
"k2pF/4hYsB437RROGmYtTmIMcEAE2/dw3neBd0XMO4uqxCakfopfRbBPlevWBa6KYPUaZQ8mPdF0\n",
"TSvOaWSN07JPT0kaLekBSdMkHV3jvBGSZkvarbBvaUmXSpoqaYqkLcqM1dqTxKK5WaTVfRrqmk/o\n",
"AmDv3JldNJbUlLW9xIerXLcl8LdaBUfwdgQTnDCsP0pLGpIGA6cDo4H1gTGS1uvmvFOAq4Bi5vsJ\n",
"8OeIWI/02ODUsmK1tnY48LjE17se6Ww1ueloK+CGns6NYBrpKaYzux5NzVN/7wb8mDTX0enFx3Wz\n",
"rUhrXZuVqsyaxkhgekTMiIhZwDjSXPOVxpIWH5k7YlTSUGDriDgPICJmR8QrJcZq7WsE6SmkHYC/\n",
"5QFqrWZt4M1ezE90Aulppa4V275EGj/xPGlxoGeAWyS2KVzTY03DbCCUmTSGwTy/JE/mfXNJGkZK\n",
"JGfmXV0dLGsAMyWdL+kuSWdLWqzEWK19bUb6QvIx0ofmtXm95VayJb2oBeRxCfsAZ0k8Shrk9rN8\n",
"LIBdgB8BF0h8V+LdwAowbwe7WRnKTBr19LCfBhwTqTdevNM8NYT0YXBGRGwG/Is08tQ6iMRaEuPy\n",
"U0N9uX45YGnSWIEgra1wPTC+r2WWZCt6WQuI4E7Sl6ztSdNuTyocm5MH6Y0gDYb7LelJJvdVWOnK\n",
"nBr9KWB4YXs4qbZRtDkwThLA8sBOkmaRRsc+GRG35/MupZukIemEwubEiJjY78itUT5PmpDuD1RZ\n",
"k7kOmwKTuz4s82OrRwE3k9ZcOG+A4uyvrUh9dL2SR2Z3u/JcBM/nkdw30Dqv1VqQpFEwMHNklfbI\n",
"raQhwIOkb0pPk0aSjomIqh3aks4HroyIy/P2jcABEfFQTgyLRsTRFdf4kds2kJfDXCKCv1bsvxe4\n",
"mDRmYL16Z2ItXH8UsHLEvHM2SWwO/AlYD9iaNAgN0ijsq4FLI3isL6+lt/K6z9OA5cpY6znfY3Fg\n",
"Vm/fP1twteQiTBExW9KhwARgMHBuREyVdFA+flYPRYwFfiNpYeBh0geLtadTSP/X5iaNPK5gOdJj\n",
"pFsBBwBn9LLczUjJYR4R3ClxOe9MS3EE8DzwbmBn4E6JoyLm/XaeP+DfjOD1XsZRy5bALWUlDIAI\n",
"/lVW2WaVPLjPSiWxDmmK7kWA90XwXN7/P6Qps8fmgWsTSZ3FVwM/q6d9XuIh4FOVI6zzsWVITVRn\n",
"RfBGlZiuI819dAepNvxJ0nTckGZd/d8IHuz1C54/jh8Ar0Twnf6WZTZQPCLcWpY0d1rr1Ugjkc/J\n",
"+yeRJrW7IW+/m9SUdDRp/YTv9VDuUqRmz6UjmN2HuNYhTafxb1IN6I/AtaRJ+Y4C1o/g470tt+Ie\n",
"i5PWpvh416ytZq3AScNaUh5s9wQpGWwG7B3Bx3PT1ERgWGWzTV5X+g7gCxFcW6PsrYEfRNDnmQKq\n",
"rTiX9y8GPA6MiODRfpT/TdJU45/t8WSzBmrZaUSsuSQkcaLEEgNU1rq9vGxX0mI8DwF/AbbOy4n+\n",
"Fvh+tXb+vGzm54AL89TclXEsLLEp6cmru3r7OiruVfUbU27OupA0ZXjx3qtLnC/xFYm1apWda06H\n",
"kWaVNesYThptSGJZibXrOHUt0joJBw3AbbcEpkjsWM/JEosC3yUNQiOvNf33/HMvaUqMqiKYmI//\n",
"rjg1SB57cS0p6SwN9PQwRX/8H7BfXlmuy/6kzvRNgX9UWcWu6DjgNxE8XGKMZg3npNHCqswv1OV/\n",
"gUkSO/VQxA7A7cDXJd7Vz3D2IHUQ/0pitTrO/zZpQaE/Fvb9htRcdVAday78gDRdxo8K+w4h/Z/d\n",
"IILPRnB33dH3Uu4Ev48051PX/FGfI607cQBpXETVzu08oeDupPfArLNERNv+pPCbH0c5ry0Wh7gH\n",
"YkzF/sUgXoDYE+I5iD1rlHEZxF4Q4yEOgVgb4mqIjQvn7AVxAsRSNcoZBPEExAYQR0JMgtgWYlDh\n",
"nBUhvg7xVYgvQjwDsUKVstSL92AoxFSIcRBbQ8yEWLeB/wYfh5gCsRDESIgHu+LPsT0LsVnFNYvm\n",
"83Zv9v8h//inu5/+fHa6ptECJHYuTnuRv9WeSVo3ec+K0/cAbo3gYuAzpGaQamUOJq0Gdx2pmeh4\n",
"0kjpoaRvwV0OI42TeEhi525CHAG8Tlqq9IfA2aQRzjMkJkpcQ1rfeQNgXeALpLUjZlYWFFH/Mp6R\n",
"mrQ2J81wPAH4cQQP1Hv9APgzaWaDL5NqGRd1xZ9jOx74WX5Kquvf7fvApAgua2CcZg3jp6eaLLeL\n",
"Pw18LPL6zBL7k6b8/ijpA3OlyGMNJP5GWhL0ipwYnieNpn62otyRpFXe3p+3jyV9CC4HnBTBB/Oa\n",
"DQ+SJrv7MGkqj3UjeLWirB8Ab0XwzcI+kRLESqT1nW+K4OUBfGvm0dulRAfwvu8nPZI7B9gmCmM3\n",
"8vt/LvAh0jQ3Y4ElgZ0izUhr1pL89FR724b0Lf4QmLt2winAZyIt23kbpM5nifWBNcmjoCM9fTQR\n",
"2K5KuTvAO4+sRnBSpD6AvwPr5cn+dgSui2BWpPESE3hnyg3yPUWqmVxa3J9rqlMjuD6CK8tMGPl+\n",
"rzQ6YeT73gdcAjwVFYP9Ii1qtC+ptncycCXwIScM62SuaTSZxP+RxgQcAXwA+CqwSARfyccPJTUP\n",
"7QtcDtwTwfGF68eS1oU+oKLcvwI/jJh/mg2JP5IeKf0EMDGCs/P+lUlPNn2U1DQ2gpRYViMNdmvf\n",
"/yz9kB8iWC66WXvbrN14cF8bk3iQ1G+xN7Ay6QN7g67mpjxWYTJpGvmPAaMirbfQdf36wJ8iWKOw\n",
"bylSW/wqEfPPkpoT0QeAjwObRWFxIImvAieSmsUmk/pErovgpYF83WbWPE4abSqPfr6PNC38e0lT\n",
"TpxYrEnk824nTS2/eeW33dx89DTw4cijlyX2A3aJ4FPd3HftfN9pXX0eZrbgcJ9G+9oGuDHSojrT\n",
"SKu1nVrlvGNI8xfN1zySm4z+Spp0r8u+wAU17judVBO5qm9hm9mCyjWNJpI4C5gS0fsFeirK2Q8Y\n",
"HcFn8vQWfwdWjWBWjWv2Be7IHb1mtgBx81QbyZPhbQO8TOqM3j36ObI5rwNxO3ASaYnQJaNiYSIz\n",
"sy4tuQiTzSs/0z+WNPX3NNL6Es+RnlbqlwhmSmxPevx2KdKssmZmA85Jo3H2Jo2UHt3fmkU1ETyc\n",
"14s+IIJ7Brp8MzNw81TDSNwCfC+CK5sdi5kt2Pz0VIuT2AhYlbSmhJlZ23LSaIwDgXOjD8uSmpm1\n",
"EjdPlSzPgPoEsEkEjzc7HjMzN0+1tl1IU5k7YZhZ23PSKN9HSFOSm5m1PSeN8m1HmvTPzKztOWmU\n",
"SGJN4F2kGWPNzNqek0a5tgP+uqCuQ2FmncdJY4BJbCHxgby5PWkGWjOzjuBHbgeQxDrA30jrSW8J\n",
"3Ax8MIIZzYzLzKzIExaWLC90dCBwdgRzujlnKDAe+Cbpfb0B+JcThpl1klKbpySNlvSApGmSjq5x\n",
"3ghJsyXtXtg3Q9I9kiZJuq3MOOvwfuD/gA1qnPMj4PoIfgGcCfwe+EMDYjMza5jSahqSBgOnAzuQ\n",
"Vom7XdL4iJha5bxTmH8VuQBGRcSLZcXYC7uRmpy2ocpU5nk9i92AtWDuanpjcw3FzKxjlFnTGAlM\n",
"j4gZETELGAfsWuW8scClwMwqx1rlQ3d34GxS0qjmAODyCF4o7vRTU2bWacpMGsNIcy51eTLvm0vS\n",
"MFIiOTPvKn7IBnCtpDskfanEOGuSWBtYATgZ+K/K2oPEEOBg4OdNCM/MrKHK7Aiv51v2acAxERGS\n",
"xLw1iy0j4hlJKwDXSHogIm6qLEDSCYXNiRExsT9BV/Ep4IoIZki8AazLvIP1dgGeiOCuAb6vmdmA\n",
"kDQKGDUQZZWZNJ4Chhe2h5NqG0WbA+NSvmB5YCdJsyJifEQ8AxARMyX9ntTcNV/SiIgTSoi9aHfS\n",
"E1GQnojahpw0JFYn1UC+VXIMZmZ9lr9MT+zalnR8X8sqs3nqDmBtSatLWhjYk/RI6lwRsWZErBER\n",
"a5D6NQ6OiPGSFpO0JICkxYEdGYC1tHtLYiVgbd55s28gZ2uJkcDfgZ9HcEmjYzMza4bSahoRMVvS\n",
"ocAEYDBwbkRMlXRQPn5WjctXAi7PNZAhwG8i4uqyYq1hFHBjBLPy9g3ASRK/IDVbHRDhx2rNbMHh\n",
"EeE1y+csYEoEP8nbIiWOm4FTIniprHubmZWlP5+dTho1y+ch4NMR3F3WPczMGs0r95VAYhiwLE3o\n",
"SzEza1VOGt0bBdzQ3VxTZmYLIieN7o2i8IiamZk5adSyLXB9s4MwM2slThpVSKwKDAWmNDsWM7NW\n",
"4qRR3RbALe7PMDObl5NGdSOBZq/hYWbWcpw0qhuBk4aZ2Xw8uG++MhkMvASsHkErLABlZjagPLhv\n",
"YK0LPOuEYWY2v5oTFkpaGhjNO4snPQlMiIiXyw6sidyfYWbWjW5rGpK+ANxJGuS2aP7ZDrhL0j4N\n",
"ia45nDTMzLrRbZ+GpIeAkZW1CknLALdFxNoNiK+mkvo07gS+EsE/BrJcM7NW0eg+jfbtOe+BxKKk\n",
"Po3JzY7FzKwV1erT+C5wp6SreWeZ1uGkVfS+XXZgTbIJ8EAE/252IGZmrajmI7eSlgU+CqySdz1F\n",
"6ghvicWHBrp5SuJQYMMIDhqoMs3MWk1/PjtrPj0VES8CF/Upqva0OXBLs4MwM2tVfRqnIalTFyba\n",
"nPTEmJmZVdFtTUPS7lV2ByBg5dIiapLcCb4WcF+zYzEza1W1mqfGAb+F+WZ6FfCu0iJqno1JneBv\n",
"NTsQM7NWVStp3AucGhHzNUVJ2r68kJrGTVNmZj2o1adxGPBqN8d2KyGWZtsMJw0zs5o8y+3cspgM\n",
"fCmC2weiPDOzVtWwEeGS7urLTVqdxLuAdUhNcmZm1o3ePnI7oPM8tZCNgIc8EtzMrLbeJo0/lxJF\n",
"822E55syM+tRr5JGRPxPWYE02TrAg80Owsys1dVaT+MASUcVtp+S9Jqk1yUdXE/hkkZLekDSNElH\n",
"1zhvhKTZknar2D9Y0iRJV9Zzv35w0jAzq0OtmsaXgfMK2/+MiCWB5YExPRUsaTBwOmnlv/WBMZLW\n",
"6+a8U4CrmL/P5GvAFMqfjv19wEMl38PMrO3VShqKiOcL278DiIh/k1bx68lIYHpEzIiIWaQR5rtW\n",
"OW8scCkwc56bS6sCHwPOocQOeIkhwBrAw2Xdw8ysU9RKGkOLGxHxPQBJg4Dl6ih7GPBEYftJ3llr\n",
"nFzWMFIiObPrNoXDPwb+m/mnMRloqwHPRvBmyfcxM2t7taYRuUbSdyLim107JIm0ANPVdZRdT5PS\n",
"acAxERG5bOX77ExqDpskaVStAiSdUNicGBET67hv0Tq4acrMOlj+HB01IGXVWCN8CVLT0Ajg7rx7\n",
"Y+AO4ICIeK2HILcAToiI0Xn7WGBORJxSOOcR3ml6Wh54AzgQ+CCwNzCbNDniUsBlEfGFinv0e0S4\n",
"xNeAtSM4tD/lmJm1i/58dvY4jYik9wIbkGoOUyNiep1BDSE9kbQ98DRwGzAmIqZ2c/75wJURcXnF\n",
"/m2AIyNilyrXDETSOIM0u+1P+1OOmVm7KGXlPkmjgSUj4ncUOokl7QG8EhHX1Co4ImZLOhSYAAwG\n",
"zo2IqZIOysfP6kWcZT49tQ7whxLLNzPrGLWap24GPhkR/6zYvwKpRrBFA+KraYBqGo8D20Tw6ACF\n",
"ZWbW0sqasHCRyoQBEBEzgcX7crNWI7EYsALweLNjMTNrB7WSxpKSFqrcmfd1ysp9awGPRPB2swMx\n",
"M2sHtZLG5cAv8lNUAEhaEjgrH+sEHgluZtYLtZLGt4DngBmS7spraTxKGrn9zRrXtZP1gAeaHYSZ\n",
"Wbuo55HbxYD35s3pEdEyI6f72xEu8XtgXAQXD2BYZmYtrbRxGpKWBz4HrJt3TQEuiogX+nKzgTYA\n",
"SWMGsGOEm6jMbMFRytNTeUbae4HNSYP0ppEmIbxP0rrdXdcuJJYhzaFV12BFMzOrPffUd4CvRcQl\n",
"xZ2Sdge+C+xeZmANsAlwT0TpEyKamXWMWh3hG1YmDICIuAzYsLyQGmZTYFKzgzAzaye1ksa/+nis\n",
"XThpmJn1Uq3mqRUkHUH1BZBWKCmeASVxOnByBE9WObwJaWp2MzOrU62axjnAksASFT9LAmeXH1r/\n",
"SIwAvkKVOeQlFiWNBr+vwWGZmbW1bmsaEXFCA+Mow1dJKwduAvy64tj7gYcieKvhUZmZtbFaU6Mf\n",
"382hAIiIE0uJaABIrATsTKpp7FfllE2AyQ0NysysA/TUEf56xU8A+wNHlx9avxwEXAz8FdhEmq9f\n",
"Zgfg5oZHZWbW5nqcRgRA0lKk5p79gUuAH1abNr3Rqo1qlHgXaY6sHSK4X+JZYEQET+Tjy5MG9K0R\n",
"wUsND9pQx59KAAAO+UlEQVTMrMnKWk8DSctJ+g5pjfCFgM0i4uhWSBg17AVMiuD+vD2Z1BxVPP5H\n",
"Jwwzs96rNY3IqaR1vV8DNoqI4yOipT9oJQYBRwKnFnZPIieN3Ey1P+nJMDMz66VaNY0jgGGkadCf\n",
"lvRa4efVxoTXazuT+mKuL+ybTBrIB2nurEWBGxocl5lZR6j1yG3NpqsWdSTwgwiKHTWTgZMkhgDH\n",
"AedWHDczszrV1RHeqoqdOblpajawcASz3zmHwcDLwHXAwsCnPD7DzBZk/ekIrzWNSLtZFHizmDAA\n",
"Inhb4u58fFcnDDOzvuukpLEY0N2qgp8BXnDCMDPrn05LGm9UOxDB0w2OxcysI7VjZ3d3FqWbpGFm\n",
"ZgOjk5JGreYpMzMbAJ2WNFzTMDMrUSclDTdPmZmVrNSkIWm0pAckTZPU7cy4kkZImi1p97z9Lkm3\n",
"SposaYqkk+q4nZunzMxKVlrSkDQYOB0YDawPjJG0XjfnnQJc1bUvIv4NbBsRmwAbAdtK2qqHW7p5\n",
"ysysZGXWNEYC0yNiRkTMAsYBu1Y5byxwKTCzuDMiuhLAwsBg4MUe7ufmKTOzkpWZNIaRllvt8mTe\n",
"N5ekYaREcmbeFYVjgyRNBp4Dro+IKT3cz81TZmYlK3NwXz2TWp0GHBMRIUnwzgp7ETEH2ETSUGCC\n",
"pFERMbGyAEknpL99ckv4xAvwxQEI3cysc0gaBYwaiLLKTBpPAcML28NJtY2izYFxKV+wPLCTpFkR\n",
"Mb7rhIh4RdKfgA8AEytvEhEnAEgcT2c9DWZmNiDyF+6JXduSju9rWWV+yN4BrC1pdUkLA3sC44sn\n",
"RMSaEbFGRKxB6tc4OCLGS1pe0tIAkhYFPkJaTKkWN0+ZmZWstJpGRMyWdCgwgdSRfW5ETJV0UD5+\n",
"Vo3LVwZ+KWkQKbFdGBHX9XDLxUi1GzMzK0knradxDvCPCC/lamZWS3/W0+ikPgA3T5mZlazTkobH\n",
"aZiZlaiTkoYH95mZlayTkoabp8zMStZpScM1DTOzEnVS0nDzlJlZyTopabh5ysysZJ2WNFzTMDMr\n",
"USclDTdPmZmVrCOShoRw85SZWek6ImmQFmp6O4LZzQ7EzKyTdUrScNOUmVkDdErScCe4mVkDdFLS\n",
"cH+GmVnJOilpuKZhZlayTkka7tMwM2uATkkabp4yM2uATkoarmmYmZWsU5KGm6fMzBqgU5KGm6fM\n",
"zBqgk5KGaxpmZiXrlKTh5ikzswbolKTh5ikzswbopKThmoaZWck6JWm4ecrMrAE6JWm4ecrMrAE6\n",
"KWm4pmFmVrJOSRpunjIza4DSk4ak0ZIekDRN0tE1zhshabak3fL2cEnXS7pf0n2SvlrjNm6eMjNr\n",
"gFKThqTBwOnAaGB9YIyk9bo57xTgKkB59yzg8IjYANgC+Eq1azM3T5mZNUDZNY2RwPSImBERs4Bx\n",
"wK5VzhsLXArM7NoREc9GxOT899eBqcAq3dzHzVNmZg1QdtIYBjxR2H4y75tL0jBSIjkz74rKQiSt\n",
"DmwK3NrNfdw8ZWbWAGUnjfkSQBWnAcdERJCaplQ8KGkJUi3ka7nGUY2bp8zMGmBIyeU/BQwvbA8n\n",
"1TaKNgfGSQJYHthJ0qyIGC9pIeAy4NcRcUW1G0g6AY55N1zwZenZKyNi4kC/CDOzdiZpFDBqQMpK\n",
"X/DLIWkI8CCwPfA0cBswJiKmdnP++cCVEXG5Uhb5JfBCRBzezfkREZJ4CXhvBC+W8kLMzDpI12dn\n",
"X64ttXkqImYDhwITgCnAxRExVdJBkg7q4fItgb2AbSVNyj+juznXzVNmZg1Qak2jbJICYgjp8dzB\n",
"EXX1oZiZLdBatqbRIIsCbzphmJmVrxOShpumzMwapBOapwYBQyKY1ex4zMzaQX+ap8p+5LZ0uVnK\n",
"CcPMrAE6oXnKzMwaxEnDzMzq5qRhZmZ1c9IwM7O6OWmYmVndnDTMzKxuThpmZlY3Jw0zM6ubk4aZ\n",
"mdXNScPMzOrmpGFmZnVz0jAzs7o5aZiZWd2cNMzMrG5OGmZmVjcnDTMzq5uThpmZ1c1Jw8zM6uak\n",
"YWZmdXPSMDOzujlpmJlZ3Zw0zMysbk4aZmZWNycNMzOrW+lJQ9JoSQ9Imibp6BrnjZA0W9LuhX3n\n",
"SXpO0r1lx2lmZj0rNWlIGgycDowG1gfGSFqvm/NOAa6qOHR+vratSRrV7Bjq4TgHluMcOO0QI7RP\n",
"nP1Rdk1jJDA9ImZExCxgHLBrlfPGApcCM4s7I+Im4KWSY2yEUc0OoE6jmh1AnUY1O4A6jWp2AHUa\n",
"1ewA6jCq2QHUaVSzAyhb2UljGPBEYfvJvG8uScNIieTMvCtKjsnMzPqo7KRRTwI4DTgmIgJQ/jEz\n",
"sxak9FldUuHSFsAJETE6bx8LzImIUwrnPMI7iWJ54A3gSxExPh9fHbgyIjasUr5rJWZmfRARffqC\n",
"PmSgA6lwB7B2/uB/GtgTGFM8ISLW7Pq7pPNJCWJ8PYX39UWbmVnflNo8FRGzgUOBCcAU4OKImCrp\n",
"IEkH9XS9pIuAm4F1JD0h6YtlxmtmZrWV2jxlZmadpW1HhNc7aLDRJA2XdL2k+yXdJ+mref+ykq6R\n",
"9JCkqyUt3QKxDpY0SdKVLRzj0pIulTRV0hRJH2zROI/N/+b3SvqtpEVaIc5qA2RrxZVfx7T8u7Vj\n",
"k+P8Qf53v1vS5ZKGtmKchWNflzRH0rKtGqeksfk9vU9SsW+5/jgjou1+gMHAdGB1YCFgMrBes+PK\n",
"sa0EbJL/vgTwILAe8H3gqLz/aODkFoj1COA3wPi83Yox/hLYL/99CDC01eLM/w8fARbJ2xcD+7RC\n",
"nMDWwKbAvYV9VeMiDcCdnH+nVs+/Y4OaGOdHuu4PnNyqceb9w0mDkx8Flm3FOIFtgWuAhfL2Cn2J\n",
"s11rGvUOGmy4iHg2Iibnv78OTCWNTfkE6QOQ/OcnmxNhImlV4GPAObzz9FqrxTgU2DoizoPURxYR\n",
"r9BicQKvArOAxSQNARYjPfjR9Dij+gDZ7uLaFbgoImZFxAzSh8fIZsUZEddExJy8eSuwaivGmf0I\n",
"OKpiX6vFeTBwUv7MJCK6BlP3Ks52TRo9DhpsBfmpsU1J/+FXjIjn8qHngBWbFFaXHwP/Dcwp7Gu1\n",
"GNcAZko6X9Jdks6WtDgtFmdEvAj8EHiclCxejohraLE4C7qLaxXS71KXVvq92g/4c/57S8UpaVfg\n",
"yYi4p+JQS8UJrA38l6R/SJoo6QN5f6/ibNek0fK995KWAC4DvhYRrxWPRaoTNu01SNoZ+GdETKKb\n",
"wZTNjjEbAmwGnBERmwH/Ao4pntAKcUp6L3AYqWq/CrCEpL2K57RCnNXUEVfTY5b0P8B/IuK3NU5r\n",
"SpySFgO+ARxf3F3jkma+n0OAZSJiC9IXxktqnNttnO2aNJ4itSF2Gc68mbKpJC1EShgXRsQVefdz\n",
"klbKx1cG/tms+IAPA5+Q9ChwEbCdpAtbLEZI/6ZPRsTteftSUhJ5tsXi/ABwc0S8EOkx88uBD9F6\n",
"cXbp7t+58vdq1byvaSTtS2pG/XxhdyvF+V7Sl4W78+/TqsCdklakteKE9Pt0OUD+nZojaXl6GWe7\n",
"Jo25gwYlLUwaNFjXgMCySRJwLjAlIk4rHBpP6hwl/3lF5bWNEhHfiIjhEbEG8FngrxGxdyvFCKl/\n",
"CHhC0jp51w7A/cCVtFCcwAPAFpIWzf/+O5DGJbVanF26+3ceD3xW0sKS1iA1Z9zWhPiA9IQk6Rvx\n",
"rhHx78KhlokzIu6NiBUjYo38+/QksFlu/muZOLMrgO0A8u/UwhHxPL2NsxE9+SU9HbAT6cmk6cCx\n",
"zY6nENdWpH6CycCk/DMaWBa4FngIuBpYutmx5ni34Z2np1ouRmBj4HbgbtK3pKEtGudRpIR2L6lz\n",
"eaFWiJNUk3wa+A+pH/CLteIiNbVMJyXCjzYxzv2AacBjhd+jM1oozre63s+K44+Qn55qtTjz/8kL\n",
"8//RO4FRfYnTg/vMzKxu7do8ZWZmTeCkYWZmdXPSMDOzujlpmJlZ3Zw0zMysbk4aZmZWNycNa1t5\n",
"GupTC9tHSjq+1jW9KPsCSbsPRFk93OfTecr36yr2ryZpTHfXmTWLk4a1s/8An5K0XN4eyEFHfS4r\n",
"z3Rbr/2BAyJi+4r9awCfG4DyzQaUk4a1s1nAL4DDKw9U1hQkvZ7/HCXpBklXSHpY0smS9pZ0m6R7\n",
"JK1ZKGYHSbdLelDSx/P1g/PiQLflxYEOLJR7k6Q/kEaGV8YzJpd/r6ST877jgC2B8yR9v+KSk4Gt\n",
"lRbJOkzSPpLG5xrJNZIWywvt3JpnAP5ED/GtLOnGXN69krbq43tuCzh/Y7F2dwZwT5UP3cqaQnF7\n",
"I2Bd0noDjwJnR8RIpVUWx5KSkIDVImKEpLWA6/Of+5CmPh8paRHgb5KuzuVuCmwQEY8VbyxpFVIS\n",
"2Ax4Gbha0q4RcaKkbYGvR8RdFfEeDRwZEbvkMvbN5W8YES9L+h5wXUTsp7Ty3q2SrgX26ia+3YCr\n",
"IuJ7eX6sxXt+a83m56RhbS0iXpP0K+CrwJt1XnZ75PUkJE0HJuT995FWN4OUZC7J95gu6RFSotkR\n",
"2FDSHvm8pYC1gNnAbZUJIxsBXB8RL+R7/gb4L+AP+Xi1qbQr9wVwTUS8nLd3BHaRdGTeXgR4T434\n",
"bifVaBYCroiIu6u+M2Y9cNKwTnAacBdwfmHfbHLzq6RBwMKFY28V/j6nsD2H2r8TXbWVQyMtsjSX\n",
"pFGk9T66u66YBMS8NZ96+08qy98tIqZVxFE1vnxsa2Bn4AJJP4qIC+u8r9lc7tOwthcRL5FqBfvz\n",
"zgfwDGDz/PdPkGb47A0Bn1byXmBN0gygE4BDujqjJa2jtBBPLbcD20haTtJg0nT0N/RwzavAkhXx\n",
"FE0g1a7IcWxa2D9ffJLeA8yMiHNIS/xuilkfuKZh7az4Df2HwKGF7bOBP0iaDFwFvN7NdZXlReHv\n",
"j5PWFVgKOCgi/iPpHNKiO3flvoF/Ap+quHbeQiOekXQMcD3pw/+PEXFlD6/tHuDtHP8FpP6XYvnf\n",
"Bk6TdA/py98jpOTYXXyjgP+WNAt4DfhCD/c3q8pTo5uZWd3cPGVmZnVz0jAzs7o5aZiZWd2cNMzM\n",
"rG5OGmZmVjcnDTMzq5uThpmZ1c1Jw8zM6vb/Y0BwwYi8XgMAAAAASUVORK5CYII=\n"
],
"text/plain": [
"<matplotlib.figure.Figure at 0x18f38160>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%time\n",
"learning_curve_ndcg(lmart,Xts,Yts,Qts,rank = 10)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Bagged GBDT (mse)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 18min 54s\n"
]
}
],
"source": [
"%%time\n",
"from sklearn.ensemble import BaggingRegressor\n",
"from sklearn.ensemble import GradientBoostingRegressor\n",
"gb = GradientBoostingRegressor(learning_rate=0.07,loss='ls',n_estimators=150, max_depth=6,\n",
" verbose=0, random_state=42)#verbose >= 2 to show OOB scores\n",
"bgb = BaggingRegressor(base_estimator=gb,n_estimators=50,n_jobs =-1)\n",
"bgb.fit(Xtr, Ytr)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train:\n",
"NDCG5 : 0.629402717766\n",
"NDCG10 : 0.640113217848\n",
"NDCGfull: 0.71993510113\n",
"rank AUC: 0.867161231555\n",
"MSE : 0.204445368295\n",
"test\n",
"NDCG5 : 0.435450537551\n",
"NDCG10 : 0.455145012167\n",
"NDCGfull: 0.618441849627\n",
"rank AUC: 0.692347321722\n",
"MSE : 0.296617139072\n",
"Wall time: 19 s\n"
]
}
],
"source": [
"%%time\n",
"print \"train:\"\n",
"score(bgb,Xtr,Ytr,Qtr)\n",
"print \"test\"\n",
"score(bgb,Xts,Yts,Qts)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## bagged lmart (nDCG)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 31min 37s\n"
]
}
],
"source": [
"%%time\n",
"from lambdamart import LambdaMART\n",
"from bagged_ranking import BaggingRanker\n",
"lmart= LambdaMART(n_estimators=150, max_depth=6,\n",
" learning_rate=0.1, max_rank=70,gain = \"exponential\")\n",
"blmart = BaggingRanker(base_estimator=lmart,n_estimators=50,sample_whole_queries=True,bootstrap=True,n_jobs =-1,max_samples=1.)\n",
"\n",
"blmart.fit(Xtr,Ytr,Q = Qtr)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train:\n",
"NDCG5 : 0.634078012056\n",
"NDCG10 : 0.635608697184\n",
"NDCGfull: 0.731482172316\n",
"rank AUC: 0.749955084222\n",
"MSE : 3.4084853623\n",
"test\n",
"NDCG5 : 0.44761318217\n",
"NDCG10 : 0.470524871397\n",
"NDCGfull: 0.625900891177\n",
"rank AUC: 0.675724301816\n",
"MSE : 3.46963788724\n",
"Wall time: 16.5 s\n"
]
}
],
"source": [
"%%time\n",
"print \"train:\"\n",
"score(blmart,Xtr,Ytr,Qtr)\n",
"print \"test\"\n",
"score(blmart,Xts,Yts,Qts)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"#from sklearn.externals import joblib\n",
"#joblib.dump(blmart, 'dumps/20x300 ndcg-50 bag-whole-querries.pkl') "
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.9"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment