vaex-catboost.ipynb

vaex-catboost.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "vaex-catboost.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/domvwt/8dee71dcf26a721ce42464e0809b034e/vaex-catboost.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ywOOu8YGu0Mj"
      },
      "source": [
        "# Out-of-Core Gradient Boosting with Vaex and Catboost"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mM2JrQ4VpJaY",
        "outputId": "38cc7c04-c096-4ad0-fab2-0f3d8ab19b5a"
      },
      "source": [
        "%%shell\r\n",
        "echo \"Setting up environment...\"\r\n",
        "if [ ! -e \"data/yellow_tripdata_2019-12.csv\" ]\r\n",
        "  then \r\n",
        "    echo \"...Downloading data...\" \r\n",
        "    wget -q \\\r\n",
        "      https://nyc-tlc.s3.amazonaws.com/trip+data/yellow_tripdata_2019-{10..12}.csv \\\r\n",
        "      -P data/\r\n",
        "fi\r\n",
        "echo \"...Installing libraries...\"\r\n",
        "pip install -Uqq vaex catboost optuna \"ipython>=7.0.0\" &> /dev/null\r\n",
        "echo \"Setup Complete!\"\r\n",
        "echo \"Runtime restart may be required to load new packages.\""
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setting up environment...\n",
            "...Installing libraries...\n",
            "Setup Complete!\n",
            "Runtime restart may be required to load new packages.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tO6_1lOhugqt"
      },
      "source": [
        "import gc\n",
        "import vaex as vx\n",
        "import vaex.ml.catboost as cb\n",
        "\n",
        "from pathlib import Path"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Xulcfa3zyLa"
      },
      "source": [
        "data_files = Path(\"data\").iterdir()"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t13hPzuV0PfH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2042d7f4-957f-4724-97ab-80f55646ab59"
      },
      "source": [
        "# Vaex can read in multiple files and combine them\r\n",
        "if not Path(\"data/taxi-data-combined.hdf5\").is_file():\r\n",
        "  for f in data_files:\r\n",
        "    df_temp = vx.from_csv(str(f), copy_index=False, convert=True)\r\n",
        "\r\n",
        "  # Slight performance gains made by converting files and reloading as one \r\n",
        "  df00 = vx.open(\"data/*.hdf5\")\r\n",
        "  df00.export(\"data/taxi-data-combined.hdf5\")\r\n",
        "  \r\n",
        "df00 = vx.open(\"data/taxi-data-combined.hdf5\")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py:3343: DtypeWarning: Columns (6) have mixed types.Specify dtype option on import or set low_memory=False.\n",
            "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 885
        },
        "id": "4N3ft6K2l_CK",
        "outputId": "1cdc67bb-8962-4000-f176-1f3d80bb0f3f"
      },
      "source": [
        "# Very fast summary\r\n",
        "df00.info()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>.vaex-description pre {\n",
              "          max-width : 450px;\n",
              "          white-space : nowrap;\n",
              "          overflow : hidden;\n",
              "          text-overflow: ellipsis;\n",
              "        }\n",
              "\n",
              "        .vex-description pre:hover {\n",
              "          max-width : initial;\n",
              "          white-space: pre;\n",
              "        }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div><h2>taxi-data-combined</h2> <b>rows</b>: 20,988,319</div><div><b>path</b>: <i>/content/data/taxi-data-combined.hdf5</i></div><div><b>Description</b>: file exported by vaex, by user root, on date 2021-02-02 19:11:58.646962, from source /has/no/path/arrays-/has/no/path/arrays-/has/no/path/arrays</div><h2>Columns:</h2><table class='table-striped'><thead><tr><th>column</th><th>type</th><th>unit</th><th>description</th><th>expression</th></tr></thead><tr><td>VendorID</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>tpep_pickup_datetime</td><td>str</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>tpep_dropoff_datetime</td><td>str</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>passenger_count</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>trip_distance</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>RatecodeID</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>store_and_fwd_flag</td><td>str</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>PULocationID</td><td>int64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>DOLocationID</td><td>int64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>payment_type</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>fare_amount</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>extra</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>mta_tax</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>tip_amount</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>tolls_amount</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>improvement_surcharge</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>total_amount</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>congestion_surcharge</td><td>float64</td><td></td><td ><pre></pre></td><td></td></tr></table><h2>Data:</h2><table>\n",
              "<thead>\n",
              "<tr><th>#                                     </th><th>VendorID  </th><th>tpep_pickup_datetime  </th><th>tpep_dropoff_datetime  </th><th>passenger_count  </th><th>trip_distance  </th><th>RatecodeID  </th><th>store_and_fwd_flag  </th><th>PULocationID  </th><th>DOLocationID  </th><th>payment_type  </th><th>fare_amount  </th><th>extra  </th><th>mta_tax  </th><th>tip_amount  </th><th>tolls_amount  </th><th>improvement_surcharge  </th><th>total_amount  </th><th>congestion_surcharge  </th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td><i style='opacity: 0.6'>0</i>         </td><td>1.0       </td><td>2019-10-01 00:19:55   </td><td>2019-10-01 00:23:57    </td><td>1.0              </td><td>0.4            </td><td>1.0         </td><td>N                   </td><td>48            </td><td>163           </td><td>2.0           </td><td>4.5          </td><td>3.0    </td><td>0.5      </td><td>0.0         </td><td>0.0           </td><td>0.3                    </td><td>8.3           </td><td>2.5                   </td></tr>\n",
              "<tr><td><i style='opacity: 0.6'>1</i>         </td><td>1.0       </td><td>2019-10-01 00:40:19   </td><td>2019-10-01 00:55:17    </td><td>2.0              </td><td>4.3            </td><td>1.0         </td><td>N                   </td><td>144           </td><td>141           </td><td>1.0           </td><td>14.5         </td><td>3.0    </td><td>0.5      </td><td>2.0         </td><td>0.0           </td><td>0.3                    </td><td>20.3          </td><td>2.5                   </td></tr>\n",
              "<tr><td><i style='opacity: 0.6'>2</i>         </td><td>1.0       </td><td>2019-10-01 00:06:52   </td><td>2019-10-01 00:21:23    </td><td>1.0              </td><td>5.0            </td><td>1.0         </td><td>N                   </td><td>137           </td><td>80            </td><td>1.0           </td><td>17.0         </td><td>3.0    </td><td>0.5      </td><td>5.2         </td><td>0.0           </td><td>0.3                    </td><td>26.0          </td><td>2.5                   </td></tr>\n",
              "<tr><td><i style='opacity: 0.6'>3</i>         </td><td>2.0       </td><td>2019-10-01 00:36:08   </td><td>2019-10-01 00:36:15    </td><td>1.0              </td><td>0.0            </td><td>1.0         </td><td>N                   </td><td>25            </td><td>25            </td><td>4.0           </td><td>-2.5         </td><td>-0.5   </td><td>-0.5     </td><td>0.0         </td><td>0.0           </td><td>-0.3                   </td><td>-3.8          </td><td>0.0                   </td></tr>\n",
              "<tr><td><i style='opacity: 0.6'>4</i>         </td><td>2.0       </td><td>2019-10-01 00:36:08   </td><td>2019-10-01 00:36:15    </td><td>1.0              </td><td>0.0            </td><td>1.0         </td><td>N                   </td><td>25            </td><td>25            </td><td>2.0           </td><td>2.5          </td><td>0.5    </td><td>0.5      </td><td>0.0         </td><td>0.0           </td><td>0.3                    </td><td>3.8           </td><td>0.0                   </td></tr>\n",
              "<tr><td>...                                   </td><td>...       </td><td>...                   </td><td>...                    </td><td>...              </td><td>...            </td><td>...         </td><td>...                 </td><td>...           </td><td>...           </td><td>...           </td><td>...          </td><td>...    </td><td>...      </td><td>...         </td><td>...           </td><td>...                    </td><td>...           </td><td>...                   </td></tr>\n",
              "<tr><td><i style='opacity: 0.6'>20,988,314</i></td><td>nan       </td><td>2019-12-31 00:07:00   </td><td>2019-12-31 00:46:00    </td><td>nan              </td><td>12.78          </td><td>nan         </td><td>None                </td><td>230           </td><td>72            </td><td>nan           </td><td>32.32        </td><td>2.75   </td><td>0.5      </td><td>0.0         </td><td>6.12          </td><td>0.3                    </td><td>41.99         </td><td>0.0                   </td></tr>\n",
              "<tr><td><i style='opacity: 0.6'>20,988,315</i></td><td>nan       </td><td>2019-12-31 00:20:00   </td><td>2019-12-31 00:47:00    </td><td>nan              </td><td>18.52          </td><td>nan         </td><td>None                </td><td>219           </td><td>32            </td><td>nan           </td><td>51.63        </td><td>2.75   </td><td>0.5      </td><td>0.0         </td><td>6.12          </td><td>0.3                    </td><td>61.3          </td><td>0.0                   </td></tr>\n",
              "<tr><td><i style='opacity: 0.6'>20,988,316</i></td><td>nan       </td><td>2019-12-31 00:50:00   </td><td>2019-12-31 01:21:00    </td><td>nan              </td><td>13.13          </td><td>nan         </td><td>None                </td><td>161           </td><td>76            </td><td>nan           </td><td>38.02        </td><td>2.75   </td><td>0.5      </td><td>0.0         </td><td>6.12          </td><td>0.3                    </td><td>47.69         </td><td>0.0                   </td></tr>\n",
              "<tr><td><i style='opacity: 0.6'>20,988,317</i></td><td>nan       </td><td>2019-12-31 00:38:19   </td><td>2019-12-31 01:19:37    </td><td>nan              </td><td>14.51          </td><td>nan         </td><td>None                </td><td>230           </td><td>21            </td><td>nan           </td><td>41.86        </td><td>2.75   </td><td>0.0      </td><td>0.0         </td><td>6.12          </td><td>0.3                    </td><td>51.03         </td><td>0.0                   </td></tr>\n",
              "<tr><td><i style='opacity: 0.6'>20,988,318</i></td><td>nan       </td><td>2019-12-31 00:21:00   </td><td>2019-12-31 00:56:00    </td><td>nan              </td><td>-17.16         </td><td>nan         </td><td>None                </td><td>193           </td><td>219           </td><td>nan           </td><td>44.62        </td><td>2.75   </td><td>0.5      </td><td>0.0         </td><td>0.0           </td><td>0.3                    </td><td>48.17         </td><td>0.0                   </td></tr>\n",
              "</tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3TiEihMuvtxi"
      },
      "source": [
        "# Efficient shuffle and train / test split\r\n",
        "df_train, df_test = df00.sample(frac=1).ml.train_test_split(verbose=False)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-7vln7ogxvwJ"
      },
      "source": [
        "target = \"tip_amount\"\r\n",
        "features = [x for x in df00.get_column_names() if x != target]"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "co7rL9rJwIM6"
      },
      "source": [
        "# Collect garbage because Python leaves trash everywhere\r\n",
        "gc.collect()\r\n",
        "\r\n",
        "# Catboost model parameters\r\n",
        "cbm_params = dict(\r\n",
        "  loss_function=\"MAE\",\r\n",
        "  early_stopping_rounds=10,\r\n",
        "  verbose=10,\r\n",
        "  used_ram_limit=\"8gb\"\r\n",
        ")\r\n",
        "\r\n",
        "# Vaex wrapper parameters\r\n",
        "vcb_params = dict(\r\n",
        "    features=features,\r\n",
        "    target=target,\r\n",
        "    params=cbm_params\r\n",
        ")    \r\n",
        "\r\n",
        "cbm = cb.CatBoostModel(**vcb_params)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TMO2QjHewgC5",
        "outputId": "0e2242ef-4e2d-47ef-b2ba-874e957275e1"
      },
      "source": [
        "# ~1.3hrs for 1000 iterations over 21 million records (on CPU only!)\r\n",
        "cbm.fit(df_train, evals=[df_test])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0:\tlearn: 1.6928684\ttest: 1.7053601\tbest: 1.7053601 (0)\ttotal: 7.29s\tremaining: 2h 1m 23s\n",
            "10:\tlearn: 1.4139167\ttest: 1.4291894\tbest: 1.4291894 (10)\ttotal: 1m 12s\tremaining: 1h 48m 22s\n",
            "20:\tlearn: 1.2024777\ttest: 1.2196115\tbest: 1.2196115 (20)\ttotal: 2m 16s\tremaining: 1h 45m 53s\n",
            "30:\tlearn: 1.0537415\ttest: 1.0723423\tbest: 1.0723423 (30)\ttotal: 3m 15s\tremaining: 1h 42m 4s\n",
            "40:\tlearn: 0.9571214\ttest: 0.9773547\tbest: 0.9773547 (40)\ttotal: 4m 15s\tremaining: 1h 39m 46s\n",
            "50:\tlearn: 0.8716443\ttest: 0.8923921\tbest: 0.8923921 (50)\ttotal: 5m 14s\tremaining: 1h 37m 37s\n",
            "60:\tlearn: 0.8136729\ttest: 0.8348873\tbest: 0.8348873 (60)\ttotal: 6m 16s\tremaining: 1h 36m 33s\n",
            "70:\tlearn: 0.7546232\ttest: 0.7754324\tbest: 0.7754324 (70)\ttotal: 7m 13s\tremaining: 1h 34m 32s\n",
            "80:\tlearn: 0.7132797\ttest: 0.7339606\tbest: 0.7339606 (80)\ttotal: 8m 9s\tremaining: 1h 32m 31s\n",
            "90:\tlearn: 0.6653240\ttest: 0.6850152\tbest: 0.6850152 (90)\ttotal: 9m 3s\tremaining: 1h 30m 28s\n",
            "100:\tlearn: 0.6309959\ttest: 0.6501066\tbest: 0.6501066 (100)\ttotal: 9m 55s\tremaining: 1h 28m 20s\n",
            "110:\tlearn: 0.5908915\ttest: 0.6087920\tbest: 0.6087920 (110)\ttotal: 10m 46s\tremaining: 1h 26m 15s\n",
            "120:\tlearn: 0.5475404\ttest: 0.5639261\tbest: 0.5639261 (120)\ttotal: 11m 33s\tremaining: 1h 23m 59s\n",
            "130:\tlearn: 0.5162067\ttest: 0.5315280\tbest: 0.5315280 (130)\ttotal: 12m 20s\tremaining: 1h 21m 54s\n",
            "140:\tlearn: 0.4870182\ttest: 0.5014272\tbest: 0.5014272 (140)\ttotal: 13m 8s\tremaining: 1h 20m 5s\n",
            "150:\tlearn: 0.4645559\ttest: 0.4782631\tbest: 0.4782631 (150)\ttotal: 13m 56s\tremaining: 1h 18m 21s\n",
            "160:\tlearn: 0.4503989\ttest: 0.4636278\tbest: 0.4636278 (160)\ttotal: 14m 44s\tremaining: 1h 16m 48s\n",
            "170:\tlearn: 0.4363300\ttest: 0.4491676\tbest: 0.4491676 (170)\ttotal: 15m 31s\tremaining: 1h 15m 18s\n",
            "180:\tlearn: 0.4284812\ttest: 0.4410726\tbest: 0.4410726 (180)\ttotal: 16m 19s\tremaining: 1h 13m 51s\n",
            "190:\tlearn: 0.4223071\ttest: 0.4347403\tbest: 0.4347403 (190)\ttotal: 17m 5s\tremaining: 1h 12m 21s\n",
            "200:\tlearn: 0.4151820\ttest: 0.4274432\tbest: 0.4274432 (200)\ttotal: 17m 51s\tremaining: 1h 10m 59s\n",
            "210:\tlearn: 0.4102676\ttest: 0.4223745\tbest: 0.4223745 (210)\ttotal: 18m 39s\tremaining: 1h 9m 47s\n",
            "220:\tlearn: 0.4066448\ttest: 0.4186316\tbest: 0.4186316 (220)\ttotal: 19m 27s\tremaining: 1h 8m 35s\n",
            "230:\tlearn: 0.4049713\ttest: 0.4169115\tbest: 0.4169115 (230)\ttotal: 20m 17s\tremaining: 1h 7m 33s\n",
            "240:\tlearn: 0.4031656\ttest: 0.4150495\tbest: 0.4150495 (240)\ttotal: 21m 5s\tremaining: 1h 6m 25s\n",
            "250:\tlearn: 0.4019513\ttest: 0.4138107\tbest: 0.4138107 (250)\ttotal: 21m 53s\tremaining: 1h 5m 18s\n",
            "260:\tlearn: 0.3997633\ttest: 0.4115344\tbest: 0.4115344 (260)\ttotal: 22m 40s\tremaining: 1h 4m 12s\n",
            "270:\tlearn: 0.3987803\ttest: 0.4105182\tbest: 0.4105182 (270)\ttotal: 23m 28s\tremaining: 1h 3m 9s\n",
            "280:\tlearn: 0.3973583\ttest: 0.4090556\tbest: 0.4090556 (280)\ttotal: 24m 17s\tremaining: 1h 2m 10s\n",
            "290:\tlearn: 0.3945904\ttest: 0.4062663\tbest: 0.4062663 (290)\ttotal: 25m 6s\tremaining: 1h 1m 9s\n",
            "300:\tlearn: 0.3928638\ttest: 0.4044957\tbest: 0.4044957 (300)\ttotal: 25m 54s\tremaining: 1h 9s\n",
            "310:\tlearn: 0.3915851\ttest: 0.4031730\tbest: 0.4031730 (310)\ttotal: 26m 42s\tremaining: 59m 9s\n",
            "320:\tlearn: 0.3885112\ttest: 0.4001099\tbest: 0.4001099 (320)\ttotal: 27m 31s\tremaining: 58m 13s\n",
            "330:\tlearn: 0.3875525\ttest: 0.3991172\tbest: 0.3991172 (330)\ttotal: 28m 17s\tremaining: 57m 11s\n",
            "340:\tlearn: 0.3864145\ttest: 0.3979384\tbest: 0.3979384 (340)\ttotal: 29m 3s\tremaining: 56m 10s\n",
            "350:\tlearn: 0.3845533\ttest: 0.3960163\tbest: 0.3960163 (350)\ttotal: 29m 54s\tremaining: 55m 17s\n",
            "360:\tlearn: 0.3825845\ttest: 0.3939411\tbest: 0.3939411 (360)\ttotal: 30m 44s\tremaining: 54m 24s\n",
            "370:\tlearn: 0.3806977\ttest: 0.3919765\tbest: 0.3919765 (370)\ttotal: 31m 32s\tremaining: 53m 28s\n",
            "380:\tlearn: 0.3789194\ttest: 0.3901907\tbest: 0.3901907 (380)\ttotal: 32m 22s\tremaining: 52m 36s\n",
            "390:\tlearn: 0.3776134\ttest: 0.3888039\tbest: 0.3888039 (390)\ttotal: 33m 13s\tremaining: 51m 45s\n",
            "400:\tlearn: 0.3757225\ttest: 0.3868742\tbest: 0.3868742 (400)\ttotal: 34m 3s\tremaining: 50m 51s\n",
            "410:\tlearn: 0.3744337\ttest: 0.3855392\tbest: 0.3855392 (410)\ttotal: 34m 54s\tremaining: 50m 1s\n",
            "420:\tlearn: 0.3731948\ttest: 0.3842706\tbest: 0.3842706 (420)\ttotal: 35m 45s\tremaining: 49m 10s\n",
            "430:\tlearn: 0.3720340\ttest: 0.3830637\tbest: 0.3830637 (430)\ttotal: 36m 35s\tremaining: 48m 18s\n",
            "440:\tlearn: 0.3712737\ttest: 0.3822671\tbest: 0.3822671 (440)\ttotal: 37m 25s\tremaining: 47m 26s\n",
            "450:\tlearn: 0.3693821\ttest: 0.3803382\tbest: 0.3803382 (450)\ttotal: 38m 15s\tremaining: 46m 33s\n",
            "460:\tlearn: 0.3670183\ttest: 0.3778983\tbest: 0.3778983 (460)\ttotal: 39m 3s\tremaining: 45m 40s\n",
            "470:\tlearn: 0.3657383\ttest: 0.3765788\tbest: 0.3765788 (470)\ttotal: 39m 57s\tremaining: 44m 53s\n",
            "480:\tlearn: 0.3642693\ttest: 0.3750727\tbest: 0.3750727 (480)\ttotal: 40m 49s\tremaining: 44m 2s\n",
            "490:\tlearn: 0.3633873\ttest: 0.3741619\tbest: 0.3741619 (490)\ttotal: 41m 40s\tremaining: 43m 11s\n",
            "500:\tlearn: 0.3615285\ttest: 0.3722460\tbest: 0.3722460 (500)\ttotal: 42m 31s\tremaining: 42m 21s\n",
            "510:\tlearn: 0.3606710\ttest: 0.3713726\tbest: 0.3713726 (510)\ttotal: 43m 24s\tremaining: 41m 31s\n",
            "520:\tlearn: 0.3596970\ttest: 0.3703828\tbest: 0.3703828 (520)\ttotal: 44m 17s\tremaining: 40m 42s\n",
            "530:\tlearn: 0.3592577\ttest: 0.3699641\tbest: 0.3699641 (530)\ttotal: 45m 5s\tremaining: 39m 49s\n",
            "540:\tlearn: 0.3576823\ttest: 0.3683441\tbest: 0.3683441 (540)\ttotal: 45m 53s\tremaining: 38m 56s\n",
            "550:\tlearn: 0.3569612\ttest: 0.3676098\tbest: 0.3676098 (550)\ttotal: 46m 41s\tremaining: 38m 2s\n",
            "560:\tlearn: 0.3561434\ttest: 0.3667715\tbest: 0.3667715 (560)\ttotal: 47m 29s\tremaining: 37m 9s\n",
            "570:\tlearn: 0.3553208\ttest: 0.3659301\tbest: 0.3659301 (570)\ttotal: 48m 17s\tremaining: 36m 16s\n",
            "580:\tlearn: 0.3550750\ttest: 0.3656863\tbest: 0.3656863 (580)\ttotal: 49m 4s\tremaining: 35m 23s\n",
            "590:\tlearn: 0.3546644\ttest: 0.3652726\tbest: 0.3652726 (590)\ttotal: 49m 50s\tremaining: 34m 29s\n",
            "600:\tlearn: 0.3539238\ttest: 0.3645131\tbest: 0.3645131 (600)\ttotal: 50m 37s\tremaining: 33m 36s\n",
            "610:\tlearn: 0.3532087\ttest: 0.3637667\tbest: 0.3637667 (610)\ttotal: 51m 23s\tremaining: 32m 43s\n",
            "620:\tlearn: 0.3530105\ttest: 0.3635677\tbest: 0.3635677 (620)\ttotal: 52m 10s\tremaining: 31m 50s\n",
            "630:\tlearn: 0.3524065\ttest: 0.3629561\tbest: 0.3629561 (630)\ttotal: 52m 59s\tremaining: 30m 59s\n",
            "640:\tlearn: 0.3517995\ttest: 0.3623343\tbest: 0.3623343 (640)\ttotal: 53m 47s\tremaining: 30m 7s\n",
            "650:\tlearn: 0.3515637\ttest: 0.3621173\tbest: 0.3621173 (650)\ttotal: 54m 37s\tremaining: 29m 17s\n",
            "660:\tlearn: 0.3512135\ttest: 0.3617793\tbest: 0.3617793 (660)\ttotal: 55m 26s\tremaining: 28m 26s\n",
            "670:\tlearn: 0.3509650\ttest: 0.3615207\tbest: 0.3615207 (670)\ttotal: 56m 16s\tremaining: 27m 35s\n",
            "680:\tlearn: 0.3503143\ttest: 0.3608640\tbest: 0.3608640 (680)\ttotal: 57m 3s\tremaining: 26m 43s\n",
            "690:\tlearn: 0.3470166\ttest: 0.3575298\tbest: 0.3575298 (690)\ttotal: 57m 49s\tremaining: 25m 51s\n",
            "700:\tlearn: 0.3454248\ttest: 0.3559151\tbest: 0.3559151 (700)\ttotal: 58m 37s\tremaining: 25m\n",
            "710:\tlearn: 0.3442789\ttest: 0.3547381\tbest: 0.3547381 (710)\ttotal: 59m 25s\tremaining: 24m 9s\n",
            "720:\tlearn: 0.3432498\ttest: 0.3536738\tbest: 0.3536738 (720)\ttotal: 1h 13s\tremaining: 23m 18s\n",
            "730:\tlearn: 0.3424126\ttest: 0.3528033\tbest: 0.3528033 (730)\ttotal: 1h 1m 5s\tremaining: 22m 28s\n",
            "740:\tlearn: 0.3422675\ttest: 0.3526594\tbest: 0.3526594 (740)\ttotal: 1h 1m 53s\tremaining: 21m 37s\n",
            "750:\tlearn: 0.3420059\ttest: 0.3523926\tbest: 0.3523926 (750)\ttotal: 1h 2m 40s\tremaining: 20m 46s\n",
            "760:\tlearn: 0.3418179\ttest: 0.3522006\tbest: 0.3522006 (760)\ttotal: 1h 3m 28s\tremaining: 19m 55s\n",
            "770:\tlearn: 0.3410647\ttest: 0.3514332\tbest: 0.3514332 (770)\ttotal: 1h 4m 13s\tremaining: 19m 4s\n",
            "780:\tlearn: 0.3403482\ttest: 0.3506930\tbest: 0.3506930 (780)\ttotal: 1h 4m 59s\tremaining: 18m 13s\n",
            "790:\tlearn: 0.3399770\ttest: 0.3502981\tbest: 0.3502981 (790)\ttotal: 1h 5m 47s\tremaining: 17m 22s\n",
            "800:\tlearn: 0.3389604\ttest: 0.3492594\tbest: 0.3492594 (800)\ttotal: 1h 6m 35s\tremaining: 16m 32s\n",
            "810:\tlearn: 0.3384936\ttest: 0.3487687\tbest: 0.3487687 (810)\ttotal: 1h 7m 21s\tremaining: 15m 41s\n",
            "820:\tlearn: 0.3383249\ttest: 0.3485931\tbest: 0.3485931 (820)\ttotal: 1h 8m 7s\tremaining: 14m 51s\n",
            "830:\tlearn: 0.3381256\ttest: 0.3483885\tbest: 0.3483885 (830)\ttotal: 1h 8m 53s\tremaining: 14m\n",
            "840:\tlearn: 0.3378507\ttest: 0.3481054\tbest: 0.3481054 (840)\ttotal: 1h 9m 38s\tremaining: 13m 10s\n",
            "850:\tlearn: 0.3376397\ttest: 0.3478904\tbest: 0.3478904 (850)\ttotal: 1h 10m 24s\tremaining: 12m 19s\n",
            "860:\tlearn: 0.3374103\ttest: 0.3476592\tbest: 0.3476592 (860)\ttotal: 1h 11m 11s\tremaining: 11m 29s\n",
            "870:\tlearn: 0.3370881\ttest: 0.3473417\tbest: 0.3473417 (870)\ttotal: 1h 11m 56s\tremaining: 10m 39s\n",
            "880:\tlearn: 0.3367286\ttest: 0.3469834\tbest: 0.3469834 (880)\ttotal: 1h 12m 42s\tremaining: 9m 49s\n",
            "890:\tlearn: 0.3361878\ttest: 0.3464304\tbest: 0.3464304 (890)\ttotal: 1h 13m 28s\tremaining: 8m 59s\n",
            "900:\tlearn: 0.3361126\ttest: 0.3463604\tbest: 0.3463604 (900)\ttotal: 1h 14m 13s\tremaining: 8m 9s\n",
            "910:\tlearn: 0.3357000\ttest: 0.3459066\tbest: 0.3459066 (910)\ttotal: 1h 15m\tremaining: 7m 19s\n",
            "920:\tlearn: 0.3354891\ttest: 0.3456804\tbest: 0.3456804 (920)\ttotal: 1h 15m 47s\tremaining: 6m 30s\n",
            "930:\tlearn: 0.3353045\ttest: 0.3454820\tbest: 0.3454820 (930)\ttotal: 1h 16m 34s\tremaining: 5m 40s\n",
            "940:\tlearn: 0.3350048\ttest: 0.3451632\tbest: 0.3451632 (940)\ttotal: 1h 17m 21s\tremaining: 4m 50s\n",
            "950:\tlearn: 0.3348775\ttest: 0.3450330\tbest: 0.3450330 (950)\ttotal: 1h 18m 8s\tremaining: 4m 1s\n",
            "960:\tlearn: 0.3348215\ttest: 0.3449749\tbest: 0.3449749 (960)\ttotal: 1h 18m 54s\tremaining: 3m 12s\n",
            "970:\tlearn: 0.3346815\ttest: 0.3448349\tbest: 0.3448349 (970)\ttotal: 1h 19m 40s\tremaining: 2m 22s\n",
            "980:\tlearn: 0.3346708\ttest: 0.3448275\tbest: 0.3448275 (980)\ttotal: 1h 20m 26s\tremaining: 1m 33s\n",
            "990:\tlearn: 0.3346314\ttest: 0.3447902\tbest: 0.3447902 (990)\ttotal: 1h 21m 12s\tremaining: 44.2s\n",
            "999:\tlearn: 0.3345737\ttest: 0.3447316\tbest: 0.3447316 (999)\ttotal: 1h 21m 54s\tremaining: 0us\n",
            "\n",
            "bestTest = 0.3447316114\n",
            "bestIteration = 999\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M0RmUPCUzNhP"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}