Skip to content

Instantly share code, notes, and snippets.

@markste-in

markste-in/ray_issue.ipynb

Created January 3, 2022 22:59
Show Gist options
  • Save markste-in/5a80cb9b5d7a71888c31f1c1d53c544e to your computer and use it in GitHub Desktop.
Save markste-in/5a80cb9b5d7a71888c31f1c1d53c544e to your computer and use it in GitHub Desktop.
Download ZIP
ray_issue.ipynb
Display the source blob
Display the rendered blob
Raw
ray_issue.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "ray_issue.ipynb",
"provenance": [],
"collapsed_sections": [],
"authorship_tag": "ABX9TyPAUyk38pk0aH8ihbIfEYye",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/markste-in/5a80cb9b5d7a71888c31f1c1d53c544e/ray_issue.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "lM_MokjW-iop",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "0cf85b6a-9dd1-4332-963d-e20f99c2e7eb"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\u001b[K |████████████████████████████████| 57.6 MB 1.1 MB/s \n",
"\u001b[K |████████████████████████████████| 171 kB 7.7 MB/s \n",
"\u001b[K |████████████████████████████████| 54 kB 1.7 MB/s \n",
"\u001b[K |████████████████████████████████| 65 kB 1.5 MB/s \n",
"\u001b[K |████████████████████████████████| 3.0 MB 19.0 MB/s \n",
"\u001b[K |████████████████████████████████| 58 kB 3.0 MB/s \n",
"\u001b[K |████████████████████████████████| 201 kB 52.8 MB/s \n",
"\u001b[K |████████████████████████████████| 82 kB 220 kB/s \n",
"\u001b[K |████████████████████████████████| 51 kB 598 kB/s \n",
"\u001b[K |████████████████████████████████| 192 kB 46.5 MB/s \n",
"\u001b[K |████████████████████████████████| 1.1 MB 47.6 MB/s \n",
"\u001b[K |████████████████████████████████| 128 kB 61.9 MB/s \n",
"\u001b[K |████████████████████████████████| 271 kB 64.4 MB/s \n",
"\u001b[K |████████████████████████████████| 160 kB 70.3 MB/s \n",
"\u001b[K |████████████████████████████████| 85 kB 901 kB/s \n",
"\u001b[K |████████████████████████████████| 57 kB 4.7 MB/s \n",
"\u001b[K |████████████████████████████████| 10.9 MB 35.3 MB/s \n",
"\u001b[K |████████████████████████████████| 61 kB 221 kB/s \n",
"\u001b[K |████████████████████████████████| 78 kB 6.4 MB/s \n",
"\u001b[K |████████████████████████████████| 54 kB 2.9 MB/s \n",
"\u001b[?25h Building wheel for gpustat (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
]
}
],
"source": [
"!pip install ray[\"serve\"] -q"
]
},
{
"cell_type": "code",
"source": [
"from sklearn.pipeline import Pipeline\n",
"from sklearn.preprocessing import StandardScaler\n",
"from sklearn.decomposition import PCA\n",
"import numpy as np\n",
"import pickle\n",
"import os"
],
"metadata": {
"id": "7BFmmnBY-q3U"
},
"execution_count": 2,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import sys\n",
"sys.version"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 35
},
"id": "4VoXerZ7EuLG",
"outputId": "2de688f4-2e41-448e-aa22-bc8c5429a941"
},
"execution_count": 3,
"outputs": [
{
"output_type": "execute_result",
"data": {
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "string"
},
"text/plain": [
"'3.7.12 (default, Sep 10 2021, 00:21:48) \\n[GCC 7.5.0]'"
]
},
"metadata": {},
"execution_count": 3
}
]
},
{
"cell_type": "code",
"source": [
"pipe = Pipeline([('scaler', StandardScaler()), ('pca', PCA(n_components=2))])"
],
"metadata": {
"id": "Ib6BCpa1_XlU"
},
"execution_count": 4,
"outputs": []
},
{
"cell_type": "code",
"source": [
"data = np.random.rand(100,100)"
],
"metadata": {
"id": "cbGiL412_lTc"
},
"execution_count": 5,
"outputs": []
},
{
"cell_type": "code",
"source": [
"pipe.fit(data)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "-kdPDymwAFRM",
"outputId": "25fe447c-661d-477c-b544-e261a365aa55"
},
"execution_count": 6,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"Pipeline(steps=[('scaler', StandardScaler()), ('pca', PCA(n_components=2))])"
]
},
"metadata": {},
"execution_count": 6
}
]
},
{
"cell_type": "code",
"source": [
"file_path = os.path.join(os.getcwd(),\"dump.pkl\")\n",
"print(\"Writing dump to\", file_path)\n",
"with open(file_path,\"wb\") as f:\n",
" pickle.dump(pipe,f)"
],
"metadata": {
"id": "0kJXdVBtAlr2",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "b724243b-52ec-43b0-b8bf-c0151939d306"
},
"execution_count": 8,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Writing dump to /content/dump.pkl\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import ray\n",
"from ray import serve\n",
"ray.__version__"
],
"metadata": {
"id": "G_K7lr6MA7cp",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 35
},
"outputId": "1e8cc0fc-9996-4ff8-b45c-0c16c02a7836"
},
"execution_count": 9,
"outputs": [
{
"output_type": "execute_result",
"data": {
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "string"
},
"text/plain": [
"'1.9.1'"
]
},
"metadata": {},
"execution_count": 9
}
]
},
{
"cell_type": "code",
"source": [
"serve.start()"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "CyRgW_e0BDmi",
"outputId": "ba3f91ab-62d5-4a33-c477-948b2e6d16d0"
},
"execution_count": 10,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"2022-01-03 22:59:03,410\tINFO services.py:1340 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n",
"\u001b[2m\u001b[36m(ServeController pid=293)\u001b[0m 2022-01-03 22:59:06,270\tINFO checkpoint_path.py:16 -- Using RayInternalKVStore for controller checkpoint and recovery.\n",
"\u001b[2m\u001b[36m(ServeController pid=293)\u001b[0m 2022-01-03 22:59:06,275\tINFO http_state.py:101 -- Starting HTTP proxy with name 'SERVE_CONTROLLER_ACTOR:irGDtl:SERVE_PROXY_ACTOR-node:172.28.0.2-0' on node 'node:172.28.0.2-0' listening on '127.0.0.1:8000'\n",
"2022-01-03 22:59:06,978\tINFO api.py:463 -- Started Serve instance in namespace 'serve'.\n",
"\u001b[2m\u001b[36m(HTTPProxyActor pid=292)\u001b[0m INFO: Started server process [292]\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<ray.serve.api.Client at 0x7f8cb38af7d0>"
]
},
"metadata": {},
"execution_count": 10
}
]
},
{
"cell_type": "code",
"source": [
"@serve.deployment(num_replicas=1)\n",
"class Model():\n",
" def __init__(self):\n",
" print(\"Trying to open pipeline file\", file_path)\n",
" with open(file_path,\"wb\") as f:\n",
" self.pipe = pickle.load(f)\n",
" def __call__(self):\n",
" return self.pipe.transform(np.random.random(100,100))\n"
],
"metadata": {
"id": "eAYc44PaBJ5i"
},
"execution_count": 13,
"outputs": []
},
{
"cell_type": "code",
"source": [
"Model.deploy()"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 453
},
"id": "8wwATjQFC0bY",
"outputId": "c30bb191-9542-48da-d02d-29b3083074bf"
},
"execution_count": 14,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"2022-01-03 22:59:15,790\tINFO api.py:242 -- Updating deployment 'Model'. component=serve deployment=Model\n",
"\u001b[2m\u001b[36m(ServeController pid=293)\u001b[0m 2022-01-03 22:59:15,800\tINFO deployment_state.py:912 -- Adding 1 replicas to deployment 'Model'. component=serve deployment=Model\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"\u001b[2m\u001b[36m(Model pid=481)\u001b[0m Trying to open pipeline file /content/dump.pkl\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"\u001b[2m\u001b[36m(ServeController pid=293)\u001b[0m 2022-01-03 22:59:17,393\tINFO deployment_state.py:912 -- Adding 1 replicas to deployment 'Model'. component=serve deployment=Model\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"\u001b[2m\u001b[36m(Model pid=513)\u001b[0m Trying to open pipeline file /content/dump.pkl\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"\u001b[2m\u001b[36m(ServeController pid=293)\u001b[0m 2022-01-03 22:59:18,994\tINFO deployment_state.py:912 -- Adding 1 replicas to deployment 'Model'. component=serve deployment=Model\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"\u001b[2m\u001b[36m(Model pid=544)\u001b[0m Trying to open pipeline file /content/dump.pkl\n"
]
},
{
"output_type": "error",
"ename": "RuntimeError",
"evalue": "ignored",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-14-4f0efa4e4d04>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mModel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeploy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ray/serve/api.py\u001b[0m in \u001b[0;36mdeploy\u001b[0;34m(self, _blocking, *init_args, **init_kwargs)\u001b[0m\n\u001b[1;32m 798\u001b[0m \u001b[0mroute_prefix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_route_prefix\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 799\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 800\u001b[0;31m _blocking=_blocking)\n\u001b[0m\u001b[1;32m 801\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 802\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mPublicAPI\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ray/serve/api.py\u001b[0m in \u001b[0;36mcheck\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shutdown\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRayServeException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Client has already been shut down.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mcheck\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ray/serve/api.py\u001b[0m in \u001b[0;36mdeploy\u001b[0;34m(self, name, deployment_def, init_args, init_kwargs, ray_actor_options, config, version, prev_version, route_prefix, url, _blocking)\u001b[0m\n\u001b[1;32m 246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 247\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m_blocking\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 248\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_goal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgoal_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 249\u001b[0m logger.info(\n\u001b[1;32m 250\u001b[0m \u001b[0;34mf\"Deployment '{name}{':'+version if version else ''}' is ready\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ray/serve/api.py\u001b[0m in \u001b[0;36m_wait_for_goal\u001b[0;34m(self, goal_id, timeout)\u001b[0m\n\u001b[1;32m 182\u001b[0m \u001b[0masync_goal_exception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mready\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 183\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0masync_goal_exception\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 184\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0masync_goal_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 185\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mRuntimeError\u001b[0m: Deployment 'Model' failed, deleting it asynchronously."
]
}
]
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "b00wQ7k_C3r8"
},
"execution_count": null,
"outputs": []
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment