pytorch_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resnet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "resnet = torchvision.models.resnet18(pretrained=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.random.randn(1,3*224*224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = resnet(torch.Tensor(arr).view(-1, 3, 224, 224))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_func(model, xs):\n",
    "    import torch\n",
    "    import json\n",
    "    model.eval()\n",
    "    \n",
    "    # xs is numpy array of shape (batch_size, 3x224x224)\n",
    "    inp_tensor = torch.Tensor(xs).view(-1, 3, 224, 224)\n",
    "    print(inp_tensor.shape)\n",
    "    result = model(inp_tensor)\n",
    "    \n",
    "    # result is a pytorch variable of shape (batch_size, 1000)\n",
    "    result_np = result.detach().numpy()\n",
    "    print(result_np.shape)\n",
    "    result_string = [json.dumps(single_res.tolist()) for single_res in result_np]\n",
    "    \n",
    "    # result_string is a list of json serialized 1000 element string\n",
    "    return result_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 3, 224, 224])\n",
      "(10, 1000)\n"
     ]
    }
   ],
   "source": [
    "result = predict_func(resnet, np.random.randn(10, 3*224*224))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clipper"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Boilerplate to start clipper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from clipper_admin import ClipperConnection, DockerContainerManager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "clipper_conn = ClipperConnection(DockerContainerManager())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18-07-12:17:59:46 INFO     [docker_container_manager.py:119] Starting managed Redis instance in Docker\n",
      "18-07-12:17:59:54 INFO     [clipper_admin.py:126] Clipper is running\n"
     ]
    }
   ],
   "source": [
    "clipper_conn.start_clipper()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Register App"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18-07-12:17:59:54 INFO     [clipper_admin.py:201] Application scalabel was successfully registered\n"
     ]
    }
   ],
   "source": [
    "clipper_conn.register_application(\n",
    "    name='scalabel',\n",
    "    input_type='floats',\n",
    "    default_output='This query time out',\n",
    "    slo_micros=1000000*10 # 1000000 is one second, so our slo is 10 seconds\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['scalabel']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clipper_conn.get_all_apps()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deploy Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from clipper_admin.deployers.pytorch import deploy_pytorch_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The following code might take more than few minutes at the first time, because it will download about 1G image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18-07-12:17:59:54 INFO     [deployer_utils.py:44] Saving function to /tmp/clipper/tmpod8xc69e\n",
      "18-07-12:17:59:54 INFO     [deployer_utils.py:54] Serialized and supplied predict function\n",
      "18-07-12:18:00:21 INFO     [pytorch.py:204] Torch model saved\n",
      "18-07-12:18:00:21 INFO     [pytorch.py:217] Using Python 3.6 base image\n",
      "18-07-12:18:00:22 INFO     [clipper_admin.py:452] Building model Docker image with model data from /tmp/clipper/tmpod8xc69e\n",
      "18-07-12:18:00:27 INFO     [clipper_admin.py:456] {'stream': 'Step 1/2 : FROM clipper/pytorch36-container:0.3.0'}\n",
      "18-07-12:18:00:27 INFO     [clipper_admin.py:456] {'stream': '\\n'}\n",
      "18-07-12:18:00:27 INFO     [clipper_admin.py:456] {'stream': ' ---> 37545e712105\\n'}\n",
      "18-07-12:18:00:27 INFO     [clipper_admin.py:456] {'stream': 'Step 2/2 : COPY /tmp/clipper/tmpod8xc69e /model/'}\n",
      "18-07-12:18:00:27 INFO     [clipper_admin.py:456] {'stream': '\\n'}\n",
      "18-07-12:18:00:27 INFO     [clipper_admin.py:456] {'stream': ' ---> 02116d879238\\n'}\n",
      "18-07-12:18:00:27 INFO     [clipper_admin.py:456] {'aux': {'ID': 'sha256:02116d879238637b0e83214cb1ae5ef207388b398a5af9ba5b7f52444b471433'}}\n",
      "18-07-12:18:00:27 INFO     [clipper_admin.py:456] {'stream': 'Successfully built 02116d879238\\n'}\n",
      "18-07-12:18:00:27 INFO     [clipper_admin.py:456] {'stream': 'Successfully tagged resnet-18:1\\n'}\n",
      "18-07-12:18:00:27 INFO     [clipper_admin.py:458] Pushing model Docker image to resnet-18:1\n",
      "18-07-12:18:00:28 INFO     [docker_container_manager.py:257] Found 0 replicas for resnet-18:1. Adding 1\n",
      "18-07-12:18:00:35 INFO     [clipper_admin.py:635] Successfully registered model resnet-18:1\n",
      "18-07-12:18:00:35 INFO     [clipper_admin.py:553] Done deploying model resnet-18:1.\n"
     ]
    }
   ],
   "source": [
    "deploy_pytorch_model(\n",
    "    clipper_conn=clipper_conn,\n",
    "    name='resnet-18',\n",
    "    version=1,\n",
    "    input_type='floats',\n",
    "    func=predict_func,\n",
    "    pytorch_model=resnet\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Link this Model to the Application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18-07-12:18:00:35 INFO     [clipper_admin.py:263] Model resnet-18 is now linked to application scalabel\n"
     ]
    }
   ],
   "source": [
    "clipper_conn.link_model_to_app(app_name='scalabel', model_name='resnet-18')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clipper thing is over"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'localhost:1337'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clipper_conn.get_query_addr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['scalabel']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clipper_conn.get_all_apps()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "addr = 'http://' + clipper_conn.get_query_addr() + '/' + 'scalabel' + '/predict'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'http://localhost:1337/scalabel/predict'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "addr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "payload = {\n",
    "    'input': np.random.randn(3*224*224).tolist()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = requests.post(addr, json=payload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = res.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['query_id', 'output', 'default'])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = result['output']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_arr = np.array(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Teardown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18-07-12:18:03:22 INFO     [clipper_admin.py:1258] Stopped all Clipper cluster and all model containers\n"
     ]
    }
   ],
   "source": [
    "clipper_conn.stop_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}