RCNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras.backend\n",
    "import keras.engine\n",
    "import keras.layers\n",
    "import numpy\n",
    "import skimage.io\n",
    "import tensorflow\n",
    "import keras_resnet.models\n",
    "\n",
    "import keras_resnet.blocks\n",
    "\n",
    "import keras_rcnn.backend\n",
    "import keras_rcnn.classifiers\n",
    "import keras_rcnn.datasets.malaria\n",
    "import keras_rcnn.layers\n",
    "import keras_rcnn.losses.rcnn\n",
    "import keras_rcnn.layers.object_detection\n",
    "import keras_rcnn.preprocessing\n",
    "\n",
    "import sklearn.preprocessing\n",
    "import skimage.transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "configuration = tensorflow.ConfigProto()\n",
    "\n",
    "configuration.gpu_options.allow_growth = True\n",
    "\n",
    "configuration.gpu_options.visible_device_list = \"1\"\n",
    "\n",
    "session = tensorflow.Session(config=configuration)\n",
    "\n",
    "keras.backend.set_session(session)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "image = keras.layers.Input((224, 224, 1), name=\"image\")\n",
    "\n",
    "# Bounding boxes are the ground truth bounding boxes\n",
    "bounding_boxes = keras.layers.Input((None, 4), name=\"bounding_boxes\")\n",
    "\n",
    "labels = keras.layers.Input((None, 2), name=\"labels\")\n",
    "\n",
    "# Metadata is a 3-tuple that contains the feature width, feature height, and scale\n",
    "metadata = keras.layers.Input((3,), name=\"metadata\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convolutional Neural Network (CNN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "options = {\n",
    "    \"activation\": \"relu\",\n",
    "    \"kernel_size\": (3, 3),\n",
    "    \"padding\": \"same\"\n",
    "}\n",
    "\n",
    "features = keras.layers.Conv2D(64, **options, name=\"convolution_1_1\")(image)\n",
    "features = keras.layers.Conv2D(64, **options, name=\"convolution_1_2\")(features)\n",
    "\n",
    "features = keras.layers.MaxPooling2D(strides=(2, 2), name=\"max_pooling_1\")(features)\n",
    "\n",
    "features = keras.layers.Conv2D(128, **options, name=\"convolution_2_1\")(features)\n",
    "features = keras.layers.Conv2D(128, **options, name=\"convolution_2_2\")(features)\n",
    "\n",
    "features = keras.layers.MaxPooling2D(strides=(2, 2), name=\"max_pooling_2\")(features)\n",
    "\n",
    "features = keras.layers.Conv2D(256, **options, name=\"convolution_3_1\")(features)\n",
    "features = keras.layers.Conv2D(256, **options, name=\"convolution_3_2\")(features)\n",
    "features = keras.layers.Conv2D(256, **options, name=\"convolution_3_3\")(features)\n",
    "\n",
    "features = keras.layers.MaxPooling2D(strides=(2, 2), name=\"max_pooling_3\")(features)\n",
    "\n",
    "features = keras.layers.Conv2D(512, **options, name=\"convolution_4_1\")(features)\n",
    "features = keras.layers.Conv2D(512, **options, name=\"convolution_4_2\")(features)\n",
    "features = keras.layers.Conv2D(512, **options, name=\"convolution_4_3\")(features)\n",
    "\n",
    "features = keras.layers.MaxPooling2D(strides=(2, 2), name=\"max_pooling_4\")(features)\n",
    "\n",
    "features = keras.layers.Conv2D(512, **options, name=\"convolution_5_1\")(features)\n",
    "features = keras.layers.Conv2D(512, **options, name=\"convolution_5_2\")(features)\n",
    "features = keras.layers.Conv2D(512, **options, name=\"convolution_5_3\")(features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Region Proposal Network (RPN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "convolution_3x3 = keras.layers.Conv2D(512, **options, name=\"convolution_3x3\")(features)\n",
    "\n",
    "deltas = keras.layers.Conv2D(9 * 4, (1, 1), name=\"deltas\")(convolution_3x3)\n",
    "scores = keras.layers.Conv2D(9 * 2, (1, 1), name=\"scores\", activation=\"sigmoid\")(convolution_3x3)\n",
    "\n",
    "rpn_labels, bounding_box_targets = keras_rcnn.layers.AnchorTarget()([scores, bounding_boxes, image])\n",
    "\n",
    "deltas = keras_rcnn.layers.RegressionLoss(9)([deltas, bounding_box_targets, rpn_labels])\n",
    "scores = keras_rcnn.layers.ClassificationLoss(9)([scores, rpn_labels])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Region of Interest (ROI) Object Proposals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "proposals = keras_rcnn.layers.ObjectProposal()([metadata, deltas, scores])\n",
    "\n",
    "proposals, predicted_labels, bounding_box_targets = keras_rcnn.layers.ProposalTarget()([proposals, bounding_boxes, labels])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Region-based Convolutional Neural Network (RCNN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = keras_rcnn.layers.RegionOfInterest(32, (7, 7))([features, proposals])\n",
    "\n",
    "# y = keras.layers.TimeDistributed(keras.layers.Flatten())(y)\n",
    "\n",
    "# y = keras.layers.TimeDistributed(keras.layers.Dense(4096, activation=\"relu\"))(y)\n",
    "# y = keras.layers.TimeDistributed(keras.layers.Dense(4096, activation=\"relu\"))(y)\n",
    "\n",
    "# klass = keras.layers.TimeDistributed(keras.layers.Dense(2, activation=\"softmax\"))(y)\n",
    "# boxes = keras.layers.TimeDistributed(keras.layers.Dense(4, activation=\"linear\"))(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = keras.models.Model([image, bounding_boxes, metadata, labels], [y])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(\"adam\", [None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_image = numpy.random.choice(range(0, 255 + 1), (1, 224, 224, 1))\n",
    "x_boxes = numpy.random.choice(range(0, 224 + 1), (1, 4, 4))\n",
    "x_metadata = numpy.expand_dims([224, 224, 1], 0)\n",
    "x_labels = keras.utils.to_categorical(numpy.random.choice(range(0, 1 + 1), (1, 4)))\n",
    "x_labels = numpy.expand_dims(x_labels, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1/1 [==============================] - 0s - loss: 0.8675\n",
      "Epoch 2/10\n",
      "1/1 [==============================] - 0s - loss: 2.0415\n",
      "Epoch 3/10\n",
      "1/1 [==============================] - 0s - loss: 0.6288\n",
      "Epoch 4/10\n",
      "1/1 [==============================] - 0s - loss: 1.0856\n",
      "Epoch 5/10\n",
      "1/1 [==============================] - 0s - loss: 0.9948\n",
      "Epoch 6/10\n",
      "1/1 [==============================] - 0s - loss: 0.6870\n",
      "Epoch 7/10\n",
      "1/1 [==============================] - 0s - loss: 0.5891\n",
      "Epoch 8/10\n",
      "1/1 [==============================] - 0s - loss: 0.3866\n",
      "Epoch 9/10\n",
      "1/1 [==============================] - 0s - loss: 0.3893\n",
      "Epoch 10/10\n",
      "1/1 [==============================] - 0s - loss: 0.3609\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fb4f4305208>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit([x_image, x_boxes, x_metadata, x_labels], epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}