parking_spots_hunter.ipynb

detect_cars_in_video.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "parking_spots_hunter.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/x13-caesar/39cfc7c6ab4b5cd9f459151c9e33a052/detect_cars_in_video.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MEi42OkrZhlG",
        "colab_type": "code",
        "outputId": "3c772201-f2a4-4968-e04e-4a641eae8ff7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "import os\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CqDSKb3kbnIm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Set install dir where mcrnn package must be there\n",
        "# https://github.com/matterport/Mask_RCNN\n",
        "os.chdir(\"/content/drive/My Drive/Study @ Fordham/Deep Learning - Parking Spots Hunter/Mask_RCNN-master\")\n",
        "\n",
        "!pip3 install -r requirements.txt\n",
        "!python3 setup.py install\n",
        "!pip3 install twilio\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C2N3NolMZFNx",
        "colab_type": "code",
        "outputId": "e2d87246-210b-4387-a612-8b92e908bd1b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        }
      },
      "source": [
        "import numpy as np\n",
        "import cv2\n",
        "import mrcnn.config\n",
        "import mrcnn.utils\n",
        "from mrcnn.model import MaskRCNN\n",
        "from pathlib import Path\n",
        "from twilio.rest import Client\n",
        "from google.colab.patches import cv2_imshow"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_apkDG-EZDMM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Configuration that will be used by the Mask-RCNN library\n",
        "class MaskRCNNConfig(mrcnn.config.Config):\n",
        "    NAME = \"coco_pretrained_model_config\"\n",
        "    IMAGES_PER_GPU = 1\n",
        "    GPU_COUNT = 1\n",
        "    NUM_CLASSES = 1 + 80  # COCO dataset has 80 classes + one background class\n",
        "    DETECTION_MIN_CONFIDENCE = 0.6\n",
        "\n",
        "\n",
        "# Filter a list of Mask R-CNN detection results to get only the detected cars / trucks\n",
        "def get_car_boxes(boxes, class_ids):\n",
        "    car_boxes = []\n",
        "    for i, box in enumerate(boxes):\n",
        "        # If the detected object isn't a car / truck, skip it\n",
        "        if class_ids[i] in [3, 8, 6]:\n",
        "            car_boxes.append(box)\n",
        "    return np.array(car_boxes)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5YjNs2Du0oWE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Twilio config\n",
        "twilio_account_sid = ''\n",
        "twilio_auth_token = ''\n",
        "twilio_phone_number = '+12055510660'\n",
        "destination_phone_number = '+13472599593'\n",
        "client = Client(twilio_account_sid, twilio_auth_token)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3IYi6VpzcUGh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Root directory of the project\n",
        "ROOT_DIR = Path(\"/content/drive/My Drive/Study @ Fordham/Deep Learning - Parking Spots Hunter/\")\n",
        "\n",
        "# Directory to save logs and trained model\n",
        "MODEL_DIR = os.path.join(ROOT_DIR, \"logs\")\n",
        "\n",
        "# Local path to trained weights file\n",
        "COCO_MODEL_PATH = os.path.join(ROOT_DIR, \"mask_rcnn_coco.h5\")\n",
        "\n",
        "# Download COCO trained weights from Releases if needed\n",
        "if not os.path.exists(COCO_MODEL_PATH):\n",
        "    mrcnn.utils.download_trained_weights(COCO_MODEL_PATH)\n",
        "\n",
        "# Directory of images to run detection on\n",
        "IMAGE_DIR = os.path.join(ROOT_DIR, \"images\")\n",
        "\n",
        "# Video file or camera to process - set this to 0 to use your webcam instead of a video file\n",
        "VIDEO_SOURCE = \"/content/drive/My Drive/Study @ Fordham/Deep Learning - Parking Spots Hunter/test_images/Clip (October 22 2019 at 304 PM).mp4\"\n",
        "\n",
        "# Create a Mask-RCNN model in inference mode\n",
        "model = MaskRCNN(mode=\"inference\", model_dir=MODEL_DIR, config=MaskRCNNConfig())\n",
        "\n",
        "# Load pre-trained model\n",
        "model.load_weights(COCO_MODEL_PATH, by_name=True)\n",
        "\n",
        "# Location of parking spaces\n",
        "parked_car_boxes = None\n",
        "\n",
        "# Load the video file we want to run detection on\n",
        "video_capture = cv2.VideoCapture(VIDEO_SOURCE)\n",
        "\n",
        "\n",
        "# How many frames of video we've seen in a row with a parking space open\n",
        "free_space_frames = 0\n",
        "\n",
        "# Have we sent an SMS alert yet?\n",
        "sms_sent = False\n",
        "\n",
        "\n",
        "INTERVAL = 40  # How many frames we skip by one catching?\n",
        "i = 0  # Initialize the frame counter\n",
        "\n",
        "# Loop over each frame of video\n",
        "while video_capture.isOpened():\n",
        "    # catch the picture per interval frames\n",
        "    if i//INTERVAL != 0:\n",
        "        continue\n",
        "    else:\n",
        "        success, frame = video_capture.read()\n",
        "        if not success:\n",
        "            break\n",
        "\n",
        "        # Convert the image from BGR color (which OpenCV uses) to RGB color\n",
        "        rgb_image = frame[:, :, ::-1]\n",
        "\n",
        "        # Run the image through the Mask R-CNN model to get results.\n",
        "        results = model.detect([rgb_image], verbose=0)\n",
        "\n",
        "        # Mask R-CNN assumes we are running detection on multiple images.\n",
        "        # We only passed in one image to detect, so only grab the first result.\n",
        "        r = results[0]\n",
        "\n",
        "        # The r variable will now have the results of detection:\n",
        "        # - r['rois'] are the bounding box of each detected object\n",
        "        # - r['class_ids'] are the class id (type) of each detected object\n",
        "        # - r['scores'] are the confidence scores for each detection\n",
        "        # - r['masks'] are the object masks for each detected object (which gives you the object outline)\n",
        "\n",
        "        if parked_car_boxes is None:\n",
        "            # This is the first frame of video - assume all the cars detected are in parking spaces.\n",
        "            # Save the location of each car as a parking space box and go to the next frame of video.\n",
        "            parked_car_boxes = get_car_boxes(r['rois'], r['class_ids'])\n",
        "        else:\n",
        "            # We already know where the parking spaces are. Check if any are currently unoccupied.\n",
        "\n",
        "            # Get where cars are currently located in the frame\n",
        "            car_boxes = get_car_boxes(r['rois'], r['class_ids'])\n",
        "\n",
        "            # See how much those cars overlap with the known parking spaces\n",
        "            overlaps = mrcnn.utils.compute_overlaps(parked_car_boxes, car_boxes)\n",
        "\n",
        "            # Assume no spaces are free until we find one that is free\n",
        "            free_space = False\n",
        "\n",
        "            # Loop through each known parking space box\n",
        "            for parking_area, overlap_areas in zip(parked_car_boxes, overlaps):\n",
        "\n",
        "                # For this parking space, find the max amount it was covered by any\n",
        "                # car that was detected in our image (doesn't really matter which car)\n",
        "                max_IoU_overlap = np.max(overlap_areas)\n",
        "\n",
        "                # Get the top-left and bottom-right coordinates of the parking area\n",
        "                y1, x1, y2, x2 = parking_area\n",
        "\n",
        "                # Check if the parking space is occupied by seeing if any car overlaps\n",
        "                # it by more than 0.15 using IoU\n",
        "                if max_IoU_overlap < 0.15:\n",
        "                    # Parking space not occupied! Draw a green box around it\n",
        "                    cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 255, 0), 3)\n",
        "                    # Flag that we have seen at least one open space\n",
        "                    free_space = True\n",
        "                else:\n",
        "                    # Parking space is still occupied - draw a red box around it\n",
        "                    cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 0, 255), 1)\n",
        "\n",
        "                # Write the IoU measurement inside the box\n",
        "                font = cv2.FONT_HERSHEY_DUPLEX\n",
        "                cv2.putText(frame, f\"{max_IoU_overlap:0.2}\", (x1 + 6, y2 - 6), font, 0.3, (255, 255, 255))\n",
        "\n",
        "            # If at least one space was free, start counting frames\n",
        "            # This is so we don't alert based on one frame of a spot being open.\n",
        "            # This helps prevent the script triggered on one bad detection.\n",
        "            if free_space:\n",
        "                free_space_frames += 1\n",
        "            else:\n",
        "                # If no spots are free, reset the count\n",
        "                free_space_frames = 0\n",
        "\n",
        "            # If a space has been free for several frames, we are pretty sure it is really free!\n",
        "            if free_space_frames > 20:\n",
        "                # Write SPACE AVAILABLE!! at the top of the screen\n",
        "                font = cv2.FONT_HERSHEY_DUPLEX\n",
        "                cv2.putText(frame, f\"SPACE AVAILABLE!\", (10, 150), font, 3.0, (0, 255, 0), 2, cv2.FILLED)\n",
        "\n",
        "                # If we haven't sent an SMS yet, sent it!\n",
        "                if not sms_sent:\n",
        "                    print(\"SENDING SMS!!!\")\n",
        "                    message = client.messages.create(\n",
        "                        body=\"Parking space open - go go go!\",\n",
        "                        from_=twilio_phone_number,\n",
        "                        to=destination_phone_number\n",
        "                    )\n",
        "                    sms_sent = True\n",
        "\n",
        "            # Show the frame of video on the screen\n",
        "            cv2_imshow(frame)\n",
        "\n",
        "    i += 1 # add up the frame counter\n",
        "\n",
        "\n",
        "    # Hit 'q' to quit\n",
        "    if cv2.waitKey(1) & 0xFF == ord('q'):\n",
        "        break\n",
        "\n",
        "# Clean up everything when finished\n",
        "video_capture.release()\n",
        "cv2.destroyAllWindows()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}