demo_camera.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mask R-CNN Demo\n",
    "\n",
    "A quick intro to using the pre-trained model to detect and segment objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/michael/tensorflow/envs/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import random\n",
    "import math\n",
    "import numpy as np\n",
    "import skimage.io\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import coco\n",
    "import utils\n",
    "import model as modellib\n",
    "# import visualize\n",
    "\n",
    "%matplotlib inline \n",
    "\n",
    "# Root directory of the project\n",
    "ROOT_DIR = os.getcwd()\n",
    "\n",
    "# Directory to save logs and trained model\n",
    "MODEL_DIR = os.path.join(ROOT_DIR, \"logs\")\n",
    "\n",
    "# Local path to trained weights file\n",
    "COCO_MODEL_PATH = os.path.join(ROOT_DIR, \"mask_rcnn_coco.h5\")\n",
    "# Download COCO trained weights from Releases if needed\n",
    "if not os.path.exists(COCO_MODEL_PATH):\n",
    "    utils.download_trained_weights(COCO_MODEL_PATH)\n",
    "\n",
    "# Directory of images to run detection on\n",
    "IMAGE_DIR = os.path.join(ROOT_DIR, \"images\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configurations\n",
    "\n",
    "We'll be using a model trained on the MS-COCO dataset. The configurations of this model are in the ```CocoConfig``` class in ```coco.py```.\n",
    "\n",
    "For inferencing, modify the configurations a bit to fit the task. To do so, sub-class the ```CocoConfig``` class and override the attributes you need to change."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Configurations:\n",
      "BACKBONE_SHAPES                [[256 256]\n",
      " [128 128]\n",
      " [ 64  64]\n",
      " [ 32  32]\n",
      " [ 16  16]]\n",
      "BACKBONE_STRIDES               [4, 8, 16, 32, 64]\n",
      "BATCH_SIZE                     1\n",
      "BBOX_STD_DEV                   [0.1 0.1 0.2 0.2]\n",
      "DETECTION_MAX_INSTANCES        100\n",
      "DETECTION_MIN_CONFIDENCE       0.7\n",
      "DETECTION_NMS_THRESHOLD        0.3\n",
      "GPU_COUNT                      1\n",
      "IMAGES_PER_GPU                 1\n",
      "IMAGE_MAX_DIM                  1024\n",
      "IMAGE_MIN_DIM                  800\n",
      "IMAGE_PADDING                  True\n",
      "IMAGE_SHAPE                    [1024 1024    3]\n",
      "LEARNING_MOMENTUM              0.9\n",
      "LEARNING_RATE                  0.001\n",
      "MASK_POOL_SIZE                 14\n",
      "MASK_SHAPE                     [28, 28]\n",
      "MAX_GT_INSTANCES               100\n",
      "MEAN_PIXEL                     [123.7 116.8 103.9]\n",
      "MINI_MASK_SHAPE                (56, 56)\n",
      "NAME                           coco\n",
      "NUM_CLASSES                    81\n",
      "POOL_SIZE                      7\n",
      "POST_NMS_ROIS_INFERENCE        1000\n",
      "POST_NMS_ROIS_TRAINING         2000\n",
      "ROI_POSITIVE_RATIO             0.33\n",
      "RPN_ANCHOR_RATIOS              [0.5, 1, 2]\n",
      "RPN_ANCHOR_SCALES              (32, 64, 128, 256, 512)\n",
      "RPN_ANCHOR_STRIDE              1\n",
      "RPN_BBOX_STD_DEV               [0.1 0.1 0.2 0.2]\n",
      "RPN_NMS_THRESHOLD              0.7\n",
      "RPN_TRAIN_ANCHORS_PER_IMAGE    256\n",
      "STEPS_PER_EPOCH                1000\n",
      "TRAIN_ROIS_PER_IMAGE           200\n",
      "USE_MINI_MASK                  True\n",
      "USE_RPN_ROIS                   True\n",
      "VALIDATION_STEPS               50\n",
      "WEIGHT_DECAY                   0.0001\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "class InferenceConfig(coco.CocoConfig):\n",
    "    # Set batch size to 1 since we'll be running inference on\n",
    "    # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU\n",
    "    GPU_COUNT = 1\n",
    "    IMAGES_PER_GPU = 1\n",
    "\n",
    "config = InferenceConfig()\n",
    "config.display()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Model and Load Trained Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Create model object in inference mode.\n",
    "model = modellib.MaskRCNN(mode=\"inference\", model_dir=MODEL_DIR, config=config)\n",
    "\n",
    "# Load weights trained on MS-COCO\n",
    "model.load_weights(COCO_MODEL_PATH, by_name=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Class Names\n",
    "\n",
    "The model classifies objects and returns class IDs, which are integer value that identify each class. Some datasets assign integer values to their classes and some don't. For example, in the MS-COCO dataset, the 'person' class is 1 and 'teddy bear' is 88. The IDs are often sequential, but not always. The COCO dataset, for example, has classes associated with class IDs 70 and 72, but not 71.\n",
    "\n",
    "To improve consistency, and to support training on data from multiple sources at the same time, our ```Dataset``` class assigns it's own sequential integer IDs to each class. For example, if you load the COCO dataset using our ```Dataset``` class, the 'person' class would get class ID = 1 (just like COCO) and the 'teddy bear' class is 78 (different from COCO). Keep that in mind when mapping class IDs to class names.\n",
    "\n",
    "To get the list of class names, you'd load the dataset and then use the ```class_names``` property like this.\n",
    "```\n",
    "# Load COCO dataset\n",
    "dataset = coco.CocoDataset()\n",
    "dataset.load_coco(COCO_DIR, \"train\")\n",
    "dataset.prepare()\n",
    "\n",
    "# Print class names\n",
    "print(dataset.class_names)\n",
    "```\n",
    "\n",
    "We don't want to require you to download the COCO dataset just to run this demo, so we're including the list of class names below. The index of the class name in the list represent its ID (first class is 0, second is 1, third is 2, ...etc.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# COCO Class names\n",
    "# Index of the class in the list is its ID. For example, to get ID of\n",
    "# the teddy bear class, use: class_names.index('teddy bear')\n",
    "class_names = ['BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane',\n",
    "               'bus', 'train', 'truck', 'boat', 'traffic light',\n",
    "               'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird',\n",
    "               'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear',\n",
    "               'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie',\n",
    "               'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball',\n",
    "               'kite', 'baseball bat', 'baseball glove', 'skateboard',\n",
    "               'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup',\n",
    "               'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',\n",
    "               'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza',\n",
    "               'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed',\n",
    "               'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote',\n",
    "               'keyboard', 'cell phone', 'microwave', 'oven', 'toaster',\n",
    "               'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors',\n",
    "               'teddy bear', 'hair drier', 'toothbrush']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Mask R-CNN\n",
    "Display and Visualization Functions.\n",
    "\n",
    "Copyright (c) 2017 Matterport, Inc.\n",
    "Licensed under the MIT License (see LICENSE for details)\n",
    "Written by Waleed Abdulla\n",
    "\"\"\"\n",
    "\n",
    "import random\n",
    "import itertools\n",
    "import colorsys\n",
    "import numpy as np\n",
    "from skimage.measure import find_contours\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "import matplotlib.lines as lines\n",
    "from matplotlib.patches import Polygon\n",
    "import IPython.display\n",
    "import cv2\n",
    "\n",
    "import utils\n",
    "\n",
    "\n",
    "############################################################\n",
    "#  Visualization\n",
    "############################################################\n",
    "\n",
    "def display_images(images, titles=None, cols=4, cmap=None, norm=None,\n",
    "                   interpolation=None):\n",
    "    \"\"\"Display the given set of images, optionally with titles.\n",
    "    images: list or array of image tensors in HWC format.\n",
    "    titles: optional. A list of titles to display with each image.\n",
    "    cols: number of images per row\n",
    "    cmap: Optional. Color map to use. For example, \"Blues\".\n",
    "    norm: Optional. A Normalize instance to map values to colors.\n",
    "    interpolation: Optional. Image interporlation to use for display.\n",
    "    \"\"\"\n",
    "    titles = titles if titles is not None else [\"\"] * len(images)\n",
    "    rows = len(images) // cols + 1\n",
    "    plt.figure(figsize=(14, 14 * rows // cols))\n",
    "    i = 1\n",
    "    for image, title in zip(images, titles):\n",
    "        plt.subplot(rows, cols, i)\n",
    "        plt.title(title, fontsize=9)\n",
    "        plt.axis('off')\n",
    "        plt.imshow(image.astype(np.uint8), cmap=cmap,\n",
    "                   norm=norm, interpolation=interpolation)\n",
    "        i += 1\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def random_colors(N, bright=True):\n",
    "    \"\"\"\n",
    "    Generate random colors.\n",
    "    To get visually distinct colors, generate them in HSV space then\n",
    "    convert to RGB.\n",
    "    \"\"\"\n",
    "    brightness = 1.0 if bright else 0.7\n",
    "    hsv = [(i / N, 1, brightness) for i in range(N)]\n",
    "    colors = list(map(lambda c: colorsys.hsv_to_rgb(*c), hsv))\n",
    "    random.shuffle(colors)\n",
    "    return colors\n",
    "\n",
    "\n",
    "def apply_mask(image, mask, color, alpha=0.5):\n",
    "    \"\"\"Apply the given mask to the image.\n",
    "    \"\"\"\n",
    "    for c in range(3):\n",
    "        image[:, :, c] = np.where(mask == 1,\n",
    "                                  image[:, :, c] *\n",
    "                                  (1 - alpha) + alpha * color[c] * 255,\n",
    "                                  image[:, :, c])\n",
    "    return image\n",
    "\n",
    "\n",
    "def display_instances(image, boxes, masks, class_ids, class_names,\n",
    "                      scores=None, title=\"\",\n",
    "                      figsize=(16, 16), ax=None, show=True):\n",
    "    \"\"\"\n",
    "    boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates.\n",
    "    masks: [height, width, num_instances]\n",
    "    class_ids: [num_instances]\n",
    "    class_names: list of class names of the dataset\n",
    "    scores: (optional) confidence scores for each box\n",
    "    figsize: (optional) the size of the image.\n",
    "    \"\"\"\n",
    "    # Number of instances\n",
    "    N = boxes.shape[0]\n",
    "    if not N:\n",
    "        print(\"\\n*** No instances to display *** \\n\")\n",
    "    else:\n",
    "        assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]\n",
    "\n",
    "    if not ax:\n",
    "        _, ax = plt.subplots(1, figsize=figsize)\n",
    "\n",
    "    # Generate random colors\n",
    "    colors = random_colors(N)\n",
    "\n",
    "    # Show area outside image boundaries.\n",
    "    height, width = image.shape[:2]\n",
    "    ax.set_ylim(height + 10, -10)\n",
    "    ax.set_xlim(-10, width + 10)\n",
    "    ax.axis('off')\n",
    "    ax.set_title(title)\n",
    "\n",
    "    masked_image = image.astype(np.uint32).copy()\n",
    "    for i in range(N):\n",
    "        color = colors[i]\n",
    "\n",
    "        # Bounding box\n",
    "        if not np.any(boxes[i]):\n",
    "            # Skip this instance. Has no bbox. Likely lost in image cropping.\n",
    "            continue\n",
    "        y1, x1, y2, x2 = boxes[i]\n",
    "        p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,\n",
    "                              alpha=0.7, linestyle=\"dashed\",\n",
    "                              edgecolor=color, facecolor='none')\n",
    "        ax.add_patch(p)\n",
    "\n",
    "        # Label\n",
    "        class_id = class_ids[i]\n",
    "        score = scores[i] if scores is not None else None\n",
    "        label = class_names[class_id]\n",
    "        x = random.randint(x1, (x1 + x2) // 2)\n",
    "        caption = \"{} {:.3f}\".format(label, score) if score else label\n",
    "        ax.text(x1, y1 + 8, caption,\n",
    "                color='w', size=11, backgroundcolor=\"none\")\n",
    "\n",
    "        # Mask\n",
    "        mask = masks[:, :, i]\n",
    "        masked_image = apply_mask(masked_image, mask, color)\n",
    "\n",
    "        # Mask Polygon\n",
    "        # Pad to ensure proper polygons for masks that touch image edges.\n",
    "        padded_mask = np.zeros(\n",
    "            (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8)\n",
    "        padded_mask[1:-1, 1:-1] = mask\n",
    "        contours = find_contours(padded_mask, 0.5)\n",
    "        for verts in contours:\n",
    "            # Subtract the padding and flip (y, x) to (x, y)\n",
    "            verts = np.fliplr(verts) - 1\n",
    "            p = Polygon(verts, facecolor=\"none\", edgecolor=color)\n",
    "            ax.add_patch(p)\n",
    "    if show:\n",
    "        ax.imshow(masked_image.astype(np.uint8))\n",
    "        plt.show()\n",
    "\n",
    "    return masked_image.astype(np.uint8)\n",
    "\n",
    "\n",
    "def draw_rois(image, rois, refined_rois, mask, class_ids, class_names, limit=10):\n",
    "    \"\"\"\n",
    "    anchors: [n, (y1, x1, y2, x2)] list of anchors in image coordinates.\n",
    "    proposals: [n, 4] the same anchors but refined to fit objects better.\n",
    "    \"\"\"\n",
    "    masked_image = image.copy()\n",
    "\n",
    "    # Pick random anchors in case there are too many.\n",
    "    ids = np.arange(rois.shape[0], dtype=np.int32)\n",
    "    ids = np.random.choice(\n",
    "        ids, limit, replace=False) if ids.shape[0] > limit else ids\n",
    "\n",
    "    fig, ax = plt.subplots(1, figsize=(12, 12))\n",
    "    if rois.shape[0] > limit:\n",
    "        plt.title(\"Showing {} random ROIs out of {}\".format(\n",
    "            len(ids), rois.shape[0]))\n",
    "    else:\n",
    "        plt.title(\"{} ROIs\".format(len(ids)))\n",
    "\n",
    "    # Show area outside image boundaries.\n",
    "    ax.set_ylim(image.shape[0] + 20, -20)\n",
    "    ax.set_xlim(-50, image.shape[1] + 20)\n",
    "    ax.axis('off')\n",
    "\n",
    "    for i, id in enumerate(ids):\n",
    "        color = np.random.rand(3)\n",
    "        class_id = class_ids[id]\n",
    "        # ROI\n",
    "        y1, x1, y2, x2 = rois[id]\n",
    "        p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,\n",
    "                              edgecolor=color if class_id else \"gray\",\n",
    "                              facecolor='none', linestyle=\"dashed\")\n",
    "        ax.add_patch(p)\n",
    "        # Refined ROI\n",
    "        if class_id:\n",
    "            ry1, rx1, ry2, rx2 = refined_rois[id]\n",
    "            p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2,\n",
    "                                  edgecolor=color, facecolor='none')\n",
    "            ax.add_patch(p)\n",
    "            # Connect the top-left corners of the anchor and proposal for easy visualization\n",
    "            ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))\n",
    "\n",
    "            # Label\n",
    "            label = class_names[class_id]\n",
    "            ax.text(rx1, ry1 + 8, \"{}\".format(label),\n",
    "                    color='w', size=11, backgroundcolor=\"none\")\n",
    "\n",
    "            # Mask\n",
    "            m = utils.unmold_mask(mask[id], rois[id]\n",
    "                                  [:4].astype(np.int32), image.shape)\n",
    "            masked_image = apply_mask(masked_image, m, color)\n",
    "\n",
    "    ax.imshow(masked_image)\n",
    "\n",
    "    # Print stats\n",
    "    print(\"Positive ROIs: \", class_ids[class_ids > 0].shape[0])\n",
    "    print(\"Negative ROIs: \", class_ids[class_ids == 0].shape[0])\n",
    "    print(\"Positive Ratio: {:.2f}\".format(\n",
    "        class_ids[class_ids > 0].shape[0] / class_ids.shape[0]))\n",
    "\n",
    "\n",
    "# TODO: Replace with matplotlib equivalent?\n",
    "def draw_box(image, box, color):\n",
    "    \"\"\"Draw 3-pixel width bounding boxes on the given image array.\n",
    "    color: list of 3 int values for RGB.\n",
    "    \"\"\"\n",
    "    y1, x1, y2, x2 = box\n",
    "    image[y1:y1 + 2, x1:x2] = color\n",
    "    image[y2:y2 + 2, x1:x2] = color\n",
    "    image[y1:y2, x1:x1 + 2] = color\n",
    "    image[y1:y2, x2:x2 + 2] = color\n",
    "    return image\n",
    "\n",
    "\n",
    "def display_top_masks(image, mask, class_ids, class_names, limit=4):\n",
    "    \"\"\"Display the given image and the top few class masks.\"\"\"\n",
    "    to_display = []\n",
    "    titles = []\n",
    "    to_display.append(image)\n",
    "    titles.append(\"H x W={}x{}\".format(image.shape[0], image.shape[1]))\n",
    "    # Pick top prominent classes in this image\n",
    "    unique_class_ids = np.unique(class_ids)\n",
    "    mask_area = [np.sum(mask[:, :, np.where(class_ids == i)[0]])\n",
    "                 for i in unique_class_ids]\n",
    "    top_ids = [v[0] for v in sorted(zip(unique_class_ids, mask_area),\n",
    "                                    key=lambda r: r[1], reverse=True) if v[1] > 0]\n",
    "    # Generate images and titles\n",
    "    for i in range(limit):\n",
    "        class_id = top_ids[i] if i < len(top_ids) else -1\n",
    "        # Pull masks of instances belonging to the same class.\n",
    "        m = mask[:, :, np.where(class_ids == class_id)[0]]\n",
    "        m = np.sum(m * np.arange(1, m.shape[-1] + 1), -1)\n",
    "        to_display.append(m)\n",
    "        titles.append(class_names[class_id] if class_id != -1 else \"-\")\n",
    "    display_images(to_display, titles=titles, cols=limit + 1, cmap=\"Blues_r\")\n",
    "\n",
    "\n",
    "def plot_precision_recall(AP, precisions, recalls):\n",
    "    \"\"\"Draw the precision-recall curve.\n",
    "\n",
    "    AP: Average precision at IoU >= 0.5\n",
    "    precisions: list of precision values\n",
    "    recalls: list of recall values\n",
    "    \"\"\"\n",
    "    # Plot the Precision-Recall curve\n",
    "    _, ax = plt.subplots(1)\n",
    "    ax.set_title(\"Precision-Recall Curve. AP@50 = {:.3f}\".format(AP))\n",
    "    ax.set_ylim(0, 1.1)\n",
    "    ax.set_xlim(0, 1.1)\n",
    "    _ = ax.plot(recalls, precisions)\n",
    "\n",
    "\n",
    "def plot_overlaps(gt_class_ids, pred_class_ids, pred_scores,\n",
    "                  overlaps, class_names, threshold=0.5):\n",
    "    \"\"\"Draw a grid showing how ground truth objects are classified.\n",
    "    gt_class_ids: [N] int. Ground truth class IDs\n",
    "    pred_class_id: [N] int. Predicted class IDs\n",
    "    pred_scores: [N] float. The probability scores of predicted classes\n",
    "    overlaps: [pred_boxes, gt_boxes] IoU overlaps of predictins and GT boxes.\n",
    "    class_names: list of all class names in the dataset\n",
    "    threshold: Float. The prediction probability required to predict a class\n",
    "    \"\"\"\n",
    "    gt_class_ids = gt_class_ids[gt_class_ids != 0]\n",
    "    pred_class_ids = pred_class_ids[pred_class_ids != 0]\n",
    "\n",
    "    plt.figure(figsize=(12, 10))\n",
    "    plt.imshow(overlaps, interpolation='nearest', cmap=plt.cm.Blues)\n",
    "    plt.yticks(np.arange(len(pred_class_ids)),\n",
    "               [\"{} ({:.2f})\".format(class_names[int(id)], pred_scores[i])\n",
    "                for i, id in enumerate(pred_class_ids)])\n",
    "    plt.xticks(np.arange(len(gt_class_ids)),\n",
    "               [class_names[int(id)] for id in gt_class_ids], rotation=90)\n",
    "\n",
    "    thresh = overlaps.max() / 2.\n",
    "    for i, j in itertools.product(range(overlaps.shape[0]),\n",
    "                                  range(overlaps.shape[1])):\n",
    "        text = \"\"\n",
    "        if overlaps[i, j] > threshold:\n",
    "            text = \"match\" if gt_class_ids[j] == pred_class_ids[i] else \"wrong\"\n",
    "        color = (\"white\" if overlaps[i, j] > thresh\n",
    "                 else \"black\" if overlaps[i, j] > 0\n",
    "                 else \"grey\")\n",
    "        plt.text(j, i, \"{:.3f}\\n{}\".format(overlaps[i, j], text),\n",
    "                 horizontalalignment=\"center\", verticalalignment=\"center\",\n",
    "                 fontsize=9, color=color)\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.xlabel(\"Ground Truth\")\n",
    "    plt.ylabel(\"Predictions\")\n",
    "\n",
    "\n",
    "def draw_boxes(image, boxes=None, refined_boxes=None,\n",
    "               masks=None, captions=None, visibilities=None,\n",
    "               title=\"\", ax=None):\n",
    "    \"\"\"Draw bounding boxes and segmentation masks with differnt\n",
    "    customizations.\n",
    "\n",
    "    boxes: [N, (y1, x1, y2, x2, class_id)] in image coordinates.\n",
    "    refined_boxes: Like boxes, but draw with solid lines to show\n",
    "        that they're the result of refining 'boxes'.\n",
    "    masks: [N, height, width]\n",
    "    captions: List of N titles to display on each box\n",
    "    visibilities: (optional) List of values of 0, 1, or 2. Determine how\n",
    "        prominant each bounding box should be.\n",
    "    title: An optional title to show over the image\n",
    "    ax: (optional) Matplotlib axis to draw on.\n",
    "    \"\"\"\n",
    "    # Number of boxes\n",
    "    assert boxes is not None or refined_boxes is not None\n",
    "    N = boxes.shape[0] if boxes is not None else refined_boxes.shape[0]\n",
    "\n",
    "    # Matplotlib Axis\n",
    "    if not ax:\n",
    "        _, ax = plt.subplots(1, figsize=(12, 12))\n",
    "\n",
    "    # Generate random colors\n",
    "    colors = random_colors(N)\n",
    "\n",
    "    # Show area outside image boundaries.\n",
    "    margin = image.shape[0] // 10\n",
    "    ax.set_ylim(image.shape[0] + margin, -margin)\n",
    "    ax.set_xlim(-margin, image.shape[1] + margin)\n",
    "    ax.axis('off')\n",
    "\n",
    "    ax.set_title(title)\n",
    "\n",
    "    masked_image = image.astype(np.uint32).copy()\n",
    "    for i in range(N):\n",
    "        # Box visibility\n",
    "        visibility = visibilities[i] if visibilities is not None else 1\n",
    "        if visibility == 0:\n",
    "            color = \"gray\"\n",
    "            style = \"dotted\"\n",
    "            alpha = 0.5\n",
    "        elif visibility == 1:\n",
    "            color = colors[i]\n",
    "            style = \"dotted\"\n",
    "            alpha = 1\n",
    "        elif visibility == 2:\n",
    "            color = colors[i]\n",
    "            style = \"solid\"\n",
    "            alpha = 1\n",
    "\n",
    "        # Boxes\n",
    "        if boxes is not None:\n",
    "            if not np.any(boxes[i]):\n",
    "                # Skip this instance. Has no bbox. Likely lost in cropping.\n",
    "                continue\n",
    "            y1, x1, y2, x2 = boxes[i]\n",
    "            p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,\n",
    "                                  alpha=alpha, linestyle=style,\n",
    "                                  edgecolor=color, facecolor='none')\n",
    "            ax.add_patch(p)\n",
    "\n",
    "        # Refined boxes\n",
    "        if refined_boxes is not None and visibility > 0:\n",
    "            ry1, rx1, ry2, rx2 = refined_boxes[i].astype(np.int32)\n",
    "            p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2,\n",
    "                                  edgecolor=color, facecolor='none')\n",
    "            ax.add_patch(p)\n",
    "            # Connect the top-left corners of the anchor and proposal\n",
    "            if boxes is not None:\n",
    "                ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))\n",
    "\n",
    "        # Captions\n",
    "        if captions is not None:\n",
    "            caption = captions[i]\n",
    "            # If there are refined boxes, display captions on them\n",
    "            if refined_boxes is not None:\n",
    "                y1, x1, y2, x2 = ry1, rx1, ry2, rx2\n",
    "            x = random.randint(x1, (x1 + x2) // 2)\n",
    "            ax.text(x1, y1, caption, size=11, verticalalignment='top',\n",
    "                    color='w', backgroundcolor=\"none\",\n",
    "                    bbox={'facecolor': color, 'alpha': 0.5,\n",
    "                          'pad': 2, 'edgecolor': 'none'})\n",
    "\n",
    "        # Masks\n",
    "        if masks is not None:\n",
    "            mask = masks[:, :, i]\n",
    "            masked_image = apply_mask(masked_image, mask, color)\n",
    "            # Mask Polygon\n",
    "            # Pad to ensure proper polygons for masks that touch image edges.\n",
    "            padded_mask = np.zeros(\n",
    "                (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8)\n",
    "            padded_mask[1:-1, 1:-1] = mask\n",
    "            contours = find_contours(padded_mask, 0.5)\n",
    "            for verts in contours:\n",
    "                # Subtract the padding and flip (y, x) to (x, y)\n",
    "                verts = np.fliplr(verts) - 1\n",
    "                p = Polygon(verts, facecolor=\"none\", edgecolor=color)\n",
    "                ax.add_patch(p)\n",
    "    ax.imshow(masked_image.astype(np.uint8))\n",
    "\n",
    "\n",
    "def display_table(table):\n",
    "    \"\"\"Display values in a table format.\n",
    "    table: an iterable of rows, and each row is an iterable of values.\n",
    "    \"\"\"\n",
    "    html = \"\"\n",
    "    for row in table:\n",
    "        row_html = \"\"\n",
    "        for col in row:\n",
    "            row_html += \"<td>{:40}</td>\".format(str(col))\n",
    "        html += \"<tr>\" + row_html + \"</tr>\"\n",
    "    html = \"<table>\" + html + \"</table>\"\n",
    "    IPython.display.display(IPython.display.HTML(html))\n",
    "\n",
    "\n",
    "def display_weight_stats(model):\n",
    "    \"\"\"Scans all the weights in the model and returns a list of tuples\n",
    "    that contain stats about each weight.\n",
    "    \"\"\"\n",
    "    layers = model.get_trainable_layers()\n",
    "    table = [[\"WEIGHT NAME\", \"SHAPE\", \"MIN\", \"MAX\", \"STD\"]]\n",
    "    for l in layers:\n",
    "        weight_values = l.get_weights()  # list of Numpy arrays\n",
    "        weight_tensors = l.weights  # list of TF tensors\n",
    "        for i, w in enumerate(weight_values):\n",
    "            weight_name = weight_tensors[i].name\n",
    "            # Detect problematic layers. Exclude biases of conv layers.\n",
    "            alert = \"\"\n",
    "            if w.min() == w.max() and not (l.__class__.__name__ == \"Conv2D\" and i == 1):\n",
    "                alert += \"<span style='color:red'>*** dead?</span>\"\n",
    "            if np.abs(w.min()) > 1000 or np.abs(w.max()) > 1000:\n",
    "                alert += \"<span style='color:red'>*** Overflow?</span>\"\n",
    "            # Add row\n",
    "            table.append([\n",
    "                weight_name + alert,\n",
    "                str(w.shape),\n",
    "                \"{:+9.4f}\".format(w.min()),\n",
    "                \"{:+10.4f}\".format(w.max()),\n",
    "                \"{:+9.4f}\".format(w.std()),\n",
    "            ])\n",
    "    display_table(table)\n",
    "    \n",
    "\n",
    "def get_masked_fixed_color(image, boxes, masks, class_ids, class_names,\n",
    "                      colors = None, scores=None, title=\"\",\n",
    "                      figsize=(16, 16), ax=None, show=True):\n",
    "    \"\"\"\n",
    "    boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates.\n",
    "    masks: [height, width, num_instances]\n",
    "    class_ids: [num_instances]\n",
    "    class_names: list of class names of the dataset\n",
    "    scores: (optional) confidence scores for each box\n",
    "    figsize: (optional) the size of the image.\n",
    "    \"\"\"\n",
    "    # Number of instances\n",
    "    N = boxes.shape[0]\n",
    "    if not N:\n",
    "        print(\"\\n*** No instances to display *** \\n\")\n",
    "    else:\n",
    "        assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]\n",
    "\n",
    "    # Generate random colors\n",
    "    if colors == None:\n",
    "        classN = len(class_names)\n",
    "        colors = random_colors(classN)\n",
    "\n",
    "    # masked_image = image.astype(np.uint32).copy()\n",
    "    masked_image = np.array(image)\n",
    "\n",
    "    for i in range(N):\n",
    "        color = colors[class_ids[i]]\n",
    "\n",
    "        # Bounding box\n",
    "        if not np.any(boxes[i]):\n",
    "            # Skip this instance. Has no bbox. Likely lost in image cropping.\n",
    "            continue\n",
    "        y1, x1, y2, x2 = boxes[i]\n",
    "        cv2.rectangle(masked_image, (x1, y1), (x2, y2), color, thickness = 2)\n",
    "\n",
    "        # Label\n",
    "        class_id = class_ids[i]\n",
    "        score = scores[i] if scores is not None else None\n",
    "        label = class_names[class_id]\n",
    "        x = random.randint(x1, (x1 + x2) // 2)\n",
    "        caption = \"{} {:.3f}\".format(label, score) if score else label\n",
    "        cv2.putText(masked_image, caption, (x1 + 5, y1 + 16), cv2.FONT_HERSHEY_SIMPLEX,\n",
    "            0.5, color)\n",
    "\n",
    "        # Mask\n",
    "        mask = masks[:, :, i]\n",
    "        masked_image = apply_mask(masked_image, mask, color)\n",
    "\n",
    "        # Mask Polygon\n",
    "        # Pad to ensure proper polygons for masks that touch image edges.\n",
    "        padded_mask = np.zeros(\n",
    "            (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8)\n",
    "        padded_mask[1:-1, 1:-1] = mask\n",
    "        contours = find_contours(padded_mask, 0.5)\n",
    "        for verts in contours:\n",
    "            # Subtract the padding and flip (y, x) to (x, y)\n",
    "            verts = np.fliplr(verts) - 1\n",
    "            verts = verts.reshape((-1, 1, 2)).astype(np.int32)\n",
    "            # Draw an edge on object contour\n",
    "            cv2.polylines(masked_image, verts, True, color)\n",
    "\n",
    "\n",
    "    return masked_image\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Object Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(images) must be equal to BATCH_SIZE\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import time\n",
    "\n",
    "colors = random_colors(len(class_names))\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "\n",
    "try:\n",
    "\n",
    "    while (True):\n",
    "\n",
    "        \n",
    "        _, image1 = cap.read()\n",
    "        image1 = cv2.resize(image1, None, fx=0.5, fy=0.5)\n",
    "\n",
    "        image_batch = [image1]\n",
    "        \n",
    "        # Run detection\n",
    "        t = time.time()\n",
    "        results = model.detect(image_batch, verbose=0)\n",
    "        t = t - time.time()\n",
    "        print (t)\n",
    "\n",
    "        masked_image_batch = []\n",
    "        # Visualize results\n",
    "        r = results[0]\n",
    "        t = time.time()\n",
    "\n",
    "        for i in range(len(results)):\n",
    "            r = results[i]\n",
    "            im = image_batch[i]\n",
    "            masked_image = get_masked_fixed_color(im, r['rois'], r['masks'], r['class_ids'], \n",
    "                                        class_names, colors, r['scores'], show=False)\n",
    "            masked_image = cv2.resize(masked_image, None, fx=3, fy=3)\n",
    "            masked_image_batch.append(masked_image)\n",
    "        t = t - time.time()\n",
    "        print (t)\n",
    "\n",
    "#         for mi in masked_image_batch:\n",
    "        cv2.imshow('Masked image', masked_image_batch[0])\n",
    "        if cv2.waitKey(1) & 0xFF == ord('q'):\n",
    "            break\n",
    "\n",
    "            \n",
    "except Exception as e:\n",
    "    print (e)\n",
    "\n",
    "finally:\n",
    "    # When everything done, release the capture\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}