Evaluate ChainerCV Faster R-CNN performance on VOC 2007 test

eval-faster-rcnn.py

import pprint

import chainer

from chainercv.datasets import voc_detection_label_names
from chainercv.datasets import VOCDetectionDataset
from chainercv.extensions import DetectionVOCEvaluator
from chainercv.links import FasterRCNNVGG16

eval_data = VOCDetectionDataset(split='test', year='2007', use_difficult=True, return_difficult=True)
eval_iter = chainer.iterators.SerialIterator(eval_data, batch_size=1, repeat=False, shuffle=False)

pretrained_model_path = 'examples/faster_rcnn/result/snapshot_model.npz'
model = FasterRCNNVGG16(n_fg_class=len(voc_detection_label_names), pretrained_model=pretrained_model_path)

evaluator = DetectionVOCEvaluator(eval_iter, model, use_07_metric=True, label_names=voc_detection_label_names)

gpu_id = 0
model.to_gpu(gpu_id)
chainer.cuda.get_device(gpu_id).use()

chainer.config.train = False
model.use_preset('evaluate')

reporter = chainer.Reporter()
reporter.add_observer('target', model)
with reporter:
    m = evaluator.evaluate()

pprint.pprint(m)

Important points:

• chainer.config.train = False will cause less proposals to be generated, and kept after NMS (leading to faster inference); note that AP slighlty drops
• model.use_preset('evaluate') configures post-processing parameters for evaluation such as threshold for confidence score
• DetectionVOCEvaluator should be instantiated with use_07_metric=True (default is False), if evaluation is conducted on VOC 2007 test dataset
• VOCDetectionDataset should return information about difficulties of bounding boxes, as the evaluation metric expects that to be included; to also include the difficult bounding boxes in evaluation, set use_difficult=True and return_difficult=True