eval_utils.py

import os
import sys
import h5py
import numpy as np
from progressbar import ProgressBar
from commons import check_mkdir

def load_gt_h5(fn):
    """ Output: pts             B x N x 3   float32
                gt_mask         B x K x N   bool
                gt_mask_label   B x K       uint8
                gt_mask_valid   B x K       bool
                gt_mask_other   B x N       bool
        All the ground-truth masks are represented as 0/1 mask over the 10k point cloud.
        All the ground-truth masks are disjoint, complete and corresponding to unique semantic labels.
        Different test shapes have different numbers of ground-truth masks, they are at the top in array gt_mask indicated by gt_valid.
    """
    with h5py.File(fn, 'r') as fin:
        gt_mask = fin['gt_mask'][:]
        gt_mask_label = fin['gt_mask_label'][:]
        gt_mask_valid = fin['gt_mask_valid'][:]
        gt_mask_other = fin['gt_mask_other'][:]
        return gt_mask, gt_mask_label, gt_mask_valid, gt_mask_other

def load_pred_h5(fn):
    """ Output: mask    B x K x N   bool
                label   B x K       uint8
                valid   B x K       bool
                conf    B x K       float32
        We only evaluate on the part predictions with valid = True.
        We assume no pre-sorting according to confidence score. 
    """
    with h5py.File(fn, 'r') as fin:
        mask = fin['mask'][:]
        label = fin['label'][:]
        valid = fin['valid'][:]
        conf = fin['conf'][:]
        return mask, label, valid, conf

def compute_ap(tp, fp, gt_npos, n_bins=100, plot_fn=None):
    assert len(tp) == len(fp), 'ERROR: the length of true_pos and false_pos is not the same!'

    tp = np.cumsum(tp)
    fp = np.cumsum(fp)

    rec = tp / gt_npos
    prec = tp / (fp + tp)

    rec = np.insert(rec, 0, 0.0)
    prec = np.insert(prec, 0, 1.0)

    ap = 0.
    delta = 1.0 / n_bins
    
    out_rec = np.arange(0, 1 + delta, delta)
    out_prec = np.zeros((n_bins+1), dtype=np.float32)

    for idx, t in enumerate(out_rec):
        prec1 = prec[rec >= t]
        if len(prec1) == 0:
            p = 0.
        else:
            p = max(prec1)
        
        out_prec[idx] = p
        ap = ap + p / (n_bins + 1)

    if plot_fn is not None:
        import matplotlib.pyplot as plt
        fig = plt.figure()
        plt.plot(out_rec, out_prec, 'b-')
        plt.title('PR-Curve (AP: %4.2f%%)' % (ap*100))
        plt.xlabel('Recall')
        plt.ylabel('Precision')
        plt.xlim([0, 1])
        plt.ylim([0, 1])
        fig.savefig(plot_fn)
        plt.close(fig)

    return ap

def eval_per_class_ap(stat_fn, gt_dir, pred_dir, iou_threshold=0.5, plot_dir=None):
    """ Input:  stat_fn contains all part ids and names 
                gt_dir contains test-xx.h5
                pred_dir contains test-xx.h5
        Output: aps: Average Prediction Scores for each part category, evaluated on all test shapes
                mAP: mean AP
    """
    print('Evaluation Start.')
    print('Ground-truth Directory: %s' % gt_dir)
    print('Prediction Directory: %s' % pred_dir)

    if plot_dir is not None:
        check_mkdir(plot_dir)

    # read stat_fn
    with open(stat_fn, 'r') as fin:
        part_name_list = [item.rstrip().split()[1] for item in fin.readlines()]
    print('Part Name List: ', part_name_list)
    n_labels = len(part_name_list)
    print('Total Number of Semantic Labels: %d' % n_labels)

    # check all h5 files
    test_h5_list = []
    for item in os.listdir(gt_dir):
        if item.startswith('test-') and item.endswith('.h5'):
            if not os.path.exists(os.path.join(pred_dir, item)):
                print('ERROR: h5 file %s is in gt directory but not in pred directory.')
                exit(1)
            test_h5_list.append(item)

    # read each h5 file and collect per-part-category true_pos, false_pos and confidence scores
    true_pos_list = [[] for item in part_name_list]
    false_pos_list = [[] for item in part_name_list]
    conf_score_list = [[] for item in part_name_list]

    gt_npos = np.zeros((n_labels), dtype=np.int32)

    for item in test_h5_list:
        print('Testing %s' % item)

        gt_mask, gt_mask_label, gt_mask_valid, gt_mask_other = load_gt_h5(os.path.join(gt_dir, item))
        pred_mask, pred_label, pred_valid, pred_conf = load_pred_h5(os.path.join(pred_dir, item))

        n_shape = gt_mask.shape[0]
        gt_n_ins = gt_mask.shape[1]
        pred_n_ins = pred_mask.shape[1]

        for i in range(n_shape):
            cur_pred_mask = pred_mask[i, ...]
            cur_pred_label = pred_label[i, :]
            cur_pred_conf = pred_conf[i, :]
            cur_pred_valid = pred_valid[i, :]
            
            cur_gt_mask = gt_mask[i, ...]
            cur_gt_label = gt_mask_label[i, :]
            cur_gt_valid = gt_mask_valid[i, :]
            cur_gt_other = gt_mask_other[i, :]

            # classify all valid gt masks by part categories
            gt_mask_per_cat = [[] for item in part_name_list]
            for j in range(gt_n_ins):
                if cur_gt_valid[j]:
                    sem_id = cur_gt_label[j]
                    gt_mask_per_cat[sem_id].append(j)
                    gt_npos[sem_id] += 1

            # sort prediction and match iou to gt masks
            cur_pred_conf[~cur_pred_valid] = 0.0
            order = np.argsort(-cur_pred_conf)

            gt_used = np.zeros((gt_n_ins), dtype=np.bool)

            for j in range(pred_n_ins):
                idx = order[j]
                if cur_pred_valid[idx]:
                    sem_id = cur_pred_label[idx]

                    iou_max = 0.0; cor_gt_id = -1;
                    for k in gt_mask_per_cat[sem_id]:
                        if not gt_used[k]:
                            # Remove points with gt label *other* from the prediction
                            # We will not evaluate them in the IoU since they can be assigned any label
                            clean_cur_pred_mask = (cur_pred_mask[idx, :] & (~cur_gt_other))

                            intersect = np.sum(cur_gt_mask[k, :] & clean_cur_pred_mask)
                            union = np.sum(cur_gt_mask[k, :] | clean_cur_pred_mask)
                            iou = intersect * 1.0 / union
                            
                            if iou > iou_max:
                                iou_max = iou
                                cor_gt_id = k
                                
                    if iou_max > iou_threshold:
                        gt_used[cor_gt_id] = True

                        # add in a true positive
                        true_pos_list[sem_id].append(True)
                        false_pos_list[sem_id].append(False)
                        conf_score_list[sem_id].append(cur_pred_conf[idx])
                    else:
                        # add in a false positive
                        true_pos_list[sem_id].append(False)
                        false_pos_list[sem_id].append(True)
                        conf_score_list[sem_id].append(cur_pred_conf[idx])

    # compute per-part-category AP
    aps = np.zeros((n_labels), dtype=np.float32)
    ap_valids = np.ones((n_labels), dtype=np.bool)
    for i in range(n_labels):
        has_pred = (len(true_pos_list[i]) > 0)
        has_gt = (gt_npos[i] > 0)

        if not has_gt:
            ap_valids[i] = False
            continue

        if has_gt and not has_pred:
            continue

        cur_true_pos = np.array(true_pos_list[i], dtype=np.float32)
        cur_false_pos = np.array(false_pos_list[i], dtype=np.float32)
        cur_conf_score = np.array(conf_score_list[i], dtype=np.float32)

        # sort according to confidence score again
        order = np.argsort(-cur_conf_score)
        sorted_true_pos = cur_true_pos[order]
        sorted_false_pos = cur_false_pos[order]

        out_plot_fn = None
        if plot_dir is not None:
            out_plot_fn = os.path.join(plot_dir, part_name_list[i].replace('/', '-')+'.png')

        aps[i] = compute_ap(sorted_true_pos, sorted_false_pos, gt_npos[i], plot_fn=out_plot_fn)

    # compute mean AP
    mean_ap = np.sum(aps * ap_valids) / np.sum(ap_valids)

    return aps, ap_valids, gt_npos, mean_ap

def eval_per_shape_mean_ap(stat_fn, gt_dir, pred_dir, iou_threshold=0.5):
    """ Input:  stat_fn contains all part ids and names 
                gt_dir contains test-xx.h5
                pred_dir contains test-xx.h5
        Output: mean_aps:       per-shape mean aps, which is the mean AP on each test shape,
                                for each shape, we only consider the parts that exist in either gt or pred
                shape_valids:   If a shape has valid parts to evaluate or not
                mean_mean_ap:   mean per-shape mean aps
    """
    print('Evaluation Start.')
    print('Ground-truth Directory: %s' % gt_dir)
    print('Prediction Directory: %s' % pred_dir)

    # read stat_fn
    with open(stat_fn, 'r') as fin:
        part_name_list = [item.rstrip().split()[1] for item in fin.readlines()]
    print('Part Name List: ', part_name_list)
    n_labels = len(part_name_list)
    print('Total Number of Semantic Labels: %d' % n_labels)

    # check all h5 files
    test_h5_list = []
    for item in os.listdir(gt_dir):
        if item.startswith('test-') and item.endswith('.h5'):
            if not os.path.exists(os.path.join(pred_dir, item)):
                print('ERROR: h5 file %s is in gt directory but not in pred directory.')
                exit(1)
            test_h5_list.append(item)

    mean_aps = []
    shape_valids = []

    # read each h5 file
    for item in test_h5_list:
        print('Testing %s' % item)

        gt_mask, gt_mask_label, gt_mask_valid, gt_mask_other = load_gt_h5(os.path.join(gt_dir, item))
        pred_mask, pred_label, pred_valid, pred_conf = load_pred_h5(os.path.join(pred_dir, item))

        n_shape = gt_mask.shape[0]
        gt_n_ins = gt_mask.shape[1]
        pred_n_ins = pred_mask.shape[1]

        for i in range(n_shape):
            cur_pred_mask = pred_mask[i, ...]
            cur_pred_label = pred_label[i, :]
            cur_pred_conf = pred_conf[i, :]
            cur_pred_valid = pred_valid[i, :]
            
            cur_gt_mask = gt_mask[i, ...]
            cur_gt_label = gt_mask_label[i, :]
            cur_gt_valid = gt_mask_valid[i, :]
            cur_gt_other = gt_mask_other[i, :]

            # per-shape evaluation
            true_pos_list = [[] for item in part_name_list]
            false_pos_list = [[] for item in part_name_list]
            gt_npos = np.zeros((n_labels), dtype=np.int32)

            # classify all valid gt masks by part categories
            gt_mask_per_cat = [[] for item in part_name_list]
            for j in range(gt_n_ins):
                if cur_gt_valid[j]:
                    sem_id = cur_gt_label[j]
                    gt_mask_per_cat[sem_id].append(j)
                    gt_npos[sem_id] += 1

            # sort prediction and match iou to gt masks
            cur_pred_conf[~cur_pred_valid] = 0.0
            order = np.argsort(-cur_pred_conf)

            gt_used = np.zeros((gt_n_ins), dtype=np.bool)

            # enumerate all pred parts
            for j in range(pred_n_ins):
                idx = order[j]
                if cur_pred_valid[idx]:
                    sem_id = cur_pred_label[idx]

                    iou_max = 0.0; cor_gt_id = -1;
                    for k in gt_mask_per_cat[sem_id]:
                        if not gt_used[k]:
                            # Remove points with gt label *other* from the prediction
                            # We will not evaluate them in the IoU since they can be assigned any label
                            clean_cur_pred_mask = (cur_pred_mask[idx, :] & (~cur_gt_other))

                            intersect = np.sum(cur_gt_mask[k, :] & clean_cur_pred_mask)
                            union = np.sum(cur_gt_mask[k, :] | clean_cur_pred_mask)
                            iou = intersect * 1.0 / union
                            
                            if iou > iou_max:
                                iou_max = iou
                                cor_gt_id = k
                                
                    if iou_max > iou_threshold:
                        gt_used[cor_gt_id] = True

                        # add in a true positive
                        true_pos_list[sem_id].append(True)
                        false_pos_list[sem_id].append(False)
                    else:
                        # add in a false positive
                        true_pos_list[sem_id].append(False)
                        false_pos_list[sem_id].append(True)

            # evaluate per-part-category AP for the shape
            aps = np.zeros((n_labels), dtype=np.float32)
            ap_valids = np.zeros((n_labels), dtype=np.bool)
            for j in range(n_labels):
                has_pred = (len(true_pos_list[j]) > 0)
                has_gt = (gt_npos[j] > 0)

                if has_pred and has_gt:
                    cur_true_pos = np.array(true_pos_list[j], dtype=np.float32)
                    cur_false_pos = np.array(false_pos_list[j], dtype=np.float32)
                    aps[j] = compute_ap(cur_true_pos, cur_false_pos, gt_npos[j])
                    ap_valids[j] = True
                elif has_pred and not has_gt:
                    aps[j] = 0.0
                    ap_valids[j] = True
                elif not has_pred and has_gt:
                    aps[j] = 0.0
                    ap_valids[j] = True

            # compute mean AP for the current shape
            if np.sum(ap_valids) > 0:
                mean_aps.append(np.sum(aps * ap_valids) / np.sum(ap_valids))
                shape_valids.append(True)
            else:
                mean_aps.append(0.0)
                shape_valids.append(False)

    # compute mean mean AP
    mean_aps = np.array(mean_aps, dtype=np.float32)
    shape_valids = np.array(shape_valids, dtype=np.bool)

    mean_mean_ap = np.sum(mean_aps * shape_valids) / np.sum(shape_valids)

    return mean_aps, shape_valids, mean_mean_ap


def eval_hier_mean_iou(gt_labels, pred_labels, tree_constraint, tree_parents):
    """
        Input:  
                gt_labels           B x N x (C+1), boolean
                pred_logits         B x N x (C+1), boolean
                tree_constraint     T x (C+1), boolean
                tree_parents        T, int32
        Output: 
                mean_iou            Scalar, float32
                part_iou            C, float32
    """
    assert gt_labels.shape[0] == pred_labels.shape[0], 'ERROR: gt and pred have different num_shape'
    assert gt_labels.shape[1] == pred_labels.shape[1], 'ERROR: gt and pred have different num_point'
    assert gt_labels.shape[2] == pred_labels.shape[2], 'ERROR: gt and pred have different num_class+1'

    num_shape = gt_labels.shape[0]
    num_point = gt_labels.shape[1]
    num_class = gt_labels.shape[2] - 1
    
    assert tree_constraint.shape[0] == tree_parents.shape[0], 'ERROR: tree_constraint and tree_parents have different num_constraint'
    assert tree_constraint.shape[1] == num_class + 1 , 'ERROR: tree_constraint.shape[1] != num_class + 1'
    assert len(tree_parents.shape) == 1, 'ERROR: tree_parents is not a 1-dim array'

    # make a copy of the prediction
    pred_labels = np.array(pred_labels, dtype=np.bool)

    num_constraint = tree_constraint.shape[0]

    part_intersect = np.zeros((num_class+1), dtype=np.float32)
    part_union = np.zeros((num_class+1), dtype=np.float32)

    part_intersect[1] = np.sum(pred_labels[:, :, 1] & gt_labels[:, :, 1])
    part_union[1] = np.sum(pred_labels[:, :, 1] | gt_labels[:, :, 1])

    all_idx = np.arange(num_class+1)
    all_visited = np.zeros((num_class+1), dtype=np.bool)

    all_visited[1] = True
    for i in range(num_constraint):
        cur_pid = tree_parents[i]
        if all_visited[cur_pid]:
            cur_cons = tree_constraint[i]

            idx = all_idx[cur_cons]
            gt_other = ((np.sum(gt_labels[:, :, idx], axis=-1) == 0) & gt_labels[:, :, cur_pid])

            for j in list(idx):
                pred_labels[:, :, j] = (pred_labels[:, :, cur_pid] & pred_labels[:, :, j] & (~gt_other))
                part_intersect[j] += np.sum(pred_labels[:, :, j] & gt_labels[:, :, j])
                part_union[j] += np.sum(pred_labels[:, :, j] | gt_labels[:, :, j])
                all_visited[j] = True

    all_valid_part_ids = all_idx[all_visited]
    part_iou = np.divide(part_intersect[all_valid_part_ids], part_union[all_valid_part_ids])
    mean_iou = np.mean(part_iou)

    return mean_iou, part_iou, part_intersect[all_valid_part_ids], part_union[all_valid_part_ids]