Faster Rcnn 代碼解讀之 anchor_target_layer.py

# --------------------------------------------------------
# Faster R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick and Xinlei Chen
# --------------------------------------------------------
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
from model.config import cfg
import numpy as np
import numpy.random as npr
from utils.cython_bbox import bbox_overlaps
from model.bbox_transform import bbox_transform

'''
anchor_target_layer主要針對RPN的輸出進行處理，
對RPN的輸出結果加工，對anchor打上标簽，
然後通過與Gt的比對，計算出與真實框的偏差
'''


def anchor_target_layer(rpn_cls_score, gt_boxes, im_info, _feat_stride, all_anchors, num_anchors):
    """Same as the anchor target layer in original Fast/er RCNN """
    A = num_anchors  # 我的了解：單通道anchor的數量
    total_anchors = all_anchors.shape[0]  # 我的了解：多通道anchor的數量，也就是說所有的框
    K = total_anchors / num_anchors  #

    # allow boxes to sit over the edge by a small amount
    # _allowed_border代表框是否允許貼近image的邊緣，0代表不允許
    _allowed_border = 0

    # map of shape (..., H, W)
    # 輸出rpn_cls_score的height和width
    height, width = rpn_cls_score.shape[1:3]

    # only keep anchors inside the image
    # 隻保留在img範圍内的box，過濾掉越界的
    inds_inside = np.where(
        (all_anchors[:, 0] >= -_allowed_border) &  # 假如_allowed_border=100，說明x>=-100，也就是允許x為負，也就是說明x在邊界外面了，0自然就不允許
        (all_anchors[:, 1] >= -_allowed_border) &  # 同理
        (all_anchors[:, 2] < im_info[1] + _allowed_border) &  # width
        (all_anchors[:, 3] < im_info[0] + _allowed_border)  # height
    )[0]  # 這樣看all_anchors的存儲方式是[x,y,w,h]

    # keep only inside anchors
    # 隻保留限定的anchors，例如_allowed_border=0，隻保留在圖像内的框
    anchors = all_anchors[inds_inside, :]

    # label: 1 is positive, 0 is negative, -1 is dont care
    # 隻給不越界的anchor指派，首先全部标記負樣本
    labels = np.empty((len(inds_inside),), dtype=np.float32)
    labels.fill(-1)

    # overlaps between the anchors and the gt boxes
    # overlaps (ex, gt)
    # 計算anchors和gt_boxes的重合率
    # np.ascontiguousarray傳回一個指定資料類型的連續數組，轉存為順序結構的資料
    # anchors*gt_boxes
    overlaps = bbox_overlaps(
        np.ascontiguousarray(anchors, dtype=np.float),
        np.ascontiguousarray(gt_boxes, dtype=np.float))
    # 求每一行的最大值下标，應該就是每個anchor對應分數最大的gt_box
    argmax_overlaps = overlaps.argmax(axis=1)
    # 将每行得到的最大值儲存
    max_overlaps = overlaps[np.arange(len(inds_inside)), argmax_overlaps]
    # 求每一列的最大值下标，應該就是每個gt_box對應分數最大的anchor
    gt_argmax_overlaps = overlaps.argmax(axis=0)
    # 同理，儲存每列的最大值
    gt_max_overlaps = overlaps[gt_argmax_overlaps,
                               np.arange(overlaps.shape[1])]  # overlaps.shape[1]是gt_box個數
    # 保留那些相等的索引
    gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0]

    # RPN_CLOBBER_POSITIVES=True 代表 如果同時滿足正負條件設定為負樣本
    # if RPN_CLOBBER_POSITIVES = False 将所有滿足負樣本label阙值的anchor标記為0
    if not cfg.TRAIN.RPN_CLOBBER_POSITIVES:
        # assign bg labels first so that positive labels can clobber them
        # first set the negatives
        labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0

    # fg label: for each gt, anchor with highest overlap
    # 前景标簽，對于gt_boxes,和anchors重合率最大的檢測框标記為1
    labels[gt_argmax_overlaps] = 1

    # fg label: above threshold IOU
    # 将anchor最大的分并且大于RPN_POSITIVE_OVERLAP的标記為1
    labels[max_overlaps >= cfg.TRAIN.RPN_POSITIVE_OVERLAP] = 1
    # 對那些與某個框的最大交疊值的門檻值小于負樣本的，設定為0，也就是負樣本
    # 意思是，有些檢測結果是某些框的最大交疊結果，但是交疊還是低于負樣本門檻值了，這些樣本作為負樣本
    if cfg.TRAIN.RPN_CLOBBER_POSITIVES:
        # assign bg labels last so that negative labels can clobber positives
        labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0

    # subsample positive labels if we have too many
    # 如果還有過多的正樣本，再采樣一次，平衡正負樣本
    '''
    # Max number of foreground examples
    __C.TRAIN.RPN_FG_FRACTION = 0.5 # 代表每次訓練RPN的比例
    '''
    num_fg = int(cfg.TRAIN.RPN_FG_FRACTION * cfg.TRAIN.RPN_BATCHSIZE)
    # 找出所有正樣本的下标,1*len(fg_inds)
    fg_inds = np.where(labels == 1)[0]
    if len(fg_inds) > num_fg:  # 如果超過了制定的數量，使用random.choice()随機采樣
        disable_inds = npr.choice(
            fg_inds, size=(len(fg_inds) - num_fg), replace=False)
        labels[disable_inds] = -1  # 将随機出來額外的标記為負樣本

    # subsample negative labels if we have too many
    # 同理，但對num的設定有所不同
    num_bg = cfg.TRAIN.RPN_BATCHSIZE - np.sum(labels == 1)
    bg_inds = np.where(labels == 0)[0]
    if len(bg_inds) > num_bg:
        disable_inds = npr.choice(
            bg_inds, size=(len(bg_inds) - num_bg), replace=False)
        labels[disable_inds] = -1

    bbox_targets = np.zeros((len(inds_inside), 4), dtype=np.float32)
    # argmax_overlaps:每個anchor對應的最大gt_box下标
    # gt_boxes[argmax_overlaps, :] : 滿足每個anchor對應的gt_box下标的gt_box資訊
    # anchors:[x,y,w,h]
    # 計算與anchor與最大重疊的GT的偏移量
    bbox_targets = _compute_targets(anchors, gt_boxes[argmax_overlaps,
                                             :])  # return bbox_transform(ex_rois, gt_rois[:, :4]).astype(np.float32, copy=False)
    # 設定正樣本回歸 loss 的權重
    bbox_inside_weights = np.zeros((len(inds_inside), 4), dtype=np.float32)
    # only the positive ones have regression targets
    # 對正樣本賦初值
    bbox_inside_weights[labels == 1, :] = np.array(
        cfg.TRAIN.RPN_BBOX_INSIDE_WEIGHTS)  # __C.TRAIN.RPN_BBOX_INSIDE_WEIGHTS = (1.0, 1.0, 1.0, 1.0)

    bbox_outside_weights = np.zeros((len(inds_inside), 4), dtype=np.float32)

    if cfg.TRAIN.RPN_POSITIVE_WEIGHT < 0:  # __C.TRAIN.RPN_POSITIVE_WEIGHT = -1.0 設定-1使用統一的權重
        # uniform weighting of examples (given non-uniform sampling)
        num_examples = np.sum(labels >= 0)  # labels>=0的個數，也就是樣本數目
        positive_weights = np.ones((1, 4)) * 1.0 / num_examples
        negative_weights = np.ones((1, 4)) * 1.0 / num_examples
    else:
        assert ((cfg.TRAIN.RPN_POSITIVE_WEIGHT > 0) &
                (cfg.TRAIN.RPN_POSITIVE_WEIGHT < 1))
        # 如果不是-1，__C.TRAIN.RPN_POSITIVE_WEIGHT = p(0<p<1)，positive權重就是p/{num positive},negative權重為(1-p)/{num negative}
        positive_weights = (cfg.TRAIN.RPN_POSITIVE_WEIGHT /
                            np.sum(labels == 1))
        negative_weights = ((1.0 - cfg.TRAIN.RPN_POSITIVE_WEIGHT) /
                            np.sum(labels == 0))
    bbox_outside_weights[labels == 1, :] = positive_weights
    bbox_outside_weights[labels == 0, :] = negative_weights

    # map up to original set of anchors
    # _unmap的作用是從在框内的anchor又擴充回全部的anchor尺寸，fill代表在邊界外的anchor需要填充的數字
    # 也就是映射回原來的total_anchor集合
    labels = _unmap(labels, total_anchors, inds_inside, fill=-1)
    bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, fill=0)
    bbox_inside_weights = _unmap(bbox_inside_weights, total_anchors, inds_inside, fill=0)
    bbox_outside_weights = _unmap(bbox_outside_weights, total_anchors, inds_inside, fill=0)

    # labels
    # A = num_anchors  # 我的了解：單通道anchor的數量
    labels = labels.reshape((1, height, width, A)).transpose(0, 3, 1, 2)
    labels = labels.reshape((1, 1, A * height, width))
    rpn_labels = labels

    # bbox_targets
    bbox_targets = bbox_targets \
        .reshape((1, height, width, A * 4))

    rpn_bbox_targets = bbox_targets
    # bbox_inside_weights
    bbox_inside_weights = bbox_inside_weights \
        .reshape((1, height, width, A * 4))

    rpn_bbox_inside_weights = bbox_inside_weights

    # bbox_outside_weights
    bbox_outside_weights = bbox_outside_weights \
        .reshape((1, height, width, A * 4))

    rpn_bbox_outside_weights = bbox_outside_weights
    return rpn_labels, rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights


def _unmap(data, count, inds, fill=0):
    """ Unmap a subset of item (data) back to the original set of items (of
    size count) """
    # 在剛開始處理時有可能會去掉一些邊界外的，這些資料就一直未處理，還原原來的大小，對未處理的填充fill值
    if len(data.shape) == 1:
        ret = np.empty((count,), dtype=np.float32)  # 生成一些随機數
        ret.fill(fill)  # 然後全部填充fill
        ret[inds] = data  # 再把原來的資料給指派回去
    else:  # 多元的
        ret = np.empty((count,) + data.shape[1:], dtype=np.float32)
        ret.fill(fill)
        ret[inds, :] = data
    return ret


def _compute_targets(ex_rois, gt_rois):
    """Compute bounding-box regression targets for an image."""

    assert ex_rois.shape[0] == gt_rois.shape[0]
    assert ex_rois.shape[1] == 4
    assert gt_rois.shape[1] == 5
    # 傳回的是(targets_dx, targets_dy, targets_dw, targets_dh))
    return bbox_transform(ex_rois, gt_rois[:, :4]).astype(np.float32, copy=False)