python evaluation code

[mingminzhen]

I write a python code to evaluate the pretrained model:

import argparse
import cv2
import numpy as np
import sys
import os
import math
import time
from PIL import Image as PILImage
import matplotlib.pyplot as plt
import argparse
import json

caffe_root = '../..'
#os.chdir(caffe_root)
sys.path.insert(0, os.path.join(caffe_root, 'python'))
sys.path.insert(0, os.path.join(caffe_root, 'cocoapi','PythonAPI'))
import caffe
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
parser = argparse.ArgumentParser()
parser.add_argument('--gpu_id', type=int, default=0, help='gpu id')
parser.add_argument('--cascade', default=0, type=int)
parser.add_argument('--FPN', default=0, type=int)
parser.add_argument('--coco_dir', default='', help='coco_dir')
parser.add_argument('--model_weights', default='', help='model_weights')
parser.add_argument('--model_proto', default='models/deploy.prototxt')
parser.add_argument('--show', action='store_true', help='visualize detections')
parser.add_argument('--json_dir', default='output')

args = parser.parse_args()
print(args)

def eval_json():
    data_type = 'val2017'
    image_dir = os.path.join(args.coco_dir, 'images', data_type)
    ann_file  = os.path.join(args.coco_dir, 'annotations','instances_' + data_type + '.json')
    coco      = COCO(ann_file)
    cat_ids   = coco.getCatIds()
    cats      = coco.loadCats(cat_ids)
    image_ids = coco.getImgIds()
    detection = CaffeDetection(args.gpu_id,
                               args.model_proto, 
                               args.model_weights,
                               cascade=args.cascade, 
                               FPN=args.FPN)
    for nn in range(detection.num_outputs):
        json_name = 'output_'+detection.output_names[nn]+'_results.json'
        json_file = os.path.join(args.json_dir, json_name)
        # json_file = '/home/mingmin/Documents/detectron.pytorch/Outputs/e2e_faster_rcnn_R-50-FPN_1x/Jun15-15-23-18_zdai-SYS-4028GR-TR_step/test/bbox_coco_2017_val_results.json'
        coco_pre  = coco.loadRes(json_file)
        cocoEval  = COCOeval(coco,coco_pre);
        cocoEval.params.imgIds = image_ids;
        cocoEval.params.useSegm= False;
        cocoEval.evaluate();
        cocoEval.accumulate();
        cocoEval.summarize();
        print('\n')
        # sys.exit()



def detect_json():
    print('eval the coco object detection')
    if not os.path.exists(args.json_dir):
        os.makedirs(args.json_dir)
    # annTypes = { 'instances', 'captions', 'person_keypoints' };
    data_type = 'val2017'
    image_dir = os.path.join(args.coco_dir, 'images', data_type)
    ann_file  = os.path.join(args.coco_dir, 'annotations','instances_' + data_type + '.json')
    coco = COCO(ann_file)
    cat_ids   = coco.getCatIds()
    cats      = coco.loadCats(cat_ids)
    image_ids = coco.getImgIds()
    image_ids.sort()
    # for idx, cat_id in enumerate(cat_ids):
    #     print(idx, cat_id)
    # print('--------------------\n')
    # for idx, cat in enumerate(cats):
    #     print(idx,cat)
    # sys.exit()
    print(len(image_ids))
    detection = CaffeDetection(args.gpu_id,
                               args.model_proto, 
                               args.model_weights,
                               cascade=args.cascade, 
                               FPN=args.FPN)
    detection.init_net()
    detect_final_boxes = [[[] for j in range(len(image_ids))] for i in range(detection.num_outputs)]
    image_num =  len(image_ids)
    for idx in range(image_num):
        img_id = image_ids[idx]
        img    = coco.loadImgs(img_id)[0]
        img_name = img['file_name'].replace('.jpg', '')
        print('process {}/{} (img_id:{},file:{})'.format(idx,len(image_ids),img_id,img_name))
        img_path = image_dir + '/' + img['file_name']
        image    = cv2.imread(img_path, cv2.IMREAD_COLOR)
        # print('image mode:',image.mode)
        results   = detection.detect(image)
        if args.show:
            det_image = image.copy()
            for item in results[len(results)-1]:# the 3rd_avg result
                xmin = int(round(item[1]))
                ymin = int(round(item[2]))
                xmax = int(round(item[1] + item[3] - 1))
                ymax = int(round(item[2] + item[4] - 1))
                cls_id = int(item[0])
                score  = round(item[-1]*1000)/1000
                if score < 0.1:
                    continue
                cv2.rectangle(det_image, (xmin, ymin), (xmax, ymax),(0,255,0),1)
                cv2.putText(det_image, cats[cls_id-1]['name'], (xmin, ymax)
                            ,cv2.FONT_HERSHEY_PLAIN
                            ,fontScale=1
                            ,color=(255,255,255)
                            ,thickness=1)
                print [cls_id, xmin, ymin, xmax, ymax, round(item[-1]*1000)/1000]
            plt.figure(1)
            plt.subplot(121)
            plt.imshow(image)
            plt.title('image')

            plt.subplot(122)
            plt.imshow(det_image, interpolation='bicubic')
            plt.title('detect image')
            plt.show()
            sys.exit()
        for nn in range(detection.num_outputs):
            # print( nn, detection.num_outputs,results[nn])
            detect_final_boxes[nn][idx].extend(results[nn])
            # print('detect_final_boxes:', detect_final_boxes[nn][idx])
        # sys.exit()
    for nn in range(detection.num_outputs):
        results = []
        for ii in range(image_num):
            det_boxes = detect_final_boxes[nn][ii]
            num_boxes = len(det_boxes)
            for jj in range(num_boxes):
                res = {}
                res['image_id'] = image_ids[ii]
                res['category_id'] = cat_ids[int(det_boxes[jj][0]-1)]
                res['bbox'] =[det_boxes[jj][1], det_boxes[jj][2],
                              det_boxes[jj][3],
                              det_boxes[jj][4]]
                res['score'] = det_boxes[jj][5]
                results.append(res)
        res_file = os.path.join(args.json_dir, 'output_'+detection.output_names[nn]+'_results.json')
        with open(res_file, 'w') as fid:
            json.dump(results, fid)

    sys.exit()

class CaffeDetection:
    def __init__(self, gpu_id, model_def, model_weights,
                 cascade=0, FPN=0):
        self.gpu_id = gpu_id
        self.model_def = model_def
        self.model_weights = model_weights
        # input preprocessing: 'data' is the name of the input blob == net.inputs[0]
        #self.transformer = caffe.io.Transformer({'data': self.net.blobs['data'].data.shape})
        #self.transformer.set_transpose('data', (2, 0, 1))
        #self.transformer.set_mean('data', np.array([104, 117, 123])) # mean pixel
        ## the reference model operates on images in [0,255] range instead of [0,1]
        #self.transformer.set_raw_scale('data', 255)
        ## the reference model has channels in BGR order instead of RGB
        #self.transformer.set_channel_swap('data', (2, 1, 0))

        self.cascade = cascade > 0
        self.FPN = FPN > 0
        # print(cascade,FPN)
        if not self.cascade:
            # baseline model
            if self.FPN:
                self.proposal_blob_names = ['proposals_to_all']
            else:
                self.proposal_blob_names = ['proposals']

            self.bbox_blob_names = ['output_bbox_1st']
            self.cls_prob_blob_names = ['cls_prob_1st']
            self.output_names = ['1st']
        else:
            # cascade-rcnn model
            if self.FPN:
                self.proposal_blob_names = ['proposals_to_all', 'proposals_to_all_2nd',
                                       'proposals_to_all_3rd', 'proposals_to_all_2nd', 'proposals_to_all_3rd']
            else:
                self.proposal_blob_names = ['proposals', 'proposals_2nd', 'proposals_3rd',
                                       'proposals_2nd', 'proposals_3rd']

            self.bbox_blob_names = ['output_bbox_1st', 'output_bbox_2nd', 'output_bbox_3rd',
                           'output_bbox_2nd', 'output_bbox_3rd']
            self.cls_prob_blob_names = ['cls_prob_1st', 'cls_prob_2nd', 'cls_prob_3rd',
                               'cls_prob_2nd_avg', 'cls_prob_3rd_avg']
            self.output_names = ['1st', '2nd', '3rd', '2nd_avg', '3rd_avg']

        self.num_outputs = len(self.proposal_blob_names)
        assert(self.num_outputs==len(self.bbox_blob_names))
        assert(self.num_outputs==len(self.cls_prob_blob_names))
        assert(self.num_outputs==len(self.output_names))
        # detection configuration
        # detect_final_boxes = np.zeros(nImg, num_outputs)
        #self.det_thr = 0.001 # threshold for testing
        self.det_thr = 0.001 # threshold for demo
        self.max_per_img = 100 # max number of detections
        self.nms_thresh = 0.5 # NMS
        if self.FPN:
            self.shortSize = 800
            self.longSize = 1312
        else:
            self.shortSize = 600
            self.longSize = 1000

        self.PIXEL_MEANS = np.array([104, 117, 123],dtype=np.uint8)
        self.num_cls = 80
    def init_net(self):
        if self.gpu_id < 0:
            caffe.set_mode_cpu()
        else:
            caffe.set_device(self.gpu_id)
            caffe.set_mode_gpu()

        # Load the net in the test phase for inference, and configure input preprocessing.
        self.net = caffe.Net(self.model_def,      # defines the structure of the model
                             self.model_weights,  # contains the trained weights
                             caffe.TEST)     # use test mode (e.g., don't perform dropout)

    def detect(self, image):
        '''
        rcnn detection
        '''
        #image = caffe.io.load_image(image_file)
        # image = cv2.imread(image_file) # BGR, default is cv2.IMREAD_COLOR 3-channel
        print('image:', image[0,0,:])
        orgH, orgW, channel = image.shape
        print("image shape:",image.shape)
        # print('short/long size:', self.shortSize, self.longSize)
        rzRatio = self.shortSize / min(float(orgH), float(orgW))
        # print('rzRatio:', rzRatio)
        imgH = min(rzRatio * orgH, self.longSize)
        imgW = min(rzRatio * orgW, self.longSize)
        # print('imgH, imgW:', imgH, imgW)
        imgH = round(imgH / 32) * 32
        imgW = round(imgW / 32) * 32 # must be the multiple of 32
        hwRatios = [imgH/orgH, imgW/orgW]
        #transformed_image = self.transformer.preprocess('data', image)
        #image = cv2.resize(im_orig, None, None, fx=im_scale, fy=im_scale,
        resized_w = int(imgW)
        resized_h = int(imgH)
        print('resized -> ',(resized_w, resized_h))
        image = cv2.resize(image, (resized_w, resized_h), interpolation=cv2.INTER_LINEAR)
        image -= self.PIXEL_MEANS
        #cv2.imwrite("transformed_image.jpg", image)
        transformed_image = np.transpose(image, (2,0,1)) # C H W
        print('transformed_image:', transformed_image.shape)
        # set net to batch size of 1
        self.net.blobs['data'].reshape(1, 3, resized_h, resized_w)
        #Run the net and examine the top_k results
        self.net.blobs['data'].data[...] = transformed_image.astype(np.float32, copy=False)
        start = time.time()
        # Forward pass.
        blobs_out = self.net.forward()
        # print('output_bbox_1st---',blobs_out['output_bbox_1st'].shape)
        #print blobs_out
        end = time.time()
        cost_millis = int((end - start) * 1000)
        # print "detection cost ms: ", cost_millis

        detect_final_boxes = [[] for i in range(self.num_outputs)]
        for nn in range(self.num_outputs):
            # detect_boxes = cell(num_cls, 1);
            tmp = self.net.blobs[self.bbox_blob_names[nn]].data.copy() # if no need modify,then no need copy
            # print(self.bbox_blob_names[nn], tmp.shape)
            #tmp = tmp.reshape((-1,5))
            tmp = tmp[:,:,0,0]
            tmp[:,1] /= hwRatios[1]
            tmp[:,3] /= hwRatios[1]
            tmp[:,2] /= hwRatios[0]
            tmp[:,4] /= hwRatios[0]

            # clipping bbs to image boarders
            tmp[:, 1] = np.maximum(0,tmp[:,1])
            tmp[:, 2] = np.maximum(0,tmp[:,2])
            tmp[:, 3] = np.minimum(orgW,tmp[:,3])
            tmp[:, 4] = np.minimum(orgH,tmp[:,4])
            tmp[:, 3] = tmp[:, 3] - tmp[:, 1] + 1 # w
            tmp[:, 4] = tmp[:, 4] - tmp[:, 2] + 1 # h

            output_bboxs = tmp[:,1:]
            tmp = self.net.blobs[self.cls_prob_blob_names[nn]].data
            # print(self.cls_prob_blob_names[nn], tmp.shape)
            cls_prob = tmp.reshape((-1,self.num_cls+1))
            tmp = self.net.blobs[self.proposal_blob_names[nn]].data.copy()
            # print(self.proposal_blob_names[nn], tmp.shape)
            tmp = tmp[:,1:]
            tmp[:, 2] = tmp[:, 2] - tmp[:, 0] + 1  # w
            tmp[:, 3] = tmp[:, 3] - tmp[:, 1] + 1  # h
            proposals = tmp
            keep_id   = np.where((proposals[:, 2] > 0) & (proposals[:, 3] > 0))[0]
            proposals = proposals[keep_id,:]
            output_bboxs = output_bboxs[keep_id,:]
            cls_prob     = cls_prob[keep_id,:]

            detect_boxes = []
            for i in range(self.num_cls):
                cls_id = i + 1
                prob = cls_prob[:, cls_id][:, np.newaxis] # 0 is background
                #print (output_bboxs.shape, prob.shape)
                bbset = np.hstack([output_bboxs, prob])
                if self.det_thr > 0:
                    keep_id = np.where(prob >= self.det_thr)[0]
                    bbset   = bbset[keep_id,:]
                keep = self.cpu_nms_single_cls(bbset, self.nms_thresh)
                if len(keep) == 0: 
                    continue
                bbset = bbset[keep,:]
                cls_ids = np.array([cls_id] * len(bbset))[:, np.newaxis]
                #print "cls_ids.shape", cls_ids.shape, bbset.shape
                detect_boxes.extend(np.hstack([cls_ids, bbset]).tolist())
            # print "detected box num: ", len(detect_boxes)
            detect_boxes = np.asarray(detect_boxes)
            if self.max_per_img > 0 and len(detect_boxes) > self.max_per_img:
                rank_scores = detect_boxes[:, 5].copy()[::-1]
                rank_scores.sort() # 'descend'
                # print len(rank_scores),self.max_per_img
                # print np.where(detect_boxes[:, 5] >= rank_scores[self.max_per_img])
                keep_id = np.where(detect_boxes[:, 5] >= rank_scores[self.max_per_img])[0]
                detect_boxes = detect_boxes[keep_id,:]
            #detect_final_boxes.extend(detect_boxes.tolist())
            #detect_final_boxes.append(detect_boxes.tolist())
            # print('detect_boxes:', detect_boxes.shape)
            # print('detect_boxes:', detect_boxes.tolist())
            detect_final_boxes[nn].extend(detect_boxes.tolist())
            # print('nn/num_outputs:', nn, self.num_outputs)
        # print('detect_final_boxes:', detect_final_boxes)
        return detect_final_boxes

    def cpu_nms_single_cls(self, dets, thresh):
        """Pure Python NMS baseline."""
        x1 = dets[:, 0]
        y1 = dets[:, 1]
        w  = dets[:, 2]
        h  = dets[:, 3]
        scores = dets[:, 4]

        x2 = x1 + w - 1
        y2 = y1 + h - 1
        # areas = (x2 - x1 + 1) * (y2 - y1 + 1)
        areas = w * h
        order = scores.argsort()[::-1]

        keep = []
        while order.size > 0:
            i = order[0]
            keep.append(i)
            xx1 = np.maximum(x1[i], x1[order[1:]])
            yy1 = np.maximum(y1[i], y1[order[1:]])
            xx2 = np.minimum(x2[i], x2[order[1:]])
            yy2 = np.minimum(y2[i], y2[order[1:]])

            w = np.maximum(0.0, xx2 - xx1 + 1)
            h = np.maximum(0.0, yy2 - yy1 + 1)
            inter = w * h
            ovr = inter / (areas[i] + areas[order[1:]] - inter)

            inds = np.where(ovr <= thresh)[0]
            order = order[inds + 1]
        return keep

def main():
    detect_json()
    eval_json()
if __name__ == '__main__':
    main()

But the evaluate result is very different from reported result. I doubt something is wrong with the code. Since there is no matlab in my machine. Can anyone help debug it?

[xiaoxiongli]

@mingminzhen have you fix it? ^_^

[Rick51]

【rank_scores.sort() # 'descend'】这一句有问题啊，虽然注释了降序排列，但是程序还是升序排列的撒，改了后和作者结果一样。

[KeepGoing2019HaHa]

@Rick51 hi, I sort rank_scores in descend order, but I still can not get the same results. Here is what I have changed

rank_scores = detect_boxes[:, 5].copy()[::-1]
rank_scores = np.sort(rank_scores)[::-1]

Is this a correct way to do so? May I ask what changes do you make to this python code? Thanks