From 0d096b2e725422e84ae97ba9d74e85a2d8583e17 Mon Sep 17 00:00:00 2001 From: ktro2828 Date: Mon, 4 Dec 2023 13:22:59 +0900 Subject: [PATCH 1/2] fix: remove checking label while matching objects for classification Signed-off-by: ktro2828 --- .../evaluation/result/object_result.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/perception_eval/perception_eval/evaluation/result/object_result.py b/perception_eval/perception_eval/evaluation/result/object_result.py index 0bb4f706..b7e835a7 100644 --- a/perception_eval/perception_eval/evaluation/result/object_result.py +++ b/perception_eval/perception_eval/evaluation/result/object_result.py @@ -386,11 +386,7 @@ def _get_object_results_with_id( raise RuntimeError( f"uuid of estimation and ground truth must be set, but got {est_object.uuid} and {gt_object.uuid}" ) - if ( - est_object.uuid == gt_object.uuid - and est_object.semantic_label == gt_object.semantic_label - and est_object.frame_id == gt_object.frame_id - ): + if est_object.uuid == gt_object.uuid and est_object.frame_id == gt_object.frame_id: object_results.append( DynamicObjectWithPerceptionResult( estimated_object=est_object, @@ -427,7 +423,11 @@ def _get_object_results_for_tlr( ground_truth_objects_ = ground_truth_objects.copy() for est_object in estimated_objects: for gt_object in ground_truth_objects_: - if est_object.uuid == gt_object.uuid and est_object.semantic_label == gt_object.semantic_label: + if est_object.uuid is None or gt_object.uuid is None: + raise RuntimeError( + f"uuid of estimation and ground truth must be set, but got {est_object.uuid} and {gt_object.uuid}" + ) + if est_object.uuid == gt_object.uuid and est_object.frame_id == gt_object.frame_id: object_results.append( DynamicObjectWithPerceptionResult( estimated_object=est_object, @@ -436,10 +436,6 @@ def _get_object_results_for_tlr( ) estimated_objects_.remove(est_object) ground_truth_objects_.remove(gt_object) - logging.info( - f"[OK] Est: {est_object.semantic_label.label}:{est_object.uuid}, " - f"GT: {gt_object.semantic_label.label}:{gt_object.uuid}" - ) # when there are rest of a GT objects, one of the estimated objects is FP. if 0 < len(ground_truth_objects_): object_results += _get_fp_object_results([estimated_objects_[0]]) From adf88b7c6d9a53a17d216d7b21165572ee8863c6 Mon Sep 17 00:00:00 2001 From: ktro2828 Date: Thu, 28 Dec 2023 06:30:51 +0900 Subject: [PATCH 2/2] feat: update matching policy as same label first Signed-off-by: ktro2828 --- .../evaluation/result/object_result.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/perception_eval/perception_eval/evaluation/result/object_result.py b/perception_eval/perception_eval/evaluation/result/object_result.py index b7e835a7..0d5ff1bb 100644 --- a/perception_eval/perception_eval/evaluation/result/object_result.py +++ b/perception_eval/perception_eval/evaluation/result/object_result.py @@ -14,7 +14,6 @@ from __future__ import annotations -import logging from typing import Callable from typing import List from typing import Optional @@ -381,7 +380,7 @@ def _get_object_results_with_id( estimated_objects_ = estimated_objects.copy() ground_truth_objects_ = ground_truth_objects.copy() for est_object in estimated_objects: - for gt_object in ground_truth_objects_: + for gt_object in ground_truth_objects: if est_object.uuid is None or gt_object.uuid is None: raise RuntimeError( f"uuid of estimation and ground truth must be set, but got {est_object.uuid} and {gt_object.uuid}" @@ -421,12 +420,37 @@ def _get_object_results_for_tlr( object_results: List[DynamicObjectWithPerceptionResult] = [] estimated_objects_ = estimated_objects.copy() ground_truth_objects_ = ground_truth_objects.copy() + # 1. matching based on same label primary for est_object in estimated_objects: - for gt_object in ground_truth_objects_: + for gt_object in ground_truth_objects: if est_object.uuid is None or gt_object.uuid is None: raise RuntimeError( f"uuid of estimation and ground truth must be set, but got {est_object.uuid} and {gt_object.uuid}" ) + + if ( + est_object.uuid == gt_object.uuid + and est_object.frame_id == gt_object.frame_id + and est_object.semantic_label == gt_object.semantic_label + ): + object_results.append( + DynamicObjectWithPerceptionResult( + estimated_object=est_object, + ground_truth_object=gt_object, + ) + ) + estimated_objects_.remove(est_object) + ground_truth_objects_.remove(gt_object) + # 2. matching based on same ID + rest_estimated_objects_ = estimated_objects_.copy() + rest_ground_truth_objects_ = ground_truth_objects_.copy() + for est_object in rest_estimated_objects_: + for gt_object in rest_ground_truth_objects_: + if est_object.uuid is None or gt_object.uuid is None: + raise RuntimeError( + f"uuid of estimation and ground truth must be set, but got {est_object.uuid} and {gt_object.uuid}" + ) + if est_object.uuid == gt_object.uuid and est_object.frame_id == gt_object.frame_id: object_results.append( DynamicObjectWithPerceptionResult( @@ -436,9 +460,6 @@ def _get_object_results_for_tlr( ) estimated_objects_.remove(est_object) ground_truth_objects_.remove(gt_object) - # when there are rest of a GT objects, one of the estimated objects is FP. - if 0 < len(ground_truth_objects_): - object_results += _get_fp_object_results([estimated_objects_[0]]) return object_results