update example eval script

.github/workflows/run_tests_os.yml

@@ -7,7 +7,8 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         # python-version: ["3.8", "3.10"]
-        python-version: ["3.8"]  # run less in parallel to avoid download problems??
+        # Download does not work properly with python 3.8 on non-linux OS
+        python-version: ["3.10"] 
     runs-on: ${{ matrix.os }}
 
     steps:

resources/evaluate.py

@@ -31,8 +31,8 @@ def calc_metrics(subject, gt_dir=None, pred_dir=None, class_map=None):
 
     r = {"subject": subject}
     for idx, roi_name in class_map.items():
-        gt = (gt_all == idx).astype(np.uint8)
-        pred = (pred_all == idx).astype(np.uint8)
+        gt = gt_all == idx
+        pred = pred_all == idx
 
         if gt.max() > 0 and pred.max() == 0:
             r[f"dice-{roi_name}"] = 0
@@ -58,24 +58,25 @@ def calc_metrics(subject, gt_dir=None, pred_dir=None, class_map=None):
     gt_dir = Path(sys.argv[1])  # directory containining all the subjects
     pred_dir = Path(sys.argv[2])  # directory of the new nnunet dataset
 
-    class_map = class_map_5_parts["class_map_part_organs"]
+    # class_map = class_map_5_parts["class_map_part_organs"]
     # class_map = class_map_5_parts["class_map_part_vertebrae"]
     # class_map = class_map_5_parts["class_map_part_cardiac"]
     # class_map = class_map_5_parts["class_map_part_muscles"]
     # class_map = class_map_5_parts["class_map_part_ribs"]
+    class_map_name = sys.argv[3]  
+    class_map = class_map_5_parts[class_map_name]
 
-    subjects = [x.stem for x in gt_dir.glob("*.nii.gz")]
+    subjects = [x.stem.split(".")[0] for x in gt_dir.glob("*.nii.gz")]
 
     # Use multiple threads to calculate the metrics
-    res = p_map(partial(calc_metrics, gt_dir=data_dir, pred_dir=data_dir,
-                        class_map=class_map), subjects, num_cpus=8)
+    res = p_map(partial(calc_metrics, gt_dir=gt_dir, pred_dir=pred_dir,
+                        class_map=class_map), subjects, num_cpus=8, disable=True)
     res = pd.DataFrame(res)
-    print(f"result shape: {res.shape}")
 
-    print("\n------Overall mean------")
     for metric in ["dice", "surface_dice_3"]:
         res_all_rois = []
         for roi_name in class_map.values():
             row_wo_nan = res[f"{metric}-{roi_name}"].dropna()
             res_all_rois.append(row_wo_nan.mean())
-        print(f"{metric}: {np.array(res_all_rois).mean():.3f}")
+            print(f"{roi_name} {metric}: {row_wo_nan.mean():.3f}")
+        # print(f"{metric}: {np.array(res_all_rois).mean():.3f}")

resources/evaluate_results.txt

@@ -0,0 +1,240 @@
+# Dice
+
+spleen: 0.984
+kidney_right: 0.947
+kidney_left: 0.944
+gallbladder: 0.845
+liver: 0.963
+stomach: 0.955
+pancreas: 0.894
+adrenal_gland_right: 0.904
+adrenal_gland_left: 0.907
+lung_upper_lobe_left: 0.968
+lung_lower_lobe_left: 0.979
+lung_upper_lobe_right: 0.893
+lung_middle_lobe_right: 0.966
+lung_lower_lobe_right: 0.975
+esophagus: 0.951
+trachea: 0.970
+thyroid_gland: 0.926
+small_bowel: 0.917
+duodenum: 0.861
+colon: 0.916
+urinary_bladder: 0.916
+prostate: 0.814
+kidney_cyst_left: 0.347
+kidney_cyst_right: 0.009
+sacrum: 0.940
+vertebrae_S1: 0.940
+vertebrae_L5: 0.984
+vertebrae_L4: 0.980
+vertebrae_L3: 0.978
+vertebrae_L2: 0.967
+vertebrae_L1: 0.948
+vertebrae_T12: 0.962
+vertebrae_T11: 0.955
+vertebrae_T10: 0.948
+vertebrae_T9: 0.965
+vertebrae_T8: 0.948
+vertebrae_T7: 0.931
+vertebrae_T6: 0.952
+vertebrae_T5: 0.971
+vertebrae_T4: 0.946
+vertebrae_T3: 0.956
+vertebrae_T2: 0.971
+vertebrae_T1: 0.982
+vertebrae_C7: 0.975
+vertebrae_C6: 0.890
+vertebrae_C5: 0.897
+vertebrae_C4: 0.884
+vertebrae_C3: 0.956
+vertebrae_C2: 0.915
+vertebrae_C1: 0.896
+heart: 0.904
+aorta: 0.982
+pulmonary_vein: 0.922
+brachiocephalic_trunk: 0.937
+subclavian_artery_right: 0.934
+subclavian_artery_left: 0.933
+common_carotid_artery_right: 0.904
+common_carotid_artery_left: 0.931
+brachiocephalic_vein_left: 0.934
+brachiocephalic_vein_right: 0.928
+atrial_appendage_left: 0.926
+superior_vena_cava: 0.943
+inferior_vena_cava: 0.934
+portal_vein_and_splenic_vein: 0.862
+iliac_artery_left: 0.936
+iliac_artery_right: 0.912
+iliac_vena_left: 0.918
+iliac_vena_right: 0.931
+humerus_left: 0.958
+humerus_right: 0.948
+scapula_left: 0.955
+scapula_right: 0.954
+clavicula_left: 0.957
+clavicula_right: 0.980
+femur_left: 0.966
+femur_right: 0.964
+hip_left: 0.989
+hip_right: 0.989
+spinal_cord: 0.951
+gluteus_maximus_left: 0.979
+gluteus_maximus_right: 0.982
+gluteus_medius_left: 0.977
+gluteus_medius_right: 0.978
+gluteus_minimus_left: 0.967
+gluteus_minimus_right: 0.971
+autochthon_left: 0.980
+autochthon_right: 0.979
+iliopsoas_left: 0.956
+iliopsoas_right: 0.968
+brain: 0.923
+skull: 0.816
+rib_left_1: 0.933
+rib_left_2: 0.931
+rib_left_3: 0.934
+rib_left_4: 0.919
+rib_left_5: 0.910
+rib_left_6: 0.934
+rib_left_7: 0.947
+rib_left_8: 0.937
+rib_left_9: 0.947
+rib_left_10: 0.946
+rib_left_11: 0.941
+rib_left_12: 0.927
+rib_right_1: 0.939
+rib_right_2: 0.946
+rib_right_3: 0.926
+rib_right_4: 0.944
+rib_right_5: 0.919
+rib_right_6: 0.938
+rib_right_7: 0.946
+rib_right_8: 0.955
+rib_right_9: 0.962
+rib_right_10: 0.967
+rib_right_11: 0.964
+rib_right_12: 0.938
+sternum: 0.906
+costal_cartilages: 0.919
+
+
+# surface_dice_3
+
+spleen: 0.987
+kidney_right: 0.957
+kidney_left: 0.962
+gallbladder: 0.919
+liver: 0.963
+stomach: 0.961
+pancreas: 0.959
+adrenal_gland_right: 0.988
+adrenal_gland_left: 0.993
+lung_upper_lobe_left: 0.973
+lung_lower_lobe_left: 0.980
+lung_upper_lobe_right: 0.928
+lung_middle_lobe_right: 0.971
+lung_lower_lobe_right: 0.976
+esophagus: 0.992
+trachea: 0.994
+thyroid_gland: 0.985
+small_bowel: 0.938
+duodenum: 0.930
+colon: 0.941
+urinary_bladder: 0.924
+prostate: 0.851
+kidney_cyst_left: 0.292
+kidney_cyst_right: 0.022
+sacrum: 0.966
+vertebrae_S1: 0.975
+vertebrae_L5: 0.996
+vertebrae_L4: 0.991
+vertebrae_L3: 0.990
+vertebrae_L2: 0.985
+vertebrae_L1: 0.965
+vertebrae_T12: 0.983
+vertebrae_T11: 0.972
+vertebrae_T10: 0.973
+vertebrae_T9: 0.984
+vertebrae_T8: 0.969
+vertebrae_T7: 0.962
+vertebrae_T6: 0.983
+vertebrae_T5: 0.988
+vertebrae_T4: 0.968
+vertebrae_T3: 0.975
+vertebrae_T2: 0.988
+vertebrae_T1: 0.998
+vertebrae_C7: 0.998
+vertebrae_C6: 0.998
+vertebrae_C5: 0.942
+vertebrae_C4: 0.918
+vertebrae_C3: 0.997
+vertebrae_C2: 0.990
+vertebrae_C1: 0.940
+heart: 0.926
+aorta: 0.997
+pulmonary_vein: 0.972
+brachiocephalic_trunk: 0.972
+subclavian_artery_right: 0.972
+subclavian_artery_left: 0.970
+common_carotid_artery_right: 0.964
+common_carotid_artery_left: 0.972
+brachiocephalic_vein_left: 0.973
+brachiocephalic_vein_right: 0.969
+atrial_appendage_left: 0.970
+superior_vena_cava: 0.969
+inferior_vena_cava: 0.976
+portal_vein_and_splenic_vein: 0.950
+iliac_artery_left: 0.986
+iliac_artery_right: 0.960
+iliac_vena_left: 0.972
+iliac_vena_right: 0.976
+humerus_left: 0.980
+humerus_right: 0.972
+scapula_left: 0.977
+scapula_right: 0.978
+clavicula_left: 0.972
+clavicula_right: 0.999
+femur_left: 0.974
+femur_right: 0.971
+hip_left: 0.999
+hip_right: 0.999
+spinal_cord: 0.993
+gluteus_maximus_left: 0.992
+gluteus_maximus_right: 0.994
+gluteus_medius_left: 0.999
+gluteus_medius_right: 0.995
+gluteus_minimus_left: 0.999
+gluteus_minimus_right: 0.999
+autochthon_left: 0.999
+autochthon_right: 0.997
+iliopsoas_left: 0.993
+iliopsoas_right: 1.000
+brain: 0.934
+skull: 0.849
+rib_left_1: 0.970
+rib_left_2: 0.964
+rib_left_3: 0.963
+rib_left_4: 0.942
+rib_left_5: 0.938
+rib_left_6: 0.968
+rib_left_7: 0.965
+rib_left_8: 0.961
+rib_left_9: 0.969
+rib_left_10: 0.971
+rib_left_11: 0.970
+rib_left_12: 0.966
+rib_right_1: 0.972
+rib_right_2: 0.973
+rib_right_3: 0.954
+rib_right_4: 0.969
+rib_right_5: 0.951
+rib_right_6: 0.961
+rib_right_7: 0.965
+rib_right_8: 0.975
+rib_right_9: 0.981
+rib_right_10: 0.987
+rib_right_11: 0.987
+rib_right_12: 0.974
+sternum: 0.953
+costal_cartilages: 0.981

resources/train_nnunet.md

@@ -6,7 +6,7 @@
 4. Preprocess `nnUNetv2_plan_and_preprocess -d <your_dataset_id> -pl ExperimentPlanner -c 3d_fullres -np 2`
 5. Train `nnUNetv2_train <your_dataset_id> 3d_fullres 0 -tr nnUNetTrainerNoMirroring` (takes several days)
 6. Predict test set `nnUNetv2_predict -i path/to/imagesTs -o path/to/labelsTs_predicted -d <your_dataset_id> -c 3d_fullres -tr nnUNetTrainerNoMirroring --disable_tta -f 0`
-7. Evaluate `python resources/evaluate.py path/to/labelsTs path/to/labelsTs_predicted`
+7. Evaluate `python resources/evaluate.py path/to/labelsTs path/to/labelsTs_predicted` (requires `pip install git+https://github.com/google-deepmind/surface-distance.git`). The resulting numbers should be similar to the ones in `resources/evaluate_results.txt`
 8. Done
 
 > Note: This will not give you the same results are TotalSegmentator for two reasons: