Skip to content

Commit

Permalink
Merge pull request #939 from AFM-SPM/SylviaWhittle/800-tests-processing
Browse files Browse the repository at this point in the history 
Fix processing tests
  • Loading branch information
@SylviaWhittle
SylviaWhittle authored Oct 10, 2024
2 parents 3327404 + 583b6d1 commit f890e6d
Show file tree
Hide file tree
Showing 10 changed files with 166 additions and 58 deletions.
5 changes: 2 additions & 3 deletions tests/_regtest_outputs/test_processing.test_process_scan_below.out
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
image_size_x_m image_size_y_m image_area_m2 image_size_x_px image_size_y_px image_area_px2 grains_number_above grains_per_m2_above grains_number_below grains_per_m2_below rms_roughness
image
minicircle_small 1.2646e-07 1.2646e-07 1.5993e-14 64 64 4096 0 0.0000e+00 1 6.2526e+13 6.8208e-10
centre_x centre_y radius_min radius_max radius_mean radius_median height_min height_max height_median height_mean volume area area_cartesian_bbox smallest_bounding_width smallest_bounding_length smallest_bounding_area aspect_ratio threshold max_feret min_feret image contour_length circular end_to_end_distance
molecule_number
0 3.2366e-08 1.4036e-08 7.7690e-10 1.2272e-08 6.4301e-09 6.4170e-09 -3.7937e-10 -2.1207e-10 -2.4477e-10 -2.6816e-10 -3.0364e-26 1.1323e-16 3.0066e-16 7.0841e-09 2.1505e-08 1.5234e-16 3.2941e-01 below 2.2092e-08 7.0841e-09 minicircle_small NaN NaN NaN
centre_x centre_y grain_number radius_min radius_max radius_mean radius_median height_min height_max height_median height_mean volume area area_cartesian_bbox smallest_bounding_width smallest_bounding_length smallest_bounding_area aspect_ratio threshold max_feret min_feret image grain_endpoints grain_junctions total_branch_lengths num_crossings avg_crossing_confidence min_crossing_confidence num_mols total_contour_length average_end_to_end_distance
0 3.2366e-08 1.4036e-08 0 7.7690e-10 1.2272e-08 6.4301e-09 6.4170e-09 -3.7937e-10 -2.1207e-10 -2.4477e-10 -2.6816e-10 -3.0364e-26 1.1323e-16 3.0066e-16 7.0841e-09 2.1505e-08 1.5234e-16 3.2941e-01 below 2.2092e-08 7.0841e-09 minicircle_small 2 0 1.3493e+01 0 None None 1 1.0799e+01 1.0076e+01
11 changes: 5 additions & 6 deletions tests/_regtest_outputs/test_processing.test_process_scan_both.out
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
image_size_x_m image_size_y_m image_area_m2 image_size_x_px image_size_y_px image_area_px2 grains_number_above grains_per_m2_above grains_number_below grains_per_m2_below rms_roughness
image
minicircle_small 1.2646e-07 1.2646e-07 1.5993e-14 64 64 4096 3 1.8758e+14 1 6.2526e+13 6.8208e-10
centre_x centre_y radius_min radius_max radius_mean radius_median height_min height_max height_median height_mean volume area area_cartesian_bbox smallest_bounding_width smallest_bounding_length smallest_bounding_area aspect_ratio threshold max_feret min_feret image contour_length circular end_to_end_distance
molecule_number
0 3.2366e-08 1.4036e-08 7.7690e-10 1.2272e-08 6.4301e-09 6.4170e-09 -3.7937e-10 -2.1207e-10 -2.4477e-10 -2.6816e-10 -3.0364e-26 1.1323e-16 3.0066e-16 7.0841e-09 2.1505e-08 1.5234e-16 3.2941e-01 below 2.2092e-08 7.0841e-09 minicircle_small NaN NaN NaN
0 7.5100e-08 4.7559e-08 3.9431e-09 2.5631e-08 1.6016e-08 1.6680e-08 9.1991e-10 2.6422e-09 1.5338e-09 1.5341e-09 1.0543e-24 6.8721e-16 1.3198e-15 2.0539e-08 5.0379e-08 1.0347e-15 4.0769e-01 above 5.0379e-08 2.0539e-08 minicircle_small 6.0226e-08 0.0000e+00 8.6738e-09
1 8.0241e-08 7.8677e-08 6.8951e-09 2.7188e-08 1.6272e-08 1.6263e-08 9.0630e-10 2.4586e-09 1.6144e-09 1.6264e-09 1.0352e-24 6.3645e-16 1.5931e-15 2.0174e-08 5.1212e-08 1.0332e-15 3.9394e-01 above 5.1262e-08 2.0174e-08 minicircle_small 6.6355e-08 1.0000e+00 0.0000e+00
2 4.0012e-08 7.5644e-08 9.9461e-09 2.3654e-08 1.7561e-08 1.8364e-08 9.0641e-10 2.1066e-09 1.5939e-09 1.5493e-09 1.1192e-24 7.2236e-16 1.5462e-15 3.3592e-08 4.1496e-08 1.3940e-15 8.0952e-01 above 4.4405e-08 3.2528e-08 minicircle_small 9.6106e-08 1.0000e+00 0.0000e+00
centre_x centre_y grain_number radius_min radius_max radius_mean radius_median height_min height_max height_median height_mean volume area area_cartesian_bbox smallest_bounding_width smallest_bounding_length smallest_bounding_area aspect_ratio threshold max_feret min_feret image grain_endpoints grain_junctions total_branch_lengths num_crossings avg_crossing_confidence min_crossing_confidence num_mols writhe_string total_contour_length average_end_to_end_distance
0 3.2366e-08 1.4036e-08 0 7.7690e-10 1.2272e-08 6.4301e-09 6.4170e-09 -3.7937e-10 -2.1207e-10 -2.4477e-10 -2.6816e-10 -3.0364e-26 1.1323e-16 3.0066e-16 7.0841e-09 2.1505e-08 1.5234e-16 3.2941e-01 below 2.2092e-08 7.0841e-09 minicircle_small 2 0 1.3493e+01 0 None None 1 NaN 1.0799e+01 1.0076e+01
1 7.5100e-08 4.7559e-08 0 3.9431e-09 2.5631e-08 1.6016e-08 1.6680e-08 9.1991e-10 2.6422e-09 1.5338e-09 1.5341e-09 1.0543e-24 6.8721e-16 1.3198e-15 2.0539e-08 5.0379e-08 1.0347e-15 4.0769e-01 above 5.0379e-08 2.0539e-08 minicircle_small 1 1 8.4571e+01 1 None None 2 6.5881e+01 8.8370e+00
2 8.0241e-08 7.8677e-08 1 6.8951e-09 2.7188e-08 1.6272e-08 1.6263e-08 9.0630e-10 2.4586e-09 1.6144e-09 1.6264e-09 1.0352e-24 6.3645e-16 1.5931e-15 2.0174e-08 5.1212e-08 1.0332e-15 3.9394e-01 above 5.1262e-08 2.0174e-08 minicircle_small 0 0 7.3054e+01 0 None None 1 NaN 5.8272e+01 0.0000e+00
3 4.0012e-08 7.5644e-08 2 9.9461e-09 2.3654e-08 1.7561e-08 1.8364e-08 9.0641e-10 2.1066e-09 1.5939e-09 1.5493e-09 1.1192e-24 7.2236e-16 1.5462e-15 3.3592e-08 4.1496e-08 1.3940e-15 8.0952e-01 above 4.4405e-08 3.2528e-08 minicircle_small 0 0 1.0447e+02 0 None None 1 NaN 8.7183e+01 0.0000e+00
Binary file modified tests/resources/process_scan_expected_below_height_profiles.pickle
View file
Binary file not shown.
Binary file modified tests/resources/process_scan_topostats_file_regtest.topostats
View file
Binary file not shown.
66 changes: 66 additions & 0 deletions tests/test_grains.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,64 @@ def test_remove_small_objects():
np.testing.assert_array_equal(result, expected)


@pytest.mark.parametrize(
("binary_image", "minimum_size_px", "minimum_bbox_size_px", "expected_image"),
[
pytest.param(
np.array(
[
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
[0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
[0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0],
[0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0],
[0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0],
[0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
]
),
8,
4,
np.array(
[
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
]
),
)
],
)
def test_remove_objects_too_small_to_process(
binary_image: npt.NDArray, minimum_size_px: int, minimum_bbox_size_px: int, expected_image: npt.NDArray
) -> None:
"""Test the remove_objects_too_small_to_process method of the Grains class."""
grains_object = Grains(
image=np.array([[0, 0], [0, 0]]),
filename="",
pixel_to_nm_scaling=1.0,
)

result = grains_object.remove_objects_too_small_to_process(
image=binary_image, minimum_size_px=minimum_size_px, minimum_bbox_size_px=minimum_bbox_size_px
)

np.testing.assert_array_equal(result, expected_image)


@pytest.mark.parametrize(
("test_labelled_image", "area_thresholds", "expected"),
[
Expand Down Expand Up @@ -383,6 +441,10 @@ def test_find_grains(
remove_edge_intersecting_grains=remove_edge_intersecting_grains,
)

# Override grains' minimum grain size just for this test to allow for small grains in the test image
grains_object.minimum_grain_size_px = 1
grains_object.minimum_bbox_size_px = 1

grains_object.find_grains()

result_removed_small_objects = grains_object.directions[direction]["removed_small_objects"]
Expand Down Expand Up @@ -543,6 +605,10 @@ def test_find_grains_unet(
remove_edge_intersecting_grains=True,
)

# Override grains' minimum grain size just for this test to allow for small grains in the test image
grains_object.minimum_grain_size_px = 1
grains_object.minimum_bbox_size_px = 1

grains_object.find_grains()

result_removed_small_objects = grains_object.directions["above"]["removed_small_objects"]
Expand Down
57 changes: 14 additions & 43 deletions tests/test_processing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,16 @@ def test_process_scan_below_height_profiles(tmp_path, process_scan_config: dict,

process_scan_config["grains"]["direction"] = "below"
img_dic = load_scan_data.img_dict
_, _, _, height_profiles = process_scan(
_, _, height_profiles, _, _, _ = process_scan(
topostats_object=img_dic["minicircle_small"],
base_dir=BASE_DIR,
filter_config=process_scan_config["filter"],
grains_config=process_scan_config["grains"],
grainstats_config=process_scan_config["grainstats"],
dnatracing_config=process_scan_config["dnatracing"],
disordered_tracing_config=process_scan_config["disordered_tracing"],
nodestats_config=process_scan_config["nodestats"],
ordered_tracing_config=process_scan_config["ordered_tracing"],
splining_config=process_scan_config["splining"],
plotting_config=process_scan_config["plotting"],
output_dir=tmp_path,
)
Expand Down Expand Up @@ -119,13 +122,16 @@ def test_process_scan_above_height_profiles(tmp_path, process_scan_config: dict,
process_scan_config["grains"]["absolute_area_threshold"]["below"] = [1, 1000000000]

img_dic = load_scan_data.img_dict
_, _, _, height_profiles = process_scan(
_, _, height_profiles, _, _, _ = process_scan(
topostats_object=img_dic["minicircle_small"],
base_dir=BASE_DIR,
filter_config=process_scan_config["filter"],
grains_config=process_scan_config["grains"],
grainstats_config=process_scan_config["grainstats"],
dnatracing_config=process_scan_config["dnatracing"],
disordered_tracing_config=process_scan_config["disordered_tracing"],
nodestats_config=process_scan_config["nodestats"],
ordered_tracing_config=process_scan_config["ordered_tracing"],
splining_config=process_scan_config["splining"],
plotting_config=process_scan_config["plotting"],
output_dir=tmp_path,
)
Expand Down Expand Up @@ -524,8 +530,8 @@ def test_check_run_steps(
False,
False,
False,
"Detection of grains disabled, returning empty data frame.",
"minicircle_small.png",
"Detection of grains disabled, GrainStats will not be run.",
"",
id="Only filtering enabled",
),
pytest.param(
Expand All @@ -534,7 +540,7 @@ def test_check_run_steps(
False,
False,
"Calculation of grainstats disabled, returning empty dataframe and empty height_profiles.",
"minicircle_small_above_masked.png",
"",
id="Filtering and Grain enabled",
),
pytest.param(
Expand Down Expand Up @@ -621,41 +627,6 @@ def test_process_scan_no_grains(process_scan_config: dict, load_scan_data: LoadS
assert "No grains exist for the above direction. Skipping grainstats for above." in caplog.text


def test_process_scan_align_grainstats_dnatracing(
process_scan_config: dict, load_scan_data: LoadScans, tmp_path: Path
) -> None:
"""Ensure molecule numbers from dnatracing align with those from grainstats.
Sometimes grains are removed from tracing due to small size, however we need to ensure that tracing statistics for
those molecules that remain align with grain statistics.
By setting processing parameters as below two molecules are purged for being too small after skeletonisation and so
do not have DNA tracing statistics (but they do have Grain Statistics).
"""
img_dic = load_scan_data.img_dict
process_scan_config["filter"]["remove_scars"]["run"] = False
process_scan_config["grains"]["absolute_area_threshold"]["above"] = [150, 3000]
process_scan_config["dnatracing"]["min_skeleton_size"] = 50
_, results, _, _, _, _ = process_scan(
topostats_object=img_dic["minicircle_small"],
base_dir=BASE_DIR,
filter_config=process_scan_config["filter"],
grains_config=process_scan_config["grains"],
grainstats_config=process_scan_config["grainstats"],
disordered_tracing_config=process_scan_config["disordered_tracing"],
nodestats_config=process_scan_config["nodestats"],
ordered_tracing_config=process_scan_config["ordered_tracing"],
splining_config=process_scan_config["splining"],
plotting_config=process_scan_config["plotting"],
output_dir=tmp_path,
)
tracing_to_check = ["contour_length", "circular", "end_to_end_distance"]

assert results.shape == (3, 25)
assert np.isnan(results.loc[2, "contour_length"])
assert np.isnan(sum(results.loc[2, tracing_to_check]))


def test_run_filters(process_scan_config: dict, load_scan_data: LoadScans, tmp_path: Path) -> None:
"""Test the filter_wrapper function of processing.py."""
img_dict = load_scan_data.img_dict
Expand Down Expand Up @@ -762,7 +733,7 @@ def test_run_grainstats(process_scan_config: dict, tmp_path: Path) -> None:

assert isinstance(grainstats_df, pd.DataFrame)
assert grainstats_df.shape[0] == 13
assert len(grainstats_df.columns) == 21
assert len(grainstats_df.columns) == 22


# ns-rse 2024-09-11 : Test disabled as run_dnatracing() has been removed in refactoring, needs updating/replacing to
Expand Down
49 changes: 48 additions & 1 deletion topostats/grains.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,11 @@ def __init__(
self.grainstats = None
self.unet_config = unet_config

# Hardcoded minimum pixel size for grains. This should not be able to be changed by the user as this is
# determined by what is processable by the rest of the pipeline.
self.minimum_grain_size_px = 10
self.minimum_bbox_size_px = 5

def tidy_border(self, image: npt.NDArray, **kwargs) -> npt.NDArray:
"""
Remove grains touching the border.
Expand Down Expand Up @@ -295,6 +300,41 @@ def remove_small_objects(self, image: np.array, **kwargs) -> npt.NDArray:
return small_objects_removed > 0.0
return image

def remove_objects_too_small_to_process(
self, image: npt.NDArray, minimum_size_px: int, minimum_bbox_size_px: int
) -> npt.NDArray[np.bool_]:
"""
Remove objects whose dimensions in pixels are too small to process.
Parameters
----------
image : npt.NDArray
2-D Numpy array of image.
minimum_size_px : int
Minimum number of pixels for an object.
minimum_bbox_size_px : int
Limit for the minimum dimension of an object in pixels. Eg: 5 means the object's bounding box must be at
least 5x5.
Returns
-------
npt.NDArray
2-D Numpy array of image with objects removed that are too small to process.
"""
labelled_image = label(image)
region_properties = self.get_region_properties(labelled_image)
for region in region_properties:
# If the number of true pixels in the region is less than the minimum number of pixels, remove the region
if region.area < minimum_size_px:
labelled_image[labelled_image == region.label] = 0
bbox_width = region.bbox[2] - region.bbox[0]
bbox_height = region.bbox[3] - region.bbox[1]
# If the minimum dimension of the bounding box is less than the minimum dimension, remove the region
if min(bbox_width, bbox_height) < minimum_bbox_size_px:
labelled_image[labelled_image == region.label] = 0

return labelled_image.astype(bool)

def area_thresholding(self, image: npt.NDArray, area_thresholds: tuple) -> npt.NDArray:
"""
Remove objects larger and smaller than the specified thresholds.
Expand Down Expand Up @@ -440,8 +480,15 @@ def find_grains(self):
self.directions[direction]["removed_noise"],
self.absolute_area_threshold[direction],
)
self.directions[direction]["removed_objects_too_small_to_process"] = (
self.remove_objects_too_small_to_process(
image=self.directions[direction]["removed_small_objects"],
minimum_size_px=self.minimum_grain_size_px,
minimum_bbox_size_px=self.minimum_bbox_size_px,
)
)
self.directions[direction]["labelled_regions_02"] = self.label_regions(
self.directions[direction]["removed_small_objects"]
self.directions[direction]["removed_objects_too_small_to_process"]
)

self.region_properties[direction] = self.get_region_properties(
Expand Down
Loading

0 comments on commit f890e6d

Please sign in to comment.