From 7973cedaae9a95936dff6450649b958138f9a8d1 Mon Sep 17 00:00:00 2001 From: Csaba Date: Fri, 8 Sep 2023 17:30:05 +0200 Subject: [PATCH 1/8] Leaf region will only trigger if the region exists --- ...nhibitory_neuron_densities_optimization.py | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py index 450a3f9..8b13037 100644 --- a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py +++ b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py @@ -491,11 +491,29 @@ def _check_variables_consistency( if np.isfinite(deltas.loc[region_name, cell_type]): for desc_id in id_set: if np.isnan(x_result.loc[desc_id, cell_type]): - raise AtlasDensitiesError( - f"Cell count estimate of region named '{region_name}' for cell type " - f"{cell_type} was given for certain whereas the cell count of " - f"descendant id {desc_id} is not certain." - ) + if ((deltas.loc[region_name, cell_type] in [0.0, np.inf]) & + (x_result.loc[id_, cell_type] == 0.0) or + (np.isnan(x_result.loc[id_, cell_type]))): + ''' + If the region's cell count value was set to 0 because the region does + not exist we don't have to raise an error. Instead of x_result + volumes.loc[id_, 'volume'] == 0.0 would be a better condition.. ''' + deltas.loc[region_name, cell_type] = np.inf + x_result.loc[id_, cell_type] = np.nan + warnings.warn( + f"Cell count estimate of region named " + f"'{region_name}' for cell type {cell_type} was given 0 for " + f"its volume is 0 whereas the cell count of descendant id {desc_id} " + f"is not certain. Cell count estimate for this region is thus set " + f"to np.nan to avoid inconsistency.", + AtlasDensitiesWarning, + ) + + else: + raise AtlasDensitiesError( + f"Cell count estimate of region named '{region_name}' for cell type " + f"{cell_type} was given for certain whereas the cell count of " + ) neuron_count = neuron_counts.loc[id_, "cell_count"] if ( not np.isnan(x_result.loc[id_, cell_type]) From ea217f54992a564994634aac5e4485c794622260 Mon Sep 17 00:00:00 2001 From: Csaba Date: Mon, 11 Sep 2023 09:37:32 +0200 Subject: [PATCH 2/8] Added back the last line of the print message which was left out from the original AtlasDensitiesError() message --- .../densities/inhibitory_neuron_densities_optimization.py | 1 + 1 file changed, 1 insertion(+) diff --git a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py index 8b13037..e85c96c 100644 --- a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py +++ b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py @@ -513,6 +513,7 @@ def _check_variables_consistency( raise AtlasDensitiesError( f"Cell count estimate of region named '{region_name}' for cell type " f"{cell_type} was given for certain whereas the cell count of " + f"descendant id {desc_id} is not certain." ) neuron_count = neuron_counts.loc[id_, "cell_count"] if ( From e04e99ae116e137d257061426184a9ca44dbc617 Mon Sep 17 00:00:00 2001 From: Csaba Date: Mon, 11 Sep 2023 11:27:02 +0200 Subject: [PATCH 3/8] The function's info was incomplete, the last assumption was missing. --- .../densities/inhibitory_neuron_densities_optimization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py index e85c96c..405a0f3 100644 --- a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py +++ b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py @@ -481,7 +481,8 @@ def _check_variables_consistency( AtlasDensitiesError if on the the following assumptions is violated: - if cell count estimate of a region is known with certainty for a given cell type, then the cell count of every descendant region is also known with certainty. - - a cell count estimate which is given for certain does not + - a neuron subtype count estimate which is given for certain is greater than + its total neuron count estimate counterpart. """ cell_count_tolerance = 1e-2 # absolute tolerance to rule out round-off errors for region_name, id_, id_set in zip( From dfc369fc8c49d8889c00735f1b1a7d7a333cb6c7 Mon Sep 17 00:00:00 2001 From: Csaba Date: Tue, 12 Sep 2023 09:50:55 +0200 Subject: [PATCH 4/8] Changed format strings and inline strings, and & -> and, formatted brackets --- ...nhibitory_neuron_densities_optimization.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py index 405a0f3..54778f0 100644 --- a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py +++ b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py @@ -492,21 +492,20 @@ def _check_variables_consistency( if np.isfinite(deltas.loc[region_name, cell_type]): for desc_id in id_set: if np.isnan(x_result.loc[desc_id, cell_type]): - if ((deltas.loc[region_name, cell_type] in [0.0, np.inf]) & - (x_result.loc[id_, cell_type] == 0.0) or - (np.isnan(x_result.loc[id_, cell_type]))): - ''' - If the region's cell count value was set to 0 because the region does - not exist we don't have to raise an error. Instead of x_result - volumes.loc[id_, 'volume'] == 0.0 would be a better condition.. ''' + if ((deltas.loc[region_name, cell_type] in [0.0, np.inf]) and + (x_result.loc[id_, cell_type] == 0.0) or (np.isnan(x_result.loc[id_, cell_type])) + ): + # If the region's cell count value was set to 0 because the region does + # not exist we don't have to raise an error. Instead of x_result + # volumes.loc[id_, 'volume'] == 0.0 would be a better condition.. deltas.loc[region_name, cell_type] = np.inf x_result.loc[id_, cell_type] = np.nan warnings.warn( - f"Cell count estimate of region named " + "Cell count estimate of region named " f"'{region_name}' for cell type {cell_type} was given 0 for " f"its volume is 0 whereas the cell count of descendant id {desc_id} " - f"is not certain. Cell count estimate for this region is thus set " - f"to np.nan to avoid inconsistency.", + "is not certain. Cell count estimate for this region is thus set " + "to np.nan to avoid inconsistency.", AtlasDensitiesWarning, ) From 6c2bf01e86e20a9aedece3fc399bf15dcd2fd869 Mon Sep 17 00:00:00 2001 From: Csaba Date: Tue, 12 Sep 2023 10:30:17 +0200 Subject: [PATCH 5/8] Made the if statement more readable. --- .../densities/inhibitory_neuron_densities_optimization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py index 54778f0..f028540 100644 --- a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py +++ b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py @@ -493,7 +493,7 @@ def _check_variables_consistency( for desc_id in id_set: if np.isnan(x_result.loc[desc_id, cell_type]): if ((deltas.loc[region_name, cell_type] in [0.0, np.inf]) and - (x_result.loc[id_, cell_type] == 0.0) or (np.isnan(x_result.loc[id_, cell_type])) + ((x_result.loc[id_, cell_type] == 0.0) or (np.isnan(x_result.loc[id_, cell_type]))) ): # If the region's cell count value was set to 0 because the region does # not exist we don't have to raise an error. Instead of x_result From 578c8e9e74fdb4894953cb030645baa4aad9d515 Mon Sep 17 00:00:00 2001 From: Mike Gevaert Date: Thu, 14 Sep 2023 11:02:34 +0200 Subject: [PATCH 6/8] fix lint/ci errors --- ...nhibitory_neuron_densities_optimization.py | 20 ++++++++++--------- setup.py | 4 ++-- .../test_excitatory_inhibitory_splitting.py | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py index f028540..886d9ab 100644 --- a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py +++ b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py @@ -481,10 +481,11 @@ def _check_variables_consistency( AtlasDensitiesError if on the the following assumptions is violated: - if cell count estimate of a region is known with certainty for a given cell type, then the cell count of every descendant region is also known with certainty. - - a neuron subtype count estimate which is given for certain is greater than + - a neuron subtype count estimate which is given for certain is greater than its total neuron count estimate counterpart. """ cell_count_tolerance = 1e-2 # absolute tolerance to rule out round-off errors + # pylint: disable=too-many-nested-blocks for region_name, id_, id_set in zip( deltas.index, hierarchy_info.index, hierarchy_info["descendant_id_set"] ): @@ -492,10 +493,11 @@ def _check_variables_consistency( if np.isfinite(deltas.loc[region_name, cell_type]): for desc_id in id_set: if np.isnan(x_result.loc[desc_id, cell_type]): - if ((deltas.loc[region_name, cell_type] in [0.0, np.inf]) and - ((x_result.loc[id_, cell_type] == 0.0) or (np.isnan(x_result.loc[id_, cell_type]))) - ): - # If the region's cell count value was set to 0 because the region does + if (deltas.loc[region_name, cell_type] in [0.0, np.inf]) and ( + (x_result.loc[id_, cell_type] == 0.0) + or (np.isnan(x_result.loc[id_, cell_type])) + ): + # If the region's cell count value was set to 0 because the region does # not exist we don't have to raise an error. Instead of x_result # volumes.loc[id_, 'volume'] == 0.0 would be a better condition.. deltas.loc[region_name, cell_type] = np.inf @@ -503,15 +505,15 @@ def _check_variables_consistency( warnings.warn( "Cell count estimate of region named " f"'{region_name}' for cell type {cell_type} was given 0 for " - f"its volume is 0 whereas the cell count of descendant id {desc_id} " + f"its volume is 0 but the cell count of descendant id {desc_id} " "is not certain. Cell count estimate for this region is thus set " "to np.nan to avoid inconsistency.", AtlasDensitiesWarning, - ) - + ) + else: raise AtlasDensitiesError( - f"Cell count estimate of region named '{region_name}' for cell type " + f"Cell count estimate of region '{region_name}' for cell type " f"{cell_type} was given for certain whereas the cell count of " f"descendant id {desc_id} is not certain." ) diff --git a/setup.py b/setup.py index 9fbd57d..a9e69bf 100644 --- a/setup.py +++ b/setup.py @@ -20,9 +20,9 @@ "click>=7.0,<=8.1.3", "cgal-pybind>=0.1.1", "joblib>=1.3.0", - "numpy>=1.15.0", + "numpy>=1.15.0,<=1.24.0", # to reduce errors when pandas uses new numpy "openpyxl>=3.0.3", - "pandas>=1.0.3", + "pandas>=1.0.3,<=2.0.0", # because https://github.com/pandas-dev/pandas/pull/54954 has broken csv parsing "PyYAML>=5.3.1", # Since version 1.6.0, scipy.optimize.linprog has fast, new methods for large, sparse problems # from the HiGHS library. We use the "highs" method in the densities module. diff --git a/tests/densities/test_excitatory_inhibitory_splitting.py b/tests/densities/test_excitatory_inhibitory_splitting.py index b212da5..c5f220f 100644 --- a/tests/densities/test_excitatory_inhibitory_splitting.py +++ b/tests/densities/test_excitatory_inhibitory_splitting.py @@ -133,7 +133,7 @@ def test_make_excitatory_density(): res = tested.make_excitatory_density(neuron_density, inhibitory_density) assert res.shape == neuron_density.shape - assert np.sum(res.raw) == np.product(neuron_density.shape) + assert np.sum(res.raw) == np.prod(neuron_density.shape) # this would create negative densities; make sure they are clipped to zero res = tested.make_excitatory_density(inhibitory_density, neuron_density) From 94146bf1e673200f1c839c9d0edc2f12364e65a6 Mon Sep 17 00:00:00 2001 From: Csaba Date: Fri, 22 Sep 2023 14:28:29 +0200 Subject: [PATCH 7/8] We want to catch the variable inconsistency where it is created, not at the _sanity check function --- ...nhibitory_neuron_densities_optimization.py | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py index 886d9ab..7061c4a 100644 --- a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py +++ b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py @@ -130,6 +130,7 @@ def _zero_descendants(id_, cell_type, x_result, deltas, hierarchy_info): deltas.loc[desc_names, cell_type] = 0.0 L.info("Preparing variables layout ...") + #Create 2 df-s with only np.nans, next we will update this based on assumptions x_result = pd.DataFrame( {cell_type: np.full(len(id_counts), KEEP) for cell_type in cell_types}, index=id_counts.index, # indexed by integer identifiers @@ -157,6 +158,7 @@ def _zero_descendants(id_, cell_type, x_result, deltas, hierarchy_info): region_counts.at[region_name, "gad67+_standard_deviation"], 0.0, atol=stddev_tolerance ) and np.isclose(region_counts.at[region_name, "gad67+"], 0.0, atol=stddev_tolerance): for cell_type in cell_types: + #(Basically, if GAD is 0 then every other inh neuron type should be zero..) _zero_descendants(id_, cell_type, x_result, deltas, hierarchy_info) # Set the (possibly non-zero) cell count estimates which are given with certainty. @@ -178,6 +180,19 @@ def _zero_descendants(id_, cell_type, x_result, deltas, hierarchy_info): # are used as definitive estimates. x_result.at[id_, cell_type] = id_counts.at[id_, cell_type] deltas.at[region_name, cell_type] = 0.0 + # IF the region is not empty of leaf regions + # AND all its leaf regions are np.nan + # AND the region's cell count is 0: + # Revert standard deviation to np.inf i.e. var is omitted + # Since cell count i.e. literature value was changed to 0 we change it to np.nan + if ( + not x_result.loc[desc_only, cell_type].empty and + x_result.loc[desc_only, cell_type].isnull().all() and + x_result.loc[id_, cell_type] == 0 + ): + print(region_name, id_, cell_type, x_result.loc[id_, cell_type]) + deltas.at[region_name, cell_type] = SKIP + x_result.at[id_, cell_type] = np.nan return x_result, deltas @@ -493,30 +508,11 @@ def _check_variables_consistency( if np.isfinite(deltas.loc[region_name, cell_type]): for desc_id in id_set: if np.isnan(x_result.loc[desc_id, cell_type]): - if (deltas.loc[region_name, cell_type] in [0.0, np.inf]) and ( - (x_result.loc[id_, cell_type] == 0.0) - or (np.isnan(x_result.loc[id_, cell_type])) - ): - # If the region's cell count value was set to 0 because the region does - # not exist we don't have to raise an error. Instead of x_result - # volumes.loc[id_, 'volume'] == 0.0 would be a better condition.. - deltas.loc[region_name, cell_type] = np.inf - x_result.loc[id_, cell_type] = np.nan - warnings.warn( - "Cell count estimate of region named " - f"'{region_name}' for cell type {cell_type} was given 0 for " - f"its volume is 0 but the cell count of descendant id {desc_id} " - "is not certain. Cell count estimate for this region is thus set " - "to np.nan to avoid inconsistency.", - AtlasDensitiesWarning, - ) - - else: - raise AtlasDensitiesError( - f"Cell count estimate of region '{region_name}' for cell type " - f"{cell_type} was given for certain whereas the cell count of " - f"descendant id {desc_id} is not certain." - ) + raise AtlasDensitiesError( + f"Cell count estimate of region '{region_name}' for cell type " + f"{cell_type} was given for certain whereas the cell count of " + f"descendant id {desc_id} is not certain." + ) neuron_count = neuron_counts.loc[id_, "cell_count"] if ( not np.isnan(x_result.loc[id_, cell_type]) From 8659d6337e605fd05dfe452e2c135f6ec4bd5aa4 Mon Sep 17 00:00:00 2001 From: Csaba Date: Fri, 22 Sep 2023 14:31:57 +0200 Subject: [PATCH 8/8] Remove print message --- .../densities/inhibitory_neuron_densities_optimization.py | 1 - 1 file changed, 1 deletion(-) diff --git a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py index 7061c4a..5065985 100644 --- a/atlas_densities/densities/inhibitory_neuron_densities_optimization.py +++ b/atlas_densities/densities/inhibitory_neuron_densities_optimization.py @@ -190,7 +190,6 @@ def _zero_descendants(id_, cell_type, x_result, deltas, hierarchy_info): x_result.loc[desc_only, cell_type].isnull().all() and x_result.loc[id_, cell_type] == 0 ): - print(region_name, id_, cell_type, x_result.loc[id_, cell_type]) deltas.at[region_name, cell_type] = SKIP x_result.at[id_, cell_type] = np.nan