Skip to content

Commit

Permalink
Handle min_count=0 (#238)
Browse files Browse the repository at this point in the history
* Hande min_count=0

* fix
  • Loading branch information
dcherian authored May 1, 2023
1 parent 32f1ac3 commit aa358a5
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 4 deletions.
7 changes: 6 additions & 1 deletion flox/aggregations.py
Original file line number Diff line number Diff line change
Expand Up @@ -557,12 +557,15 @@ def _initialize_aggregation(
assert isinstance(finalize_kwargs, dict)
agg.finalize_kwargs = finalize_kwargs

if min_count is None:
min_count = 0

# This is needed for the dask pathway.
# Because we use intermediate fill_value since a group could be
# absent in one block, but present in another block
# We set it for numpy to get nansum, nanprod tests to pass
# where the identity element is 0, 1
if min_count is not None:
if min_count > 0:
agg.min_count = min_count
agg.chunk += ("nanlen",)
agg.numpy += ("nanlen",)
Expand All @@ -571,5 +574,7 @@ def _initialize_aggregation(
agg.fill_value["numpy"] += (0,)
agg.dtype["intermediate"] += (np.intp,)
agg.dtype["numpy"] += (np.intp,)
else:
agg.min_count = 0

return agg
11 changes: 8 additions & 3 deletions flox/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -849,7 +849,7 @@ def _finalize_results(
"""
squeezed = _squeeze_results(results, axis)

if agg.min_count is not None:
if agg.min_count > 0:
counts = squeezed["intermediates"][-1]
squeezed["intermediates"] = squeezed["intermediates"][:-1]

Expand All @@ -860,7 +860,7 @@ def _finalize_results(
else:
finalized[agg.name] = agg.finalize(*squeezed["intermediates"], **agg.finalize_kwargs)

if agg.min_count is not None:
if agg.min_count > 0:
count_mask = counts < agg.min_count
if count_mask.any():
# For one count_mask.any() prevents promoting bool to dtype(fill_value) unless
Expand Down Expand Up @@ -1917,7 +1917,12 @@ def groupby_reduce(
min_count = 1

# TODO: set in xarray?
if min_count is not None and func in ["nansum", "nanprod"] and fill_value is None:
if (
min_count is not None
and min_count > 0
and func in ["nansum", "nanprod"]
and fill_value is None
):
# nansum, nanprod have fill_value=0, 1
# overwrite than when min_count is set
fill_value = np.nan
Expand Down

0 comments on commit aa358a5

Please sign in to comment.