Skip to content

Commit

Permalink
WIP: reduce meta handling
Browse files Browse the repository at this point in the history
  • Loading branch information
dougbrn committed Sep 27, 2024
1 parent f3b201e commit 29cb808
Showing 1 changed file with 13 additions and 0 deletions.
13 changes: 13 additions & 0 deletions src/nested_dask/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ def nested_columns(self) -> list:
nest_cols.append(column)
return nest_cols

# def map_partitions(self, *args, **kwargs) -> NestedFrame:
# """docstring"""
# res = super().map_partitions(*args, **kwargs)
# return res#.map_partitions(npd.NestedFrame, meta=npd.NestedFrame(res._meta.copy()))

def _is_known_hierarchical_column(self, colname) -> bool:
"""Determine whether a string is a known hierarchical column name"""
if "." in colname:
Expand Down Expand Up @@ -655,6 +660,14 @@ def reduce(self, func, *args, meta=None, **kwargs) -> NestedFrame:
"""

# Handle meta shorthands to produce nestedframe output
# route standard dict meta to nestedframe
if isinstance(meta, dict):
meta = npd.NestedFrame(meta, index=[])
# reroute series meta to nestedframe, per consistency with nested-pandas
elif isinstance(meta, tuple) and len(meta) == 2: # len 2 to only try on proper series meta
meta = npd.NestedFrame(pd.Series(name=meta[0], dtype=meta[1]).to_frame())

# apply nested_pandas reduce via map_partitions
# wrap the partition in a npd.NestedFrame call for:
# https://github.com/lincc-frameworks/nested-dask/issues/21
Expand Down

0 comments on commit 29cb808

Please sign in to comment.