Add LGDO format conversion utilities

docs/source/conf.py

@@ -51,6 +51,7 @@
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),
     "numpy": ("https://numpy.org/doc/stable", None),
+    "awkward": ("https://awkward-array.org/doc/stable", None),
     "numba": ("https://numba.readthedocs.io/en/stable", None),
     "pandas": ("https://pandas.pydata.org/docs", None),
     "h5py": ("https://docs.h5py.org/en/stable", None),

setup.cfg

@@ -30,6 +30,8 @@ classifiers =
 [options]
 packages = find:
 install_requires =
+    awkward>=2
+    awkward-pandas
     colorlog
     h5py>=3.2
     hdf5plugin
@@ -39,6 +41,7 @@ install_requires =
     pandas>=1.4.4
     parse
     pint
+    pint-pandas
 python_requires = >=3.9
 include_package_data = True
 package_dir =

src/lgdo/compression/radware.py

@@ -129,7 +129,9 @@ def encode(
             )
         # convert VectorOfVectors to ArrayOfEqualSizedArrays so it can be
         # directly passed to the low-level encoding routine
-        sig_out_nda, nbytes = encode(sig_in.to_aoesa(), shift=shift)
+        sig_out_nda, nbytes = encode(
+            sig_in.to_aoesa(fill_val=0, preserve_dtype=True), shift=shift
+        )
 
         # build the encoded LGDO
         encoded_data = lgdo.ArrayOfEqualSizedArrays(nda=sig_out_nda).to_vov(
@@ -262,7 +264,7 @@ def decode(
         # convert vector of vectors to array of equal sized arrays
         # can now decode on the 2D matrix together with number of bytes to read per row
         _, siglen = decode(
-            (sig_in.encoded_data.to_aoesa(preserve_dtype=True).nda, nbytes),
+            (sig_in.encoded_data.to_aoesa(fill_val=0, preserve_dtype=True).nda, nbytes),
             sig_out if isinstance(sig_out, np.ndarray) else sig_out.nda,
             shift=shift,
         )
@@ -288,7 +290,8 @@ def decode(
         # convert vector of vectors to array of equal sized arrays
         # can now decode on the 2D matrix together with number of bytes to read per row
         sig_out, siglen = decode(
-            (sig_in.encoded_data.to_aoesa(preserve_dtype=True).nda, nbytes), shift=shift
+            (sig_in.encoded_data.to_aoesa(fill_val=0, preserve_dtype=True).nda, nbytes),
+            shift=shift,
         )
 
         # sanity check

src/lgdo/compression/varlen.py

@@ -103,7 +103,7 @@ def encode(
             )
         # convert VectorOfVectors to ArrayOfEqualSizedArrays so it can be
         # directly passed to the low-level encoding routine
-        sig_out_nda, nbytes = encode(sig_in.to_aoesa())
+        sig_out_nda, nbytes = encode(sig_in.to_aoesa(fill_val=0, preserve_dtype=True))
 
         # build the encoded LGDO
         encoded_data = lgdo.ArrayOfEqualSizedArrays(nda=sig_out_nda).to_vov(
@@ -227,7 +227,7 @@ def decode(
         # convert vector of vectors to array of equal sized arrays
         # can now decode on the 2D matrix together with number of bytes to read per row
         _, siglen = decode(
-            (sig_in.encoded_data.to_aoesa(preserve_dtype=True).nda, nbytes),
+            (sig_in.encoded_data.to_aoesa(fill_val=0, preserve_dtype=True).nda, nbytes),
             sig_out if isinstance(sig_out, np.ndarray) else sig_out.nda,
         )
 
@@ -252,7 +252,7 @@ def decode(
         # convert vector of vectors to array of equal sized arrays
         # can now decode on the 2D matrix together with number of bytes to read per row
         sig_out, siglen = decode(
-            (sig_in.encoded_data.to_aoesa(preserve_dtype=True).nda, nbytes)
+            (sig_in.encoded_data.to_aoesa(fill_val=0, preserve_dtype=True).nda, nbytes)
         )
 
         # sanity check

src/lgdo/lh5/store.py

@@ -53,7 +53,7 @@ class LH5Store:
     >>> store = LH5Store()
     >>> obj, _ = store.read("/geds/waveform", "file.lh5")
     >>> type(obj)
-    lgdo.waveform_table.WaveformTable
+    lgdo.waveformtable.WaveformTable
     """
 
     def __init__(self, base_path: str = "", keep_open: bool = False) -> None:
@@ -890,13 +890,13 @@ def write(
         `compression` attribute.
 
         Note
-        ----
+        ----------
         The `compression` LGDO attribute takes precedence over the default HDF5
         compression settings. The `hdf5_settings` attribute takes precedence
         over `compression`. These attributes are not written to disk.
 
         Note
-        ----
+        ----------
         HDF5 compression is skipped for the `encoded_data.flattened_data`
         dataset of :class:`.VectorOfEncodedVectors` and
         :class:`.ArrayOfEncodedEqualSizedArrays`.

src/lgdo/types/__init__.py

@@ -9,7 +9,7 @@
 from .struct import Struct
 from .table import Table
 from .vectorofvectors import VectorOfVectors
-from .waveform_table import WaveformTable
+from .waveformtable import WaveformTable
 
 __all__ = [
     "Array",

src/lgdo/types/array.py

@@ -8,9 +8,14 @@
 from collections.abc import Iterator
 from typing import Any
 
+import awkward as ak
+import awkward_pandas as akpd
 import numpy as np
+import pandas as pd
+import pint_pandas  # noqa: F401
 
 from .. import utils as utils
+from ..units import default_units_registry as u
 from .lgdo import LGDO
 
 log = logging.getLogger(__name__)
@@ -138,3 +143,58 @@ def __repr__(self) -> str:
             )
             + f", attrs={repr(self.attrs)})"
         )
+
+    def view_as(
+        self, library: str, with_units: bool = False
+    ) -> pd.DataFrame | np.NDArray | ak.Array:
+        """View the Array data as a third-party format data structure.
+
+        This is a zero-copy operation. Supported third-party formats are:
+
+        - ``pd``: returns a :class:`pandas.Series`
+        - ``np``: returns the internal `nda` attribute (:class:`numpy.ndarray`)
+        - ``ak``: returns an :class:`ak.Array` initialized with `self.nda`
+
+        Parameters
+        ----------
+        library
+            format of the returned data view.
+        with_units
+            forward physical units to the output data.
+
+        See Also
+        --------
+        .LGDO.view_as
+        """
+        # TODO: does attaching units imply a copy?
+        attach_units = with_units and "units" in self.attrs
+
+        if library == "pd":
+            if attach_units:
+                if self.nda.ndim == 1:
+                    return pd.Series(
+                        self.nda, dtype=f"pint[{self.attrs['units']}]", copy=False
+                    )
+                else:
+                    raise ValueError(
+                        "Pint does not support Awkward yet, you must view the data with_units=False"
+                    )
+            else:
+                if self.nda.ndim == 1:
+                    return pd.Series(self.nda, copy=False)
+                else:
+                    return akpd.from_awkward(self.view_as("ak"))
+        elif library == "np":
+            if attach_units:
+                return self.nda * u(self.attrs["units"])
+            else:
+                return self.nda
+        elif library == "ak":
+            if attach_units:
+                raise ValueError(
+                    "Pint does not support Awkward yet, you must view the data with_units=False"
+                )
+            else:
+                return ak.Array(self.nda)
+        else:
+            raise ValueError(f"{library} is not a supported third-party format.")

src/lgdo/types/arrayofequalsizedarrays.py

@@ -7,7 +7,9 @@
 from collections.abc import Iterator
 from typing import Any
 
+import awkward as ak
 import numpy as np
+import pandas as pd
 
 from .. import utils as utils
 from . import vectorofvectors as vov
@@ -131,3 +133,14 @@ def to_vov(self, cumulative_length: np.ndarray = None) -> vov.VectorOfVectors:
             cumulative_length=cumulative_length,
             attrs=attrs,
         )
+
+    def view_as(
+        self, library: str, with_units: bool = False
+    ) -> pd.DataFrame | np.NDArray | ak.Array:
+        """View the array as a third-party format data structure.
+
+        See Also
+        --------
+        .LGDO.view_as
+        """
+        return super().view_as(library, with_units=with_units)

src/lgdo/types/encoded.py

@@ -3,7 +3,10 @@
 from collections.abc import Iterator
 from typing import Any
 
+import awkward as ak
+import awkward_pandas as akpd
 import numpy as np
+import pandas as pd
 from numpy.typing import NDArray
 
 from .. import utils as utils
@@ -225,6 +228,62 @@ def __repr__(self) -> str:
         np.set_printoptions(**npopt)
         return out
 
+    def view_as(
+        self, library: str, with_units: bool = False
+    ) -> pd.DataFrame | np.NDArray | ak.Array:
+        """View the encoded data as a third-party format data structure.
+
+        This is a zero-copy or nearly zero-copy operation.
+
+        Supported third-party formats are:
+
+        - ``pd``: returns a :class:`pandas.DataFrame`
+        - ``ak``: returns an :class:`ak.Array` (record type)
+
+        Parameters
+        ----------
+        library
+            format of the returned data view.
+        with_units
+            forward physical units to the output data.
+
+        See Also
+        --------
+        .LGDO.view_as
+        """
+        attach_units = with_units and "units" in self.attrs
+
+        if library == "ak":
+            if attach_units:
+                raise ValueError(
+                    "Pint does not support Awkward yet, you must view the data with_units=False"
+                )
+
+            records_list = {
+                "encoded_data": self.encoded_data.view_as("ak"),
+                "decoded_size": np.array(self.decoded_size),
+            }
+            return ak.Array(records_list)
+
+        if library == "np":
+            raise TypeError(f"Format {library} is not a supported for voev.")
+        if library == "pd":
+            if attach_units:
+                raise ValueError(
+                    "Pint does not support Awkward yet, you must view the data with_units=False"
+                )
+            else:
+                return pd.DataFrame(
+                    {
+                        "encoded_data": akpd.from_awkward(
+                            self.encoded_data.view_as("ak")
+                        ),
+                        "decoded_size": self.decoded_size,
+                    }
+                )
+        else:
+            raise ValueError(f"{library} is not a supported third-party format.")
+
 
 class ArrayOfEncodedEqualSizedArrays(LGDO):
     """An array of encoded arrays with equal decoded size.
@@ -388,3 +447,68 @@ def __repr__(self) -> str:
         )
         np.set_printoptions(**npopt)
         return out
+
+    def view_as(
+        self, library: str, with_units: bool = False
+    ) -> pd.DataFrame | np.NDArray | ak.Array:
+        """View the encoded data as a third-party format data structure.
+
+        This is nearly a zero-copy operation.
+
+        Supported third-party formats are:
+
+        - ``pd``: returns a :class:`pandas.DataFrame`
+        - ``ak``: returns an :class:`ak.Array` (record type)
+
+        Note
+        ----
+        In the view, `decoded_size` is expanded into an array.
+
+        Parameters
+        ----------
+        library
+            format of the returned data view.
+        with_units
+            forward physical units to the output data.
+
+        See Also
+        --------
+        .LGDO.view_as
+        """
+        attach_units = with_units and "units" in self.attrs
+
+        if library == "ak":
+            if attach_units:
+                raise ValueError(
+                    "Pint does not support Awkward yet, you must view the data with_units=False"
+                )
+
+            records_list = {
+                "encoded_data": self.encoded_data.view_as("ak"),
+                "decoded_size": np.full(
+                    len(self.encoded_data.cumulative_length), self.decoded_size.value
+                ),
+            }
+            return ak.Array(records_list)
+
+        if library == "np":
+            raise TypeError(f"Format {library} is not a supported for aoeesa.")
+        if library == "pd":
+            if attach_units:
+                raise ValueError(
+                    "Pint does not support Awkward yet, you must view the data with_units=False"
+                )
+            else:
+                return pd.DataFrame(
+                    {
+                        "encoded_data": akpd.from_awkward(
+                            self.encoded_data.view_as("ak")
+                        ),
+                        "decoded_size": np.full(
+                            len(self.encoded_data.cumulative_length),
+                            self.decoded_size.value,
+                        ),
+                    }
+                )
+        else:
+            raise ValueError(f"{library} is not a supported third-party format.")

src/lgdo/types/fixedsizearray.py

@@ -6,7 +6,7 @@
 
 from typing import Any
 
-import numpy
+import numpy as np
 
 from .array import Array
 
@@ -24,9 +24,9 @@ class FixedSizeArray(Array):
 
     def __init__(
         self,
-        nda: numpy.ndarray = None,
+        nda: np.ndarray = None,
         shape: tuple[int, ...] = (),
-        dtype: numpy.dtype = None,
+        dtype: np.dtype = None,
         fill_val: int | float = None,
         attrs: dict[str, Any] = None,
     ) -> None:
@@ -41,3 +41,12 @@ def __init__(
 
     def datatype_name(self) -> str:
         return "fixedsize_array"
+
+    def view_as(self, library: str, with_units: bool = False):
+        """View the array as a third-party format data structure.
+
+        See Also
+        --------
+        .LGDO.view_as
+        """
+        return super.view_as(library, with_units=with_units)