use pandas NA

src/nested_pandas/series/dtype.py

@@ -13,7 +13,6 @@
 from pandas.core.arrays import ExtensionArray
 from pandas.core.dtypes.base import ExtensionDtype
 
-from nested_pandas.series.na import NA, NAType
 from nested_pandas.series.utils import is_pa_type_a_list
 
 __all__ = ["NestedDtype"]
@@ -29,9 +28,9 @@ class NestedDtype(ExtensionDtype):
     """Attributes to use as metadata for __eq__ and __hash__"""
 
     @property
-    def na_value(self) -> NAType:
+    def na_value(self) -> pd.NA:
         """The missing value for this dtype"""
-        return NA
+        return pd.NA
 
     type = pd.DataFrame
     """The type of the array's elements, always pd.DataFrame"""

src/nested_pandas/series/ext_array.py

@@ -160,8 +160,16 @@ def to_numpy(self, dtype: None = None, copy: bool = False, na_value: Any = no_de
 
         # Hack with np.empty is the only way to force numpy to create 1-d array of objects
         result = np.empty(shape=array.shape, dtype=object)
+
+        # TODO: delete block once confirmed NA is better to return
+        # fill NaNs - Fill with an empty dictionary using the same columns
+        # nan_vals = pd.isna(array)
+        # if sum(nan_vals) > 0:
+        #    na_idx = np.where(nan_vals)
+        #    array[na_idx] = {key: [] for key in array[np.where(~nan_vals)[0][0]].keys()}
+
         # We do copy=False here because user's 'copy' is already handled by ArrowExtensionArray.to_numpy
-        result[:] = [pd.DataFrame(value, copy=False) for value in array]
+        result[:] = [pd.DataFrame(value, copy=False) if not pd.isna(value) else pd.NA for value in array]
         return result
 
     def __setitem__(self, key, value) -> None:

tests/nested_pandas/nestedframe/test_nestedframe.py

@@ -78,6 +78,22 @@ def test_add_nested():
     assert base.nested.nest.to_flat().equals(nested)
 
 
+def test_add_nested_with_mismatched_index():
+    """Test add_nested when index values of base are missing matches in nested"""
+
+    base = NestedFrame(data={"a": [1, 2, 3], "b": [2, 4, 6]}, index=[0, 1, 2])
+
+    nested = pd.DataFrame(
+        data={"c": [0, 2, 4, 1, 4, 3, 1, 4, 1], "d": [5, 4, 7, 5, 3, 1, 9, 3, 4]},
+        index=[0, 0, 0, 1, 1, 1, 1, 1, 1],  # no data for index value of "2"
+    )
+
+    base = base.add_nested(nested, "nested")
+
+    assert "nested" in base.columns
+    assert pd.isna(base.loc[2]["nested"])
+
+
 def test_query():
     """Test that NestedFrame.query handles nested queries correctly"""
 

tests/nested_pandas/series/test_dtype.py

@@ -3,7 +3,6 @@
 import pytest
 from nested_pandas.series.dtype import NestedDtype
 from nested_pandas.series.ext_array import NestedExtensionArray
-from nested_pandas.series.na import NA
 
 
 @pytest.mark.parametrize(
@@ -62,7 +61,7 @@ def test_from_fields():
 def test_na_value():
     """Test that NestedDtype.na_value is a singleton instance of NAType."""
     dtype = NestedDtype(pa.struct([pa.field("a", pa.list_(pa.int64()))]))
-    assert dtype.na_value is NA
+    assert dtype.na_value is pd.NA
 
 
 def test_fields():