From e2ce6760eaaa1c883c184117222e0f13b95d6d9a Mon Sep 17 00:00:00 2001
From: Chris Mutel <cmutel@gmail.com>
Date: Wed, 3 Feb 2021 22:11:34 +0100
Subject: [PATCH] Make proxied objects on package creation default false

And never proxy in-memory FSes
---
 bw_processing/datapackage.py | 57 ++++++++++++++++++++++++++----------
 tests/integration.py         |  1 -
 2 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/bw_processing/datapackage.py b/bw_processing/datapackage.py
index ad1c3f5..89a7366 100644
--- a/bw_processing/datapackage.py
+++ b/bw_processing/datapackage.py
@@ -221,9 +221,7 @@ def _create(
         name = clean_datapackage_name(name or uuid.uuid4().hex)
         check_name(name)
 
-        if fs is None:
-            fs = MemoryFS()
-        self.fs = fs
+        self.fs = fs or MemoryFS()
 
         # if dirpath is None:
         #     self.io_obj = InMemoryIO()
@@ -269,6 +267,8 @@ def _substitute_interfaces(self) -> None:
     def finalize_serialization(self) -> None:
         if self._finalized:
             raise Closed("Datapackage already finalized")
+        elif isinstance(self.fs, MemoryFS):
+            raise ValueError("In-memory file systems can't be serialized")
 
         self._substitute_interfaces()
         self._check_length_consistency()
@@ -338,6 +338,7 @@ def add_persistent_vector(
         data_array: Union[np.ndarray, None] = None,
         flip_array: Union[np.ndarray, None] = None,
         distributions_array: Union[np.ndarray, None] = None,
+        keep_proxy: bool = False,
         **kwargs,
     ) -> None:
         """"""
@@ -355,6 +356,7 @@ def add_persistent_vector(
             name=name + ".indices",
             group=name,
             kind="indices",
+            keep_proxy=keep_proxy,
             **kwargs,
         )
         if data_array is not None:
@@ -363,6 +365,7 @@ def add_persistent_vector(
                 group=name,
                 name=name + ".data",
                 kind="data",
+                keep_proxy=keep_proxy,
                 **kwargs,
             )
         if distributions_array is not None:
@@ -375,6 +378,7 @@ def add_persistent_vector(
                     name=name + ".distributions",
                     group=name,
                     kind="distributions",
+                    keep_proxy=keep_proxy,
                     **kwargs,
                 )
         if flip_array is not None and flip_array.sum():
@@ -383,6 +387,7 @@ def add_persistent_vector(
                 group=name,
                 name=name + ".flip",
                 kind="flip",
+                keep_proxy=keep_proxy,
                 **kwargs,
             )
 
@@ -394,6 +399,7 @@ def add_persistent_array(
         indices_array: np.ndarray,
         name: Union[str, None] = None,
         flip_array: Union[None, np.ndarray] = None,
+        keep_proxy: bool = False,
         **kwargs,
     ) -> None:
         """"""
@@ -409,6 +415,7 @@ def add_persistent_array(
             name=name + ".data",
             group=name,
             kind="data",
+            keep_proxy=keep_proxy,
             **kwargs,
         )
         self._add_numpy_array_resource(
@@ -416,6 +423,7 @@ def add_persistent_array(
             name=name + ".indices",
             kind="indices",
             group=name,
+            keep_proxy=keep_proxy,
             **kwargs,
         )
         if flip_array is not None and flip_array.sum():
@@ -424,29 +432,42 @@ def add_persistent_array(
                 group=name,
                 name=name + ".flip",
                 kind="flip",
+                keep_proxy=keep_proxy,
                 **kwargs,
             )
 
     def _add_numpy_array_resource(
-        self, *, array: np.ndarray, name: str, matrix: str, kind: str, **kwargs
+        self,
+        *,
+        array: np.ndarray,
+        name: str,
+        matrix: str,
+        kind: str,
+        keep_proxy: bool = False,
+        **kwargs,
     ) -> None:
         filename = check_suffix(name, ".npy")
 
-        file_writer(
-            data=array,
-            fs=self.fs,
-            resource=filename,
-            mimetype="application/octet-stream",
-        )
-        self.data.append(
-            file_reader(
+        if not isinstance(self.fs, MemoryFS):
+            file_writer(
+                data=array,
                 fs=self.fs,
                 resource=filename,
                 mimetype="application/octet-stream",
-                proxy=True,
-                **kwargs,
             )
-        )
+
+        if keep_proxy:
+            self.data.append(
+                file_reader(
+                    fs=self.fs,
+                    resource=filename,
+                    mimetype="application/octet-stream",
+                    proxy=True,
+                    **kwargs,
+                )
+            )
+        else:
+            self.data.append(array)
 
         resource = {
             # Datapackage generic
@@ -546,6 +567,7 @@ def add_dynamic_vector(
         indices_array: np.ndarray,  # Not interface
         name: Union[str, None] = None,
         flip_array: Union[None, np.ndarray] = None,  # Not interface
+        keep_proxy: bool = False,
         **kwargs,
     ) -> None:
         self._prepare_modifications()
@@ -562,6 +584,7 @@ def add_dynamic_vector(
             name=name + ".indices",
             group=name,
             kind="indices",
+            keep_proxy=keep_proxy,
             **kwargs,
         )
         if flip_array is not None and flip_array.sum():
@@ -570,6 +593,7 @@ def add_dynamic_vector(
                 group=name,
                 name=name + ".flip",
                 kind="flip",
+                keep_proxy=keep_proxy,
                 **kwargs,
             )
 
@@ -591,6 +615,7 @@ def add_dynamic_array(
         indices_array: np.ndarray,  # Not interface
         name: Union[str, None] = None,
         flip_array: Union[None, np.ndarray] = None,
+        keep_proxy: bool = False,
         **kwargs,
     ) -> None:
         """`interface` must support the presamples API."""
@@ -609,6 +634,7 @@ def add_dynamic_array(
             name=name + ".indices",
             group=name,
             kind="indices",
+            keep_proxy=keep_proxy,
             **kwargs,
         )
         if flip_array is not None:
@@ -617,6 +643,7 @@ def add_dynamic_array(
                 group=name,
                 name=name + ".flip",
                 kind="flip",
+                keep_proxy=keep_proxy,
                 **kwargs,
             )
 
diff --git a/tests/integration.py b/tests/integration.py
index aa8377b..6498686 100644
--- a/tests/integration.py
+++ b/tests/integration.py
@@ -340,7 +340,6 @@ def test_integration_test_in_memory():
     dp = create_datapackage(fs=None, name="test-fixture", id_="fixture-42")
     assert isinstance(dp.fs, MemoryFS)
     add_data(dp)
-    dp.finalize_serialization()
 
     check_metadata(dp)
     check_data(dp)