[BUG] Apply create_partitions to historical validate (#303)

* Apply create_partitions to historical validate. * Remove comments and adjusts.
quintoandar · Mar 30, 2021 · 0d30932 · 0d30932
1 parent 3d93a09
commit 0d30932
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/butterfree/load/writers/historical_feature_store_writer.py b/butterfree/load/writers/historical_feature_store_writer.py
@@ -225,7 +225,6 @@ def validate(
         Raises:
             AssertionError: if count of written data doesn't match count in current
                 feature set dataframe.
-
         """
         table_name = (
             f"{feature_set.name}"
@@ -240,7 +239,9 @@ def validate(
         written_count = (
             spark_client.read(
                 self.db_config.format_,
-                path=self.db_config.get_path_with_partitions(table_name, dataframe),
+                path=self.db_config.get_path_with_partitions(
+                    table_name, self._create_partitions(dataframe)
+                ),
             ).count()
             if self.interval_mode and not self.debug_mode
             else spark_client.read_table(table_name).count()

diff --git a/setup.py b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import find_packages, setup
 
 __package_name__ = "butterfree"
-__version__ = "1.2.0.dev3"
+__version__ = "1.2.0.dev4"
 __repository_url__ = "https://github.com/quintoandar/butterfree"
 
 with open("requirements.txt") as f: