From 0d309327258c1a61f0d1aeb484da2f67f97b7a88 Mon Sep 17 00:00:00 2001 From: Mayara Moromisato <44944954+moromimay@users.noreply.github.com> Date: Tue, 30 Mar 2021 14:58:08 -0300 Subject: [PATCH] [BUG] Apply create_partitions to historical validate (#303) * Apply create_partitions to historical validate. * Remove comments and adjusts. --- butterfree/load/writers/historical_feature_store_writer.py | 5 +++-- setup.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/butterfree/load/writers/historical_feature_store_writer.py b/butterfree/load/writers/historical_feature_store_writer.py index e3e9b9b7..5defb00b 100644 --- a/butterfree/load/writers/historical_feature_store_writer.py +++ b/butterfree/load/writers/historical_feature_store_writer.py @@ -225,7 +225,6 @@ def validate( Raises: AssertionError: if count of written data doesn't match count in current feature set dataframe. - """ table_name = ( f"{feature_set.name}" @@ -240,7 +239,9 @@ def validate( written_count = ( spark_client.read( self.db_config.format_, - path=self.db_config.get_path_with_partitions(table_name, dataframe), + path=self.db_config.get_path_with_partitions( + table_name, self._create_partitions(dataframe) + ), ).count() if self.interval_mode and not self.debug_mode else spark_client.read_table(table_name).count() diff --git a/setup.py b/setup.py index 2dece452..a86ee049 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages, setup __package_name__ = "butterfree" -__version__ = "1.2.0.dev3" +__version__ = "1.2.0.dev4" __repository_url__ = "https://github.com/quintoandar/butterfree" with open("requirements.txt") as f: