fix: test

quintoandar · Aug 15, 2024 · 9036892 · 9036892
1 parent 86522fc
commit 9036892
Showing 1 changed file with 13 additions and 4 deletions.
diff --git a/tests/unit/butterfree/load/writers/test_historical_feature_store_writer.py b/tests/unit/butterfree/load/writers/test_historical_feature_store_writer.py
@@ -152,11 +152,20 @@ def test_merge_from_historical_writer(
         # given
         spark_client = SparkClient()
 
-        builder = pyspark.sql.SparkSession.builder \
-            .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \
-            .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+        # builder = pyspark.sql.SparkSession.builder \
+        #     .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \
+        #     .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+
+        # spark_client._session = configure_spark_with_delta_pip(builder).getOrCreate()
 
-        spark_client._session = configure_spark_with_delta_pip(builder).getOrCreate()
+        spark_client._session = (
+            SparkSession.builder.master("local[*]")
+            .config("spark.jars.packages", "io.delta:delta-core_2.12:2.0.0")
+            .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+            .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+            .config("spark.delta.logStore.class", "org.apache.spark.sql.delta.storage.S3SingleDriverLogStore")
+            .getOrCreate()
+        )
 
 
         spark_client.write_table = mocker.stub("write_table")