diff --git a/fink_broker/partitioning.py b/fink_broker/partitioning.py index 0eb0b95c..055e7896 100644 --- a/fink_broker/partitioning.py +++ b/fink_broker/partitioning.py @@ -142,9 +142,11 @@ def numPart(df, partition_size=128.): .builder \ .getOrCreate() + logical_plan = df._jdf.queryExecution().logical() + mode = df._jdf.queryExecution().mode() b = spark._jsparkSession\ .sessionState()\ - .executePlan(df._jdf.queryExecution().logical())\ + .executePlan(plan, mode)\ .optimizedPlan()\ .stats()\ .sizeInBytes()