From aecdb47dfce074f3dac8838d62ede07ffce35b3b Mon Sep 17 00:00:00 2001 From: JulienPeloton Date: Sun, 19 Nov 2023 21:28:25 +0100 Subject: [PATCH] Add explicitly mode in the executePlan args --- fink_broker/partitioning.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fink_broker/partitioning.py b/fink_broker/partitioning.py index 0eb0b95c..055e7896 100644 --- a/fink_broker/partitioning.py +++ b/fink_broker/partitioning.py @@ -142,9 +142,11 @@ def numPart(df, partition_size=128.): .builder \ .getOrCreate() + logical_plan = df._jdf.queryExecution().logical() + mode = df._jdf.queryExecution().mode() b = spark._jsparkSession\ .sessionState()\ - .executePlan(df._jdf.queryExecution().logical())\ + .executePlan(plan, mode)\ .optimizedPlan()\ .stats()\ .sizeInBytes()