From f4617636a948a6d457f0ae253bc84bad2dd18cb6 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 5 Dec 2023 17:11:54 +0100 Subject: [PATCH] add options for spark-submit command recovering asteroid data --- fink_fat/command_line/association_cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fink_fat/command_line/association_cli.py b/fink_fat/command_line/association_cli.py index 4785a983..756f47c3 100644 --- a/fink_fat/command_line/association_cli.py +++ b/fink_fat/command_line/association_cli.py @@ -418,6 +418,10 @@ def get_last_roid_streaming_alert( --executor-memory {}G \ --conf spark.cores.max={} \ --conf spark.executor.cores={} \ + --conf spark.driver.maxResultSize=6G\ + --conf spark.sql.execution.arrow.pyspark.enabled=true\ + --conf spark.sql.execution.arrow.maxRecordsPerBatch=1000000\ + --conf spark.kryoserializer.buffer.max=512m\ {}".format( master_manager, principal_group,