add options for spark-submit command recovering asteroid data

FusRoman · Dec 5, 2023 · f461763 · f461763
1 parent 8ff8f37
commit f461763
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/fink_fat/command_line/association_cli.py b/fink_fat/command_line/association_cli.py
@@ -418,6 +418,10 @@ def get_last_roid_streaming_alert(
             --executor-memory {}G \
             --conf spark.cores.max={} \
             --conf spark.executor.cores={} \
+            --conf spark.driver.maxResultSize=6G\
+            --conf spark.sql.execution.arrow.pyspark.enabled=true\
+            --conf spark.sql.execution.arrow.maxRecordsPerBatch=1000000\
+            --conf spark.kryoserializer.buffer.max=512m\
             {}".format(
             master_manager,
             principal_group,