Update scripts with memory tuning

acmiyaguchi · Dec 4, 2019 · 5c3dd24 · 5c3dd24
1 parent 13fa2c2
commit 5c3dd24
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 5 deletions.
diff --git a/scripts/run-command b/scripts/run-command
@@ -5,11 +5,12 @@ set -x
 python setup.py bdist_egg
 echo "from wikicast import __main__; __main__.cli()" > runner.py
 
+ARROW_PRE_0_15_IPC_FORMAT=1 \
 SPARK_HOME=$(python -c "import pyspark; print(pyspark.__path__[0])") \
 spark-submit \
     --master 'local[*]' \
-    --conf spark.driver.memory=8g \
-    --conf spark.sql.shuffle.partitions=8 \
+    --driver-memory 24g \
+    --conf spark.sql.shuffle.partitions=24 \
     --packages \
         graphframes:graphframes:0.7.0-spark2.4-s_2.11 \
     --py-files "dist/*.egg" \

diff --git a/scripts/start-jupyter b/scripts/start-jupyter
@@ -7,7 +7,7 @@ PYSPARK_DRIVER_PYTHON=jupyter \
 PYSPARK_DRIVER_PYTHON_OPTS=notebook \
 pyspark \
     --master 'local[*]' \
-    --conf spark.driver.memory=8g \
-    --conf spark.sql.shuffle.partitions=8 \
+    --conf spark.driver.memory=24g \
+    --conf spark.sql.shuffle.partitions=16 \
     --packages \
         graphframes:graphframes:0.7.0-spark2.4-s_2.11
diff --git a/scripts/start-jupyter.ps1 b/scripts/start-jupyter.ps1
@@ -11,6 +11,6 @@ $env:PYSPARK_DRIVER_PYTHON_OPTS = "notebook"
 pyspark `
     --master 'local[*]' `
     --conf spark.driver.memory=8g `
-    --conf spark.sql.shuffle.partitions=8 `
+    --conf spark.sql.shuffle.partitions=16 `
     --packages `
     graphframes:graphframes:0.7.0-spark2.4-s_2.11