diff --git a/scripts/run-command b/scripts/run-command index b570d2c..87730fc 100755 --- a/scripts/run-command +++ b/scripts/run-command @@ -5,11 +5,12 @@ set -x python setup.py bdist_egg echo "from wikicast import __main__; __main__.cli()" > runner.py +ARROW_PRE_0_15_IPC_FORMAT=1 \ SPARK_HOME=$(python -c "import pyspark; print(pyspark.__path__[0])") \ spark-submit \ --master 'local[*]' \ - --conf spark.driver.memory=8g \ - --conf spark.sql.shuffle.partitions=8 \ + --driver-memory 24g \ + --conf spark.sql.shuffle.partitions=24 \ --packages \ graphframes:graphframes:0.7.0-spark2.4-s_2.11 \ --py-files "dist/*.egg" \ diff --git a/scripts/start-jupyter b/scripts/start-jupyter index 8d81cbe..e9aa7e5 100755 --- a/scripts/start-jupyter +++ b/scripts/start-jupyter @@ -7,7 +7,7 @@ PYSPARK_DRIVER_PYTHON=jupyter \ PYSPARK_DRIVER_PYTHON_OPTS=notebook \ pyspark \ --master 'local[*]' \ - --conf spark.driver.memory=8g \ - --conf spark.sql.shuffle.partitions=8 \ + --conf spark.driver.memory=24g \ + --conf spark.sql.shuffle.partitions=16 \ --packages \ graphframes:graphframes:0.7.0-spark2.4-s_2.11 diff --git a/scripts/start-jupyter.ps1 b/scripts/start-jupyter.ps1 index c2d4f3a..07dba6d 100644 --- a/scripts/start-jupyter.ps1 +++ b/scripts/start-jupyter.ps1 @@ -11,6 +11,6 @@ $env:PYSPARK_DRIVER_PYTHON_OPTS = "notebook" pyspark ` --master 'local[*]' ` --conf spark.driver.memory=8g ` - --conf spark.sql.shuffle.partitions=8 ` + --conf spark.sql.shuffle.partitions=16 ` --packages ` graphframes:graphframes:0.7.0-spark2.4-s_2.11