From 27d3479bcc2736bb4ea6d73097ecde5201677c65 Mon Sep 17 00:00:00 2001 From: Mohamed Mehdi BEN AISSA Date: Thu, 9 Nov 2023 12:12:33 +0100 Subject: [PATCH] fix(spark2|3): fix spark shell after setting yarn cluster mode by default --- roles/spark/common/templates/pyspark-command.j2 | 2 +- roles/spark/common/templates/spark-shell-command.j2 | 2 +- roles/spark/common/templates/spark-sql-command.j2 | 2 +- roles/spark/common/templates/sparkr-shell-command.j2 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/spark/common/templates/pyspark-command.j2 b/roles/spark/common/templates/pyspark-command.j2 index b74454d5..32ba361f 100644 --- a/roles/spark/common/templates/pyspark-command.j2 +++ b/roles/spark/common/templates/pyspark-command.j2 @@ -2,4 +2,4 @@ export SPARK_CONF_DIR={{ spark_client_conf_dir }} -{{ spark_install_dir }}/bin/pyspark "$@" +{{ spark_install_dir }}/bin/pyspark --deploy-mode client "$@" diff --git a/roles/spark/common/templates/spark-shell-command.j2 b/roles/spark/common/templates/spark-shell-command.j2 index 0e642cb3..c715f561 100644 --- a/roles/spark/common/templates/spark-shell-command.j2 +++ b/roles/spark/common/templates/spark-shell-command.j2 @@ -2,4 +2,4 @@ export SPARK_CONF_DIR={{ spark_client_conf_dir }} -{{ spark_install_dir }}/bin/spark-shell "$@" +{{ spark_install_dir }}/bin/spark-shell --deploy-mode client "$@" diff --git a/roles/spark/common/templates/spark-sql-command.j2 b/roles/spark/common/templates/spark-sql-command.j2 index 414c9636..e3a9e4a4 100644 --- a/roles/spark/common/templates/spark-sql-command.j2 +++ b/roles/spark/common/templates/spark-sql-command.j2 @@ -2,4 +2,4 @@ export SPARK_CONF_DIR={{ spark_client_conf_dir }} -{{ spark_install_dir }}/bin/spark-sql "$@" +{{ spark_install_dir }}/bin/spark-sql --deploy-mode client "$@" diff --git a/roles/spark/common/templates/sparkr-shell-command.j2 b/roles/spark/common/templates/sparkr-shell-command.j2 index a35e0ca6..c1a1b13c 100644 --- a/roles/spark/common/templates/sparkr-shell-command.j2 +++ b/roles/spark/common/templates/sparkr-shell-command.j2 @@ -4,4 +4,4 @@ export YARN_CONF_DIR=/etc/hadoop/conf -/opt/tdp/spark3/bin/sparkR "$@" +/opt/tdp/spark3/bin/sparkR --deploy-mode client "$@"