From ca6460a4e93283699621644d52adca55c6441eb3 Mon Sep 17 00:00:00 2001 From: Mohamed Mehdi BEN AISSA Date: Thu, 9 Nov 2023 11:49:53 +0100 Subject: [PATCH] fix(spark2|3): fix spark shell after setting yarn cluster mode by default --- roles/spark/common/templates/pyspark-command.j2 | 2 +- roles/spark/common/templates/spark-shell-command.j2 | 2 +- roles/spark/common/templates/spark-sql-command.j2 | 2 +- roles/spark/common/templates/sparkr-shell-command.j2 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/spark/common/templates/pyspark-command.j2 b/roles/spark/common/templates/pyspark-command.j2 index b74454d5..d84788a6 100644 --- a/roles/spark/common/templates/pyspark-command.j2 +++ b/roles/spark/common/templates/pyspark-command.j2 @@ -2,4 +2,4 @@ export SPARK_CONF_DIR={{ spark_client_conf_dir }} -{{ spark_install_dir }}/bin/pyspark "$@" +{{ spark_install_dir }}/bin/pyspark "$@" --deploy-mode client diff --git a/roles/spark/common/templates/spark-shell-command.j2 b/roles/spark/common/templates/spark-shell-command.j2 index 0e642cb3..b4fab465 100644 --- a/roles/spark/common/templates/spark-shell-command.j2 +++ b/roles/spark/common/templates/spark-shell-command.j2 @@ -2,4 +2,4 @@ export SPARK_CONF_DIR={{ spark_client_conf_dir }} -{{ spark_install_dir }}/bin/spark-shell "$@" +{{ spark_install_dir }}/bin/spark-shell "$@" --deploy-mode client diff --git a/roles/spark/common/templates/spark-sql-command.j2 b/roles/spark/common/templates/spark-sql-command.j2 index 414c9636..5422241a 100644 --- a/roles/spark/common/templates/spark-sql-command.j2 +++ b/roles/spark/common/templates/spark-sql-command.j2 @@ -2,4 +2,4 @@ export SPARK_CONF_DIR={{ spark_client_conf_dir }} -{{ spark_install_dir }}/bin/spark-sql "$@" +{{ spark_install_dir }}/bin/spark-sql "$@" --deploy-mode client diff --git a/roles/spark/common/templates/sparkr-shell-command.j2 b/roles/spark/common/templates/sparkr-shell-command.j2 index a35e0ca6..7cd2f33d 100644 --- a/roles/spark/common/templates/sparkr-shell-command.j2 +++ b/roles/spark/common/templates/sparkr-shell-command.j2 @@ -4,4 +4,4 @@ export YARN_CONF_DIR=/etc/hadoop/conf -/opt/tdp/spark3/bin/sparkR "$@" +/opt/tdp/spark3/bin/sparkR "$@" --deploy-mode client