fix(spark2|3): fix spark shell after setting yarn cluster mode by def…

…ault
TOSIT-IO · Nov 9, 2023 · 27d3479 · 27d3479
1 parent 77103c0
commit 27d3479
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 4 deletions.
diff --git a/roles/spark/common/templates/pyspark-command.j2 b/roles/spark/common/templates/pyspark-command.j2
@@ -2,4 +2,4 @@
 
 export SPARK_CONF_DIR={{ spark_client_conf_dir }}
 
-{{ spark_install_dir }}/bin/pyspark "$@"
+{{ spark_install_dir }}/bin/pyspark --deploy-mode client "$@"
diff --git a/roles/spark/common/templates/spark-shell-command.j2 b/roles/spark/common/templates/spark-shell-command.j2
@@ -2,4 +2,4 @@
 
 export SPARK_CONF_DIR={{ spark_client_conf_dir }}
 
-{{ spark_install_dir }}/bin/spark-shell "$@"
+{{ spark_install_dir }}/bin/spark-shell --deploy-mode client "$@"
diff --git a/roles/spark/common/templates/spark-sql-command.j2 b/roles/spark/common/templates/spark-sql-command.j2
@@ -2,4 +2,4 @@
 
 export SPARK_CONF_DIR={{ spark_client_conf_dir }}
 
-{{ spark_install_dir }}/bin/spark-sql "$@"
+{{ spark_install_dir }}/bin/spark-sql --deploy-mode client "$@"
diff --git a/roles/spark/common/templates/sparkr-shell-command.j2 b/roles/spark/common/templates/sparkr-shell-command.j2
@@ -4,4 +4,4 @@
 export YARN_CONF_DIR=/etc/hadoop/conf
 
 
-/opt/tdp/spark3/bin/sparkR "$@"
+/opt/tdp/spark3/bin/sparkR --deploy-mode client "$@"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,4 +2,4 @@

		export SPARK_CONF_DIR={{ spark_client_conf_dir }}

		{{ spark_install_dir }}/bin/pyspark "$@"
		{{ spark_install_dir }}/bin/pyspark --deploy-mode client "$@"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,4 +4,4 @@
		export YARN_CONF_DIR=/etc/hadoop/conf


		/opt/tdp/spark3/bin/sparkR "$@"
		/opt/tdp/spark3/bin/sparkR --deploy-mode client "$@"