From cced8926117e11f165bf2dbc3de46c74be05f2b6 Mon Sep 17 00:00:00 2001
From: Tianhao-Gu <tgu@anl.gov>
Date: Thu, 16 May 2024 15:46:57 -0500
Subject: [PATCH] add driver_host to spark config

---
 docker-compose.yaml   |  3 ++-
 scripts/entrypoint.sh | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 9e682c8..3dd4717 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -62,4 +62,5 @@ services:
       - spark-master
     environment:
       - NOTEBOOK_PORT=4041
-      - SPARK_MASTER_URL=spark://spark-master:7077
\ No newline at end of file
+      - SPARK_MASTER_URL=spark://spark-master:7077
+      - SPARK_DRIVER_HOST=spark-notebook
\ No newline at end of file
diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh
index 3a70178..277aeba 100644
--- a/scripts/entrypoint.sh
+++ b/scripts/entrypoint.sh
@@ -2,6 +2,16 @@
 
 echo "starting jupyter notebook"
 
+if [ -n "$SPARK_DRIVER_HOST" ]; then
+    echo "Setting spark.driver.host to $SPARK_DRIVER_HOST"
+    source /opt/bitnami/scripts/spark-env.sh
+    if [ -z "$SPARK_CONF_FILE" ]; then
+        echo "Error: unable to find SPARK_CONF_FILE path"
+        exit 1
+    fi
+    echo "spark.driver.host $SPARK_DRIVER_HOST" >> $SPARK_CONF_FILE
+fi
+
 WORKSPACE_DIR="/cdm_shared_workspace"
 mkdir -p "$WORKSPACE_DIR"
 cd "$WORKSPACE_DIR"