add driver_host to spark config

kbase · May 16, 2024 · cced892 · cced892
1 parent f9754a1
commit cced892
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 1 deletion.
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -62,4 +62,5 @@ services:
       - spark-master
     environment:
       - NOTEBOOK_PORT=4041
-      - SPARK_MASTER_URL=spark://spark-master:7077
+      - SPARK_MASTER_URL=spark://spark-master:7077
+      - SPARK_DRIVER_HOST=spark-notebook
diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh
@@ -2,6 +2,16 @@
 
 echo "starting jupyter notebook"
 
+if [ -n "$SPARK_DRIVER_HOST" ]; then
+    echo "Setting spark.driver.host to $SPARK_DRIVER_HOST"
+    source /opt/bitnami/scripts/spark-env.sh
+    if [ -z "$SPARK_CONF_FILE" ]; then
+        echo "Error: unable to find SPARK_CONF_FILE path"
+        exit 1
+    fi
+    echo "spark.driver.host $SPARK_DRIVER_HOST" >> $SPARK_CONF_FILE
+fi
+
 WORKSPACE_DIR="/cdm_shared_workspace"
 mkdir -p "$WORKSPACE_DIR"
 cd "$WORKSPACE_DIR"