diff --git a/docker-compose.yaml b/docker-compose.yaml index 9e682c8..3dd4717 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -62,4 +62,5 @@ services: - spark-master environment: - NOTEBOOK_PORT=4041 - - SPARK_MASTER_URL=spark://spark-master:7077 \ No newline at end of file + - SPARK_MASTER_URL=spark://spark-master:7077 + - SPARK_DRIVER_HOST=spark-notebook \ No newline at end of file diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 3a70178..277aeba 100644 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -2,6 +2,16 @@ echo "starting jupyter notebook" +if [ -n "$SPARK_DRIVER_HOST" ]; then + echo "Setting spark.driver.host to $SPARK_DRIVER_HOST" + source /opt/bitnami/scripts/spark-env.sh + if [ -z "$SPARK_CONF_FILE" ]; then + echo "Error: unable to find SPARK_CONF_FILE path" + exit 1 + fi + echo "spark.driver.host $SPARK_DRIVER_HOST" >> $SPARK_CONF_FILE +fi + WORKSPACE_DIR="/cdm_shared_workspace" mkdir -p "$WORKSPACE_DIR" cd "$WORKSPACE_DIR"