Update SetMaster, fix caching issue

NVIDIA · Oct 8, 2024 · 56de5f3 · 56de5f3
1 parent 1d21c16
commit 56de5f3
Show file tree

Hide file tree

Showing 11 changed files with 1,154 additions and 1,113 deletions.
diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb
diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb
diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb
@@ -249,6 +249,11 @@
     "conda_env = os.environ.get(\"CONDA_PREFIX\")\n",
     "\n",
     "conf = SparkConf()\n",
+    "if 'spark' not in globals():\n",
+    "    # If Spark is not already started with Jupyter, attach to Spark Standalone\n",
+    "    import socket\n",
+    "    hostname = socket.gethostname()\n",
+    "    conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n",
     "conf.set(\"spark.task.maxFailures\", \"1\")\n",
     "conf.set(\"spark.driver.memory\", \"8g\")\n",
     "conf.set(\"spark.executor.memory\", \"8g\")\n",
@@ -282,7 +287,7 @@
     "\n",
     "len(lines)\n",
     "\n",
-    "df = spark.createDataFrame(lines, ['lines']).repartition(10).cache()"
+    "df = spark.createDataFrame(lines, ['lines']).repartition(8).cache()"
    ]
   },
   {

diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb
@@ -181,6 +181,11 @@
     "conda_env = os.environ.get(\"CONDA_PREFIX\")\n",
     "\n",
     "conf = SparkConf()\n",
+    "if 'spark' not in globals():\n",
+    "    # If Spark is not already started with Jupyter, attach to Spark Standalone\n",
+    "    import socket\n",
+    "    hostname = socket.gethostname()\n",
+    "    conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n",
     "conf.set(\"spark.task.maxFailures\", \"1\")\n",
     "conf.set(\"spark.driver.memory\", \"8g\")\n",
     "conf.set(\"spark.executor.memory\", \"8g\")\n",
@@ -214,7 +219,7 @@
     "\n",
     "len(lines)\n",
     "\n",
-    "df = spark.createDataFrame(lines, ['lines']).repartition(10).cache()"
+    "df = spark.createDataFrame(lines, ['lines']).repartition(8).cache()"
    ]
   },
   {