remove sparkcontext

NVIDIA · Oct 4, 2024 · 711d6cb · 711d6cb
1 parent dcb3273
commit 711d6cb
Showing 1 changed file with 19 additions and 16 deletions.
diff --git a/examples/ML+DL-Examples/Spark-Rapids-ML/pca/notebooks/pca.ipynb b/examples/ML+DL-Examples/Spark-Rapids-ML/pca/notebooks/pca.ipynb
@@ -118,9 +118,7 @@
     "    rapids_jar = get_rapids_jar()\n",
     "    spark = initialize_spark(rapids_jar)\n",
     "else:\n",
-    "    print(\"Using existing Spark session.\")\n",
-    "\n",
-    "sc = spark.sparkContext"
+    "    print(\"Using existing Spark session.\")"
    ]
   },
   {
@@ -134,14 +132,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "24/10/04 17:05:09 WARN TaskSetManager: Stage 0 contains a task of very large size (160085 KiB). The maximum recommended task size is 1000 KiB.\n",
+      "24/10/04 18:04:45 WARN TaskSetManager: Stage 0 contains a task of very large size (160085 KiB). The maximum recommended task size is 1000 KiB.\n",
       "                                                                                \r"
      ]
     }
@@ -180,7 +178,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -224,16 +222,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "PCA_1427871a9e2f"
+       "PCA_570681141389"
       ]
      },
-     "execution_count": 9,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -254,7 +252,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -272,24 +270,29 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2024-10-04 17:06:02,217 - spark_rapids_ml.feature.PCA - INFO - CUDA managed memory enabled.\n",
-      "2024-10-04 17:06:02,326 - spark_rapids_ml.feature.PCA - INFO - Stage-level scheduling in spark-rapids-ml requires spark.executor.cores, spark.executor.resource.gpu.amount to be set.\n",
-      "2024-10-04 17:06:02,327 - spark_rapids_ml.feature.PCA - INFO - Training spark-rapids-ml with 1 worker(s) ...\n",
-      "2024-10-04 17:06:10,446 - spark_rapids_ml.feature.PCA - INFO - Finished training\n"
+      "24/10/04 18:04:58 WARN MultiFileReaderThreadPool: Configuring the file reader thread pool with a max of 32 threads instead of spark.rapids.sql.multiThreadedRead.numThreads = 20\n",
+      "2024-10-04 18:04:58,487 - spark_rapids_ml.feature.PCA - INFO - CUDA managed memory enabled.\n",
+      "2024-10-04 18:04:58,570 - spark_rapids_ml.feature.PCA - INFO - Training spark-rapids-ml with 1 worker(s) ...\n",
+      "INFO: Process 2762394 found CUDA visible device(s): 0\n",
+      "2024-10-04 18:05:01,613 - spark_rapids_ml.feature.PCA - INFO - Loading data into python worker memory\n",
+      "2024-10-04 18:05:02,551 - spark_rapids_ml.feature.PCA - INFO - Initializing cuml context\n",
+      "2024-10-04 18:05:03,795 - spark_rapids_ml.feature.PCA - INFO - Invoking cuml fit\n",
+      "2024-10-04 18:05:05,326 - spark_rapids_ml.feature.PCA - INFO - Cuml fit complete\n",
+      "2024-10-04 18:05:06,858 - spark_rapids_ml.feature.PCA - INFO - Finished training\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "GPU PCA fit took: 8.8669753074646 sec\n"
+      "GPU PCA fit took: 8.90433144569397 sec\n"
      ]
     }
    ],