From 93f78774c8d58ca1d20cdf5107c8ef840dcad8f6 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Wed, 27 Dec 2023 16:52:21 -0800 Subject: [PATCH 1/3] Revert "Shim MockTaskContext to fix Spark 3.5.1 build (#10096)" This reverts commit bc79bcf2e2d08ec4e46314b98d4b3019bb93a59b. --- .../metrics/source/MockTaskContext.scala | 4 +- .../metrics/source/MockTaskContextBase.scala | 46 ------------------- .../metrics/source/MockTaskContextBase.scala | 26 ----------- 3 files changed, 2 insertions(+), 74 deletions(-) delete mode 100644 tests/src/test/spark311/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContextBase.scala delete mode 100644 tests/src/test/spark351/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContextBase.scala diff --git a/tests/src/test/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContext.scala b/tests/src/test/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContext.scala index 4bb3d0ca2ca..84766789578 100644 --- a/tests/src/test/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContext.scala +++ b/tests/src/test/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContext.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ import org.apache.spark.scheduler.TaskLocality import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.util.{AccumulatorV2, TaskCompletionListener, TaskFailureListener} -class MockTaskContext(taskAttemptId: Long, partitionId: Int) extends MockTaskContextBase { +class MockTaskContext(taskAttemptId: Long, partitionId: Int) extends TaskContext { val listeners = new ListBuffer[TaskCompletionListener] diff --git a/tests/src/test/spark311/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContextBase.scala b/tests/src/test/spark311/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContextBase.scala deleted file mode 100644 index 8b70a8e1969..00000000000 --- a/tests/src/test/spark311/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContextBase.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -{"spark": "320"} -{"spark": "321"} -{"spark": "321cdh"} -{"spark": "321db"} -{"spark": "322"} -{"spark": "323"} -{"spark": "324"} -{"spark": "330"} -{"spark": "330cdh"} -{"spark": "330db"} -{"spark": "331"} -{"spark": "332"} -{"spark": "332cdh"} -{"spark": "332db"} -{"spark": "333"} -{"spark": "340"} -{"spark": "341"} -{"spark": "341db"} -{"spark": "350"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.metrics.source - -import org.apache.spark.TaskContext - -abstract class MockTaskContextBase extends TaskContext { -} diff --git a/tests/src/test/spark351/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContextBase.scala b/tests/src/test/spark351/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContextBase.scala deleted file mode 100644 index d39ad6a1add..00000000000 --- a/tests/src/test/spark351/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContextBase.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "351"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.metrics.source - -import org.apache.spark.TaskContext - -abstract class MockTaskContextBase extends TaskContext { - override def isFailed(): Boolean = false -} From a3f7f9e8ceabf7a168a6a368862b480c30e612ec Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Wed, 27 Dec 2023 17:10:12 -0800 Subject: [PATCH 2/3] Added isFailed to MockTaskContext --- .../spark/sql/rapids/metrics/source/MockTaskContext.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/src/test/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContext.scala b/tests/src/test/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContext.scala index 84766789578..b0aec74feb7 100644 --- a/tests/src/test/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContext.scala +++ b/tests/src/test/scala/org/apache/spark/sql/rapids/metrics/source/MockTaskContext.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -96,4 +96,10 @@ class MockTaskContext(taskAttemptId: Long, partitionId: Int) extends TaskContext def markTaskComplete(): Unit = { listeners.foreach(_.onTaskCompletion(this)) } + + /** + * This method was introduced in Spark-3.5.1. It's not shimmed and added to the common class by + * removing the override keyword. + */ + def isFailed(): Boolean = false } From 78cc54e065f2bec157cd191d5ead15a962373209 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Wed, 27 Dec 2023 17:19:10 -0800 Subject: [PATCH 3/3] Signing off Signed-off-by: Raza Jafri