Append Databricks version to the spark version for 14.3 programmatically

NVIDIA · Oct 30, 2024 · 8a874ca · 8a874ca
1 parent d4d879b
commit 8a874ca
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 3 deletions.
diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh
@@ -69,10 +69,18 @@ initialize()
 
     ## '-Pfoo=1,-Dbar=2,...' to '-Pfoo=1 -Dbar=2 ...'
     MVN_OPT=${MVN_OPT//','/' '}
-    BUILDVER=$(echo ${BASE_SPARK_VERSION} | sed 's/\.//g')db143
+    BUILDVER=$(echo ${BASE_SPARK_VERSION} | sed 's/\.//g')db
     # the version of Spark used when we install the Databricks jars in .m2
     BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS=${BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS:-$BASE_SPARK_VERSION}
     SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS=${BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS}-databricks
+    DBR_VER=$(cat /databricks/DBR_VERSION)
+    if [ $DBR_VER == '14.3' ]; then 
+        DBR_VER=$(echo $DBR_VER | sed 's/\.//g')
+        # We are appending 143 in addition to the base spark version because Databricks 14.3
+        # and Databricks 15.4 are both based on spark version 3.5.0
+        BUILDVER="$BUILDVER$DBR_VER"
+        SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS="$SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS-$DBR_VER"
+    fi
 
     # pull normal Spark artifacts and ignore errors then install databricks jars, then build again.
     # this should match the databricks init script.

diff --git a/jenkins/databricks/install_deps.py b/jenkins/databricks/install_deps.py
@@ -244,7 +244,7 @@ def install_deps(deps, spark_version_to_install_databricks_jars, m2_dir, jar_dir
                     <file>{jar}</file>
                     <groupId>{group_id}</groupId>
                     <artifactId>{artifact_id}</artifactId>
-                    <version>{spark_version_to_install_databricks_jars}143</version>
+                    <version>{spark_version_to_install_databricks_jars}</version>
                     <packaging>jar</packaging>
                 </configuration>
             </execution>"""

diff --git a/pom.xml b/pom.xml
@@ -826,7 +826,7 @@
         <spark330db.version>3.3.0-databricks</spark330db.version>
         <spark332db.version>3.3.2-databricks</spark332db.version>
         <spark341db.version>3.4.1-databricks</spark341db.version>
-        <spark350db143.version>3.5.0-databricks143</spark350db143.version>
+        <spark350db143.version>3.5.0-databricks-143</spark350db143.version>
         <spark350.version>3.5.0</spark350.version>
         <spark351.version>3.5.1</spark351.version>
         <spark352.version>3.5.2</spark352.version>