Spark 3.5 support (#220)

* Add test profile * Update to fix cve * Update to fix cve * Add new profile to the matrix build * Properly add the profile * Ok, top level profiles are still needed... * Upgrade jackson * Ah, too recent jackson is also bad :shrug * Free disk space * Free disk space PK way * Free disk space explicitly * Speed up ci, add spark 3.5 to release * Bring back prefetch * Release date * Mention CVS * Update doc/changes/changes_2.1.6.md Co-authored-by: Christoph Pirkl <[email protected]> --------- Co-authored-by: Christoph Pirkl <[email protected]>
exasol · Mar 11, 2024 · 4d09214 · 4d09214
1 parent 53a8b94
commit 4d09214
Show file tree

Hide file tree

Showing 7 changed files with 56 additions and 8 deletions.
diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml
@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         exasol-docker-version: [ 8.24.0 ]
-        profile: [ '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
+        profile: [ '-Pspark3.5', '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v4
@@ -42,6 +42,10 @@ jobs:
         run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties"
       - name: Pull docker Images
         run: docker pull exasol/docker-db:${{ matrix.exasol-docker-version }}
+      - name: Free Disk Space
+        run: |
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
       - name: Run tests and build with Maven
         run: |
           mvn --batch-mode verify ${{ matrix.profile }} \

diff --git a/.github/workflows/release_droid_release_on_maven_central.yml b/.github/workflows/release_droid_release_on_maven_central.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile: [ '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
+        profile: [ '-Pspark3.5', '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v4

diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile: [ '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
+        profile: [ '-Pspark3.5', '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v4

diff --git a/.project-keeper.yml b/.project-keeper.yml
@@ -21,6 +21,8 @@ sources:
       artifactId: "spark-connector-parent-pom"
       version: "${revision}"
       relativePath: "../parent-pom/pom.xml"
+build:
+  freeDiskSpace: true
 version:
   fromSource: parent-pom/pom.xml
 excludes:

diff --git a/doc/changes/changes_2.1.6.md b/doc/changes/changes_2.1.6.md
@@ -1,10 +1,12 @@
-# Spark Connector 2.1.6, released 2024-??-??
+# Spark Connector 2.1.6, released 2024-03-11
 
-Code name:
+Code name: Spark 3.5 support
 
 ## Summary
+Added support of spark 3.5.
 
 ## Features
 
-* ISSUE_NUMBER: description
+* #221: Add support for spark 3.5
+* #219: Updated dependency to fix CVE-2023-52428
 
diff --git a/parent-pom/pom.xml b/parent-pom/pom.xml
@@ -216,6 +216,12 @@
                 <version>${jackson.version}</version>
                 <scope>provided</scope>
             </dependency>
+            <dependency>
+                <!-- Pinned to fix CVE-2023-52428 -->
+                <groupId>com.nimbusds</groupId>
+                <artifactId>nimbus-jose-jwt</artifactId>
+                <version>9.37.3</version>
+            </dependency>
             <dependency>
                 <groupId>org.apache.hadoop</groupId>
                 <artifactId>hadoop-client</artifactId>
@@ -313,7 +319,7 @@
                 <!-- Upgrade transitive dependency of org.apache.spark:spark-core_2.13 to fix CVE-2023-42503 -->
                 <groupId>org.apache.commons</groupId>
                 <artifactId>commons-compress</artifactId>
-                <version>1.24.0</version>
+                <version>1.26.0</version>
             </dependency>
             <dependency>
                 <!-- Upgrade transitive dependency of org.apache.hadoop:hadoop-common to fix CVE-2023-39410 -->
@@ -475,7 +481,6 @@
                         <!-- CWE-22: Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal') (9.8); -->
                         <!-- https://ossindex.sonatype.org/vulnerability/CVE-2022-26612 -->
                         <exclude>CVE-2022-26612</exclude>
-
                     </excludeVulnerabilityIds>
                 </configuration>
             </plugin>
@@ -500,6 +505,24 @@
         </plugins>
     </build>
     <profiles>
+        <profile>
+            <id>spark3.5</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <properties>
+                <spark.version>3.5.1</spark.version>
+                <scala.version>2.13.11</scala.version>
+                <scala.compat.version>2.13</scala.compat.version>
+                <hadoop.version>3.3.6</hadoop.version>
+                <jackson.version>2.15.4</jackson.version>
+            </properties>
+            <modules>
+                <module>exasol-jdbc</module>
+                <module>exasol-s3</module>
+                <module>exasol-dist</module>
+            </modules>
+        </profile>
         <profile>
             <id>spark3.4</id>
             <activation>

diff --git a/pom.xml b/pom.xml
@@ -100,5 +100,22 @@
                 <module>exasol-s3</module>
             </modules>
         </profile>
+        <profile>
+            <id>spark3.5</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <properties>
+                <spark.version>3.5.1</spark.version>
+                <scala.version>2.13.10</scala.version>
+                <scala.compat.version>2.13</scala.compat.version>
+                <jackson.version>2.15.4</jackson.version>
+                <jersey.version>3.1.2</jersey.version>
+            </properties>
+            <modules>
+                <module>exasol-jdbc</module>
+                <module>exasol-s3</module>
+            </modules>
+        </profile>
     </profiles>
 </project>