#147: Added support for Apache Spark 3.4.0 version (#148)

Fixes #147
exasol · Apr 20, 2023 · 5942981 · 5942981
1 parent 0ce8e9d
commit 5942981
Show file tree

Hide file tree

Showing 10 changed files with 80 additions and 67 deletions.
diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml
@@ -20,6 +20,11 @@ jobs:
         exasol-docker-version: [ 7.1.18 ]
         mvn:
           [
+            {
+              name: 'Spark3.4',
+              profile: '-Pspark3.4',
+              project-keeper-skip: false,
+            },
             {
               name: 'Spark3.3',
               profile: '-Pspark3.3',
@@ -30,12 +35,6 @@ jobs:
               profile: '-Pspark3.3-scala2.12',
               project-keeper-skip: false,
             },
-            # Build for 3.2 disabled until a fix version is available that deals with CVE-2022-42003 and CVE-2022-42004
-            #{
-            #  name: 'Spark3.2',
-            #  profile: '-Pspark3.2',
-            #  project-keeper-skip: true,
-            #},
           ]
     steps:
       - name: Checkout the repository

diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile: [ '-Pspark3.3', '-Pspark3.3-scala2.12' ]
+        profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v3

diff --git a/.github/workflows/release_droid_print_quick_checksum.yml b/.github/workflows/release_droid_print_quick_checksum.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile: [ '-Pspark3.3', '-Pspark3.3-scala2.12' ]
+        profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v3

diff --git a/.github/workflows/release_droid_release_on_maven_central.yml b/.github/workflows/release_droid_release_on_maven_central.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile: [ '-Pspark3.3', '-Pspark3.3-scala2.12' ]
+        profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v3

diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile: [ '-Pspark3.3', '-Pspark3.3-scala2.12' ]
+        profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v3

diff --git a/dependencies.md b/dependencies.md
diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md
diff --git a/doc/changes/changes_1.3.1.md → doc/changes/changes_1.4.0.md b/doc/changes/changes_1.3.1.md → doc/changes/changes_1.4.0.md
@@ -1,9 +1,17 @@
-# The Spark Exasol Connector 1.3.1, released 2023-??-??
+# The Spark Exasol Connector 1.4.0, released 2023-04-24
 
-Code name:
+Code name: Support Apache Spark 3.4.0
 
 ## Summary
 
+In this release, we added support for the [Apache Spark 3.4.0 version](https://spark.apache.org/releases/spark-release-3-4-0.html). With this release, users could use the Exasol connector in the latest Apache Spark clusters such as Azure Databricks.
+
+We also fixed bugs due to the missing dependency classes when using the connector as a Maven dependency in the Spark applications.
+
+## Features
+
+* #147: Added support for Apache Spark 3.4.0 version
+
 ## Bug Fixes
 
 * #143: Fixed broken link in Readme file

diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom
diff --git a/pom.xml b/pom.xml
@@ -2,14 +2,14 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <artifactId>spark-connector</artifactId>
-    <version>1.3.1</version>
+    <version>1.4.0</version>
     <name>The Spark Exasol Connector</name>
     <description>A connector for Apache Spark to access Exasol</description>
     <url>https://github.com/exasol/spark-connector/</url>
     <parent>
         <artifactId>spark-connector-generated-parent</artifactId>
         <groupId>com.exasol</groupId>
-        <version>1.3.1</version>
+        <version>1.4.0</version>
         <relativePath>pk_generated_parent.pom</relativePath>
     </parent>
     <properties>
@@ -255,6 +255,10 @@
                     <groupId>javax.servlet</groupId>
                     <artifactId>*</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.eclipse.jetty</groupId>
+                    <artifactId>jetty-server</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -601,6 +605,8 @@
                     <failBuildInCaseOfConflict>false</failBuildInCaseOfConflict>
                     <ignoredResourcePatterns>
                         <ignoredResourcePattern>git.properties</ignoredResourcePattern>
+                        <ignoredResourcePattern>arrow-git.properties</ignoredResourcePattern>
+                        <ignoredResourcePattern>.*\.proto$</ignoredResourcePattern>
                     </ignoredResourcePatterns>
                 </configuration>
             </plugin>
@@ -694,10 +700,20 @@
     </build>
     <profiles>
         <profile>
-            <id>spark3.3</id>
+            <id>spark3.4</id>
             <activation>
                 <activeByDefault>true</activeByDefault>
             </activation>
+            <properties>
+                <spark.version>3.4.0</spark.version>
+                <scala.version>2.13.10</scala.version>
+                <scala.compat.version>2.13</scala.compat.version>
+                <jackson.version>2.14.2</jackson.version>
+                <jersey.version>3.1.1</jersey.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>spark3.3</id>
             <properties>
                 <spark.version>3.3.2</spark.version>
                 <scala.version>2.13.10</scala.version>
@@ -720,15 +736,5 @@
                 <jersey.version>3.1.1</jersey.version>
             </properties>
         </profile>
-        <profile>
-            <id>spark3.2</id>
-            <properties>
-                <spark.version>3.2.3</spark.version>
-                <scala.version>2.13.10</scala.version>
-                <scala.compat.version>2.13</scala.compat.version>
-                <jackson.version>2.12.7.1</jackson.version>
-                <jersey.version>3.1.1</jersey.version>
-            </properties>
-        </profile>
     </profiles>
 </project>