From 594298155aee55d451b04e384ac1e0895be762fa Mon Sep 17 00:00:00 2001 From: Muhammet Orazov <916295+morazow@users.noreply.github.com> Date: Thu, 20 Apr 2023 19:07:22 +0200 Subject: [PATCH] #147: Added support for Apache Spark 3.4.0 version (#148) Fixes #147 --- .github/workflows/ci-build.yml | 11 ++- ...elease_droid_prepare_original_checksum.yml | 2 +- .../release_droid_print_quick_checksum.yml | 2 +- ...release_droid_release_on_maven_central.yml | 2 +- ...ase_droid_upload_github_release_assets.yml | 2 +- dependencies.md | 80 +++++++++---------- doc/changes/changelog.md | 2 +- .../{changes_1.3.1.md => changes_1.4.0.md} | 12 ++- pk_generated_parent.pom | 2 +- pom.xml | 32 +++++--- 10 files changed, 80 insertions(+), 67 deletions(-) rename doc/changes/{changes_1.3.1.md => changes_1.4.0.md} (75%) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 3d7c38e4..2bb943ec 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -20,6 +20,11 @@ jobs: exasol-docker-version: [ 7.1.18 ] mvn: [ + { + name: 'Spark3.4', + profile: '-Pspark3.4', + project-keeper-skip: false, + }, { name: 'Spark3.3', profile: '-Pspark3.3', @@ -30,12 +35,6 @@ jobs: profile: '-Pspark3.3-scala2.12', project-keeper-skip: false, }, - # Build for 3.2 disabled until a fix version is available that deals with CVE-2022-42003 and CVE-2022-42004 - #{ - # name: 'Spark3.2', - # profile: '-Pspark3.2', - # project-keeper-skip: true, - #}, ] steps: - name: Checkout the repository diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml index c5630a12..5a2d9493 100644 --- a/.github/workflows/release_droid_prepare_original_checksum.yml +++ b/.github/workflows/release_droid_prepare_original_checksum.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [ '-Pspark3.3', '-Pspark3.3-scala2.12' ] + profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ] steps: - name: Checkout the repository uses: actions/checkout@v3 diff --git a/.github/workflows/release_droid_print_quick_checksum.yml b/.github/workflows/release_droid_print_quick_checksum.yml index a0a60dc3..212a4c71 100644 --- a/.github/workflows/release_droid_print_quick_checksum.yml +++ b/.github/workflows/release_droid_print_quick_checksum.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [ '-Pspark3.3', '-Pspark3.3-scala2.12' ] + profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ] steps: - name: Checkout the repository uses: actions/checkout@v3 diff --git a/.github/workflows/release_droid_release_on_maven_central.yml b/.github/workflows/release_droid_release_on_maven_central.yml index b5ade83d..b9df24db 100644 --- a/.github/workflows/release_droid_release_on_maven_central.yml +++ b/.github/workflows/release_droid_release_on_maven_central.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [ '-Pspark3.3', '-Pspark3.3-scala2.12' ] + profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ] steps: - name: Checkout the repository uses: actions/checkout@v3 diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index b8bba353..37f08677 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [ '-Pspark3.3', '-Pspark3.3-scala2.12' ] + profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ] steps: - name: Checkout the repository uses: actions/checkout@v3 diff --git a/dependencies.md b/dependencies.md index e2dd6949..baac80fb 100644 --- a/dependencies.md +++ b/dependencies.md @@ -13,16 +13,16 @@ | [Spark Project SQL][8] | [Apache 2.0 License][9] | | [Guava: Google Core Libraries for Java][10] | [Apache License, Version 2.0][11] | | [Netty/All-in-One][12] | [Apache License, Version 2.0][1] | -| [jackson-databind][13] | [The Apache Software License, Version 2.0][11] | -| [jersey-core-common][14] | [EPL 2.0][15]; [The GNU General Public License (GPL), Version 2, With Classpath Exception][16]; [Apache License, 2.0][9]; [Public Domain][17] | -| [jersey-media-jaxb][18] | [EPL 2.0][15]; [GPL2 w/ CPE][16]; [EDL 1.0][19]; [BSD 2-Clause][20]; [Apache License, 2.0][9]; [Public Domain][17]; [Modified BSD][21]; [jQuery license][22]; [MIT license][23]; [W3C license][24] | -| [jersey-core-server][25] | [EPL 2.0][15]; [The GNU General Public License (GPL), Version 2, With Classpath Exception][16]; [Apache License, 2.0][9]; [Modified BSD][21] | -| [jersey-core-client][26] | [EPL 2.0][15]; [GPL2 w/ CPE][16]; [EDL 1.0][19]; [BSD 2-Clause][20]; [Apache License, 2.0][9]; [Public Domain][17]; [Modified BSD][21]; [jQuery license][22]; [MIT license][23]; [W3C license][24] | -| [Apache Avro Mapred API][27] | [Apache License, Version 2.0][28] | -| Apache Hadoop Client Aggregator | [Apache License, Version 2.0][28] | +| [jackson-databind][13] | [The Apache Software License, Version 2.0][14] | +| [jersey-core-common][15] | [EPL 2.0][16]; [The GNU General Public License (GPL), Version 2, With Classpath Exception][17]; [Apache License, 2.0][9]; [Public Domain][18] | +| [jersey-media-jaxb][19] | [EPL 2.0][16]; [GPL2 w/ CPE][17]; [EDL 1.0][20]; [BSD 2-Clause][21]; [Apache License, 2.0][9]; [Public Domain][18]; [Modified BSD][22]; [jQuery license][23]; [MIT license][24]; [W3C license][25] | +| [jersey-core-server][26] | [EPL 2.0][16]; [The GNU General Public License (GPL), Version 2, With Classpath Exception][17]; [Apache License, 2.0][9]; [Modified BSD][22] | +| [jersey-core-client][27] | [EPL 2.0][16]; [GPL2 w/ CPE][17]; [EDL 1.0][20]; [BSD 2-Clause][21]; [Apache License, 2.0][9]; [Public Domain][18]; [Modified BSD][22]; [jQuery license][23]; [MIT license][24]; [W3C license][25] | +| [Apache Avro Mapred API][28] | [Apache License, Version 2.0][14] | +| Apache Hadoop Client Aggregator | [Apache License, Version 2.0][14] | | [Woodstox][29] | [The Apache License, Version 2.0][11] | | [Protocol Buffers [Core]][30] | [BSD-3-Clause][31] | -| [Apache Commons Text][32] | [Apache License, Version 2.0][28] | +| [Apache Commons Text][32] | [Apache License, Version 2.0][14] | ## Test Dependencies @@ -31,9 +31,9 @@ | [scalatest][33] | [the Apache License, ASL Version 2.0][34] | | [scalatestplus-mockito][35] | [Apache-2.0][34] | | [mockito-core][36] | [The MIT License][37] | -| [Apache Log4j API][38] | [Apache License, Version 2.0][28] | -| [Apache Log4j 1.x Compatibility API][39] | [Apache License, Version 2.0][28] | -| [Apache Log4j Core][40] | [Apache License, Version 2.0][28] | +| [Apache Log4j API][38] | [Apache License, Version 2.0][14] | +| [Apache Log4j 1.x Compatibility API][39] | [Apache License, Version 2.0][14] | +| [Apache Log4j Core][40] | [Apache License, Version 2.0][14] | | [Test Database Builder for Java][41] | [MIT License][42] | | [Matcher for SQL Result Sets][43] | [MIT License][44] | | [Test containers for Exasol on Docker][45] | [MIT License][46] | @@ -44,29 +44,29 @@ | ------------------------------------------------------- | ---------------------------------------------- | | [SonarQube Scanner for Maven][47] | [GNU LGPL 3][48] | | [scala-maven-plugin][49] | [Public domain (Unlicense)][50] | -| [Apache Maven Compiler Plugin][51] | [Apache-2.0][28] | -| [Apache Maven Enforcer Plugin][52] | [Apache-2.0][28] | -| [Maven Flatten Plugin][53] | [Apache Software Licenese][28] | +| [Apache Maven Compiler Plugin][51] | [Apache-2.0][14] | +| [Apache Maven Enforcer Plugin][52] | [Apache-2.0][14] | +| [Maven Flatten Plugin][53] | [Apache Software Licenese][14] | | [org.sonatype.ossindex.maven:ossindex-maven-plugin][54] | [ASL2][11] | -| [Maven Surefire Plugin][55] | [Apache-2.0][28] | -| [Versions Maven Plugin][56] | [Apache License, Version 2.0][28] | +| [Maven Surefire Plugin][55] | [Apache-2.0][14] | +| [Versions Maven Plugin][56] | [Apache License, Version 2.0][14] | | [duplicate-finder-maven-plugin Maven Mojo][57] | [Apache License 2.0][9] | -| [Apache Maven Deploy Plugin][58] | [Apache-2.0][28] | -| [Apache Maven GPG Plugin][59] | [Apache License, Version 2.0][28] | -| [Apache Maven Source Plugin][60] | [Apache License, Version 2.0][28] | -| [Apache Maven Javadoc Plugin][61] | [Apache-2.0][28] | +| [Apache Maven Deploy Plugin][58] | [Apache-2.0][14] | +| [Apache Maven GPG Plugin][59] | [Apache License, Version 2.0][14] | +| [Apache Maven Source Plugin][60] | [Apache License, Version 2.0][14] | +| [Apache Maven Javadoc Plugin][61] | [Apache-2.0][14] | | [Nexus Staging Maven Plugin][62] | [Eclipse Public License][63] | | [ScalaTest Maven Plugin][64] | [the Apache License, ASL Version 2.0][34] | -| [Apache Maven JAR Plugin][65] | [Apache License, Version 2.0][28] | -| [Apache Maven Shade Plugin][66] | [Apache License, Version 2.0][28] | -| [Maven Failsafe Plugin][67] | [Apache-2.0][28] | +| [Apache Maven JAR Plugin][65] | [Apache License, Version 2.0][14] | +| [Apache Maven Shade Plugin][66] | [Apache License, Version 2.0][14] | +| [Maven Failsafe Plugin][67] | [Apache-2.0][14] | | [JaCoCo :: Maven Plugin][68] | [Eclipse Public License 2.0][69] | | [error-code-crawler-maven-plugin][70] | [MIT License][71] | | [Reproducible Build Maven Plugin][72] | [Apache 2.0][11] | | [Project keeper maven plugin][73] | [The MIT License][74] | | [Artifact reference checker and unifier][75] | [MIT License][76] | | [OpenFastTrace Maven Plugin][77] | [GNU General Public License v3.0][78] | -| [spotless-maven-plugin][79] | [The Apache Software License, Version 2.0][28] | +| [spotless-maven-plugin][79] | [The Apache Software License, Version 2.0][14] | | [scalafix-maven-plugin][80] | [BSD-3-Clause][31] | | [Maven Clean Plugin][81] | [The Apache Software License, Version 2.0][11] | | [Maven Resources Plugin][82] | [The Apache Software License, Version 2.0][11] | @@ -86,22 +86,22 @@ [10]: https://github.com/google/guava [11]: http://www.apache.org/licenses/LICENSE-2.0.txt [12]: https://netty.io/index.html -[13]: http://github.com/FasterXML/jackson -[14]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-common -[15]: http://www.eclipse.org/legal/epl-2.0 -[16]: https://www.gnu.org/software/classpath/license.html -[17]: https://creativecommons.org/publicdomain/zero/1.0/ -[18]: https://eclipse-ee4j.github.io/jersey/ -[19]: http://www.eclipse.org/org/documents/edl-v10.php -[20]: https://opensource.org/licenses/BSD-2-Clause -[21]: https://asm.ow2.io/license.html -[22]: https://github.com/jquery/jquery/blob/main/LICENSE.txt -[23]: http://www.opensource.org/licenses/mit-license.php -[24]: https://www.w3.org/Consortium/Legal/copyright-documents-19990405 -[25]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-server -[26]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-client -[27]: https://avro.apache.org -[28]: https://www.apache.org/licenses/LICENSE-2.0.txt +[13]: https://github.com/FasterXML/jackson +[14]: https://www.apache.org/licenses/LICENSE-2.0.txt +[15]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-common +[16]: http://www.eclipse.org/legal/epl-2.0 +[17]: https://www.gnu.org/software/classpath/license.html +[18]: https://creativecommons.org/publicdomain/zero/1.0/ +[19]: https://eclipse-ee4j.github.io/jersey/ +[20]: http://www.eclipse.org/org/documents/edl-v10.php +[21]: https://opensource.org/licenses/BSD-2-Clause +[22]: https://asm.ow2.io/license.html +[23]: https://github.com/jquery/jquery/blob/main/LICENSE.txt +[24]: http://www.opensource.org/licenses/mit-license.php +[25]: https://www.w3.org/Consortium/Legal/copyright-documents-19990405 +[26]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-server +[27]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-client +[28]: https://avro.apache.org [29]: https://github.com/FasterXML/woodstox [30]: https://developers.google.com/protocol-buffers/docs/javatutorial [31]: https://opensource.org/licenses/BSD-3-Clause diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index d7734101..b9377d8e 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,6 +1,6 @@ # Changes -* [1.3.1](changes_1.3.1.md) +* [1.4.0](changes_1.4.0.md) * [1.3.0](changes_1.3.0.md) * [1.2.5](changes_1.2.5.md) * [1.2.4](changes_1.2.4.md) diff --git a/doc/changes/changes_1.3.1.md b/doc/changes/changes_1.4.0.md similarity index 75% rename from doc/changes/changes_1.3.1.md rename to doc/changes/changes_1.4.0.md index 1eca7687..c37cffa9 100644 --- a/doc/changes/changes_1.3.1.md +++ b/doc/changes/changes_1.4.0.md @@ -1,9 +1,17 @@ -# The Spark Exasol Connector 1.3.1, released 2023-??-?? +# The Spark Exasol Connector 1.4.0, released 2023-04-24 -Code name: +Code name: Support Apache Spark 3.4.0 ## Summary +In this release, we added support for the [Apache Spark 3.4.0 version](https://spark.apache.org/releases/spark-release-3-4-0.html). With this release, users could use the Exasol connector in the latest Apache Spark clusters such as Azure Databricks. + +We also fixed bugs due to the missing dependency classes when using the connector as a Maven dependency in the Spark applications. + +## Features + +* #147: Added support for Apache Spark 3.4.0 version + ## Bug Fixes * #143: Fixed broken link in Readme file diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index cd22fc87..38107fec 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,7 +3,7 @@ 4.0.0 com.exasol spark-connector-generated-parent - 1.3.1 + 1.4.0 pom UTF-8 diff --git a/pom.xml b/pom.xml index 7f51d9e0..3869b2a3 100644 --- a/pom.xml +++ b/pom.xml @@ -2,14 +2,14 @@ 4.0.0 spark-connector - 1.3.1 + 1.4.0 The Spark Exasol Connector A connector for Apache Spark to access Exasol https://github.com/exasol/spark-connector/ spark-connector-generated-parent com.exasol - 1.3.1 + 1.4.0 pk_generated_parent.pom @@ -255,6 +255,10 @@ javax.servlet * + + org.eclipse.jetty + jetty-server + @@ -601,6 +605,8 @@ false git.properties + arrow-git.properties + .*\.proto$ @@ -694,10 +700,20 @@ - spark3.3 + spark3.4 true + + 3.4.0 + 2.13.10 + 2.13 + 2.14.2 + 3.1.1 + + + + spark3.3 3.3.2 2.13.10 @@ -720,15 +736,5 @@ 3.1.1 - - spark3.2 - - 3.2.3 - 2.13.10 - 2.13 - 2.12.7.1 - 3.1.1 - -