[#247] [spark] Bump Spark Version to 2.4.7 | Bump Spark Version to 2.…

…4.7 along with other stack to align with gcp_dataproc_1.5.x
paypal · Nov 7, 2020 · cfc6c64 · cfc6c64
1 parent a83d31f
commit cfc6c64
Show file tree

Hide file tree

Showing 26 changed files with 29 additions and 44 deletions.
diff --git a/gimel-dataapi/gimel-common/pom.xml b/gimel-dataapi/gimel-common/pom.xml
@@ -150,7 +150,7 @@ under the License.
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-streaming-kafka-${kafka.binary.version}_${scala.binary.version}</artifactId>
+      <artifactId>spark-streaming-kafka-${spark.kafka.connector.version}</artifactId>
       <version>${spark.version}</version>
       <scope>${spark.packaging.scope}</scope>
     </dependency>

diff --git a/gimel-dataapi/gimel-connectors/gimel-hbase/pom.xml b/gimel-dataapi/gimel-connectors/gimel-hbase/pom.xml
@@ -47,8 +47,18 @@ under the License.
     <dependency>
       <groupId>com.hortonworks</groupId>
       <artifactId>shc-core</artifactId>
-      <version>${hortonworks.shc.version}-${spark.binary.version}-s_${scala.binary.version}</version>
+      <version>1.1.1-2.1-s_2.11</version>
       <scope>${packaging.scope}</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.scala-lang</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hbase</groupId>
@@ -179,7 +189,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/.../gimel-connectors/gimel-kafka-2.2/pom.xml → ...aapi/gimel-connectors/gimel-kafka/pom.xml b/.../gimel-connectors/gimel-kafka-2.2/pom.xml → ...aapi/gimel-connectors/gimel-kafka/pom.xml
@@ -28,7 +28,7 @@ under the License.
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>gimel-kafka-2.2</artifactId>
+    <artifactId>gimel-kafka</artifactId>
     <version>2.4.7-SNAPSHOT</version>
 
     <dependencies>
@@ -37,6 +37,14 @@ under the License.
             <artifactId>gimel-common</artifactId>
             <version>${gimel.version}-SNAPSHOT</version>
             <exclusions>
+                <exclusion>
+                    <groupId>org.apache.hbase</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-core</artifactId>
@@ -51,45 +59,18 @@ under the License.
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>com.databricks</groupId>
-            <artifactId>spark-avro_${scala.binary.version}</artifactId>
-            <version>3.2.0</version>
-            <scope>${packaging.scope}</scope>
-        </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql-kafka-${spark.kafka.connector.version}</artifactId>
             <version>${spark.version}</version>
             <scope>${packaging.scope}</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.kafka</groupId>
-                    <artifactId>kafka-clients</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
         <!--kafka Clients-->
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka-clients</artifactId>
             <version>${kafka.version}</version>
         </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
 
         <!-- Test dependencies -->
         <!-- To reuse some of the common testing utilities -->
@@ -163,12 +144,6 @@ under the License.
             <version>${confluent.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.avro</groupId>
-            <artifactId>avro</artifactId>
-            <version>1.7.7</version>
-            <scope>test</scope>
-        </dependency>
         <!-- Kafka local testing utility needs Netty 3.x at test scope for the minicluster -->
         <dependency>
             <groupId>io.netty</groupId>
@@ -234,7 +209,7 @@ under the License.
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>${maven.shade.plugin.version}</version>
                 <configuration>
                     <relocations>
                         <relocation>

diff --git a/...ala/com/paypal/gimel/kafka2/DataSet.scala → ...ala/com/paypal/gimel/kafka2/DataSet.scala b/...ala/com/paypal/gimel/kafka2/DataSet.scala → ...ala/com/paypal/gimel/kafka2/DataSet.scala
diff --git a/.../com/paypal/gimel/kafka2/DataStream.scala → .../com/paypal/gimel/kafka2/DataStream.scala b/.../com/paypal/gimel/kafka2/DataStream.scala → .../com/paypal/gimel/kafka2/DataStream.scala
diff --git a/...afka2/conf/KafkaClientConfiguration.scala → ...afka2/conf/KafkaClientConfiguration.scala b/...afka2/conf/KafkaClientConfiguration.scala → ...afka2/conf/KafkaClientConfiguration.scala
diff --git a/...ypal/gimel/kafka2/conf/KafkaConfigs.scala → ...ypal/gimel/kafka2/conf/KafkaConfigs.scala b/...ypal/gimel/kafka2/conf/KafkaConfigs.scala → ...ypal/gimel/kafka2/conf/KafkaConfigs.scala
diff --git a/...al/gimel/kafka2/conf/KafkaConstants.scala → ...al/gimel/kafka2/conf/KafkaConstants.scala b/...al/gimel/kafka2/conf/KafkaConstants.scala → ...al/gimel/kafka2/conf/KafkaConstants.scala
diff --git a/...gimel/kafka2/conf/KafkaJsonProtocol.scala → ...gimel/kafka2/conf/KafkaJsonProtocol.scala b/...gimel/kafka2/conf/KafkaJsonProtocol.scala → ...gimel/kafka2/conf/KafkaJsonProtocol.scala
diff --git a/...el/kafka2/reader/KafkaBatchConsumer.scala → ...el/kafka2/reader/KafkaBatchConsumer.scala b/...el/kafka2/reader/KafkaBatchConsumer.scala → ...el/kafka2/reader/KafkaBatchConsumer.scala
diff --git a/...l/kafka2/reader/KafkaStreamConsumer.scala → ...l/kafka2/reader/KafkaStreamConsumer.scala b/...l/kafka2/reader/KafkaStreamConsumer.scala → ...l/kafka2/reader/KafkaStreamConsumer.scala
diff --git a/...2/utilities/ImplicitKafkaConverters.scala → ...2/utilities/ImplicitKafkaConverters.scala b/...2/utilities/ImplicitKafkaConverters.scala → ...2/utilities/ImplicitKafkaConverters.scala
diff --git a/...2/utilities/ImplicitZKCheckPointers.scala → ...2/utilities/ImplicitZKCheckPointers.scala b/...2/utilities/ImplicitZKCheckPointers.scala → ...2/utilities/ImplicitZKCheckPointers.scala
diff --git a/...2/utilities/KafkaOptionsLoaderUtils.scala → ...2/utilities/KafkaOptionsLoaderUtils.scala b/...2/utilities/KafkaOptionsLoaderUtils.scala → ...2/utilities/KafkaOptionsLoaderUtils.scala
diff --git a/...mel/kafka2/utilities/KafkaUtilities.scala → ...mel/kafka2/utilities/KafkaUtilities.scala b/...mel/kafka2/utilities/KafkaUtilities.scala → ...mel/kafka2/utilities/KafkaUtilities.scala
diff --git a/...el/kafka2/writer/KafkaBatchProducer.scala → ...el/kafka2/writer/KafkaBatchProducer.scala b/...el/kafka2/writer/KafkaBatchProducer.scala → ...el/kafka2/writer/KafkaBatchProducer.scala
diff --git a/...l/kafka2/writer/KafkaStreamProducer.scala → ...l/kafka2/writer/KafkaStreamProducer.scala b/...l/kafka2/writer/KafkaStreamProducer.scala → ...l/kafka2/writer/KafkaStreamProducer.scala
diff --git a/...com/paypal/gimel/kafka2/DataSetTest.scala → ...com/paypal/gimel/kafka2/DataSetTest.scala b/...com/paypal/gimel/kafka2/DataSetTest.scala → ...com/paypal/gimel/kafka2/DataSetTest.scala
diff --git a/...afka2/reader/KafkaBatchConsumerTest.scala → ...afka2/reader/KafkaBatchConsumerTest.scala b/...afka2/reader/KafkaBatchConsumerTest.scala → ...afka2/reader/KafkaBatchConsumerTest.scala
diff --git a/...ilities/ImplicitKafkaConvertersTest.scala → ...ilities/ImplicitKafkaConvertersTest.scala b/...ilities/ImplicitKafkaConvertersTest.scala → ...ilities/ImplicitKafkaConvertersTest.scala
diff --git a/...ilities/ImplicitZKCheckPointersTest.scala → ...ilities/ImplicitZKCheckPointersTest.scala b/...ilities/ImplicitZKCheckPointersTest.scala → ...ilities/ImplicitZKCheckPointersTest.scala
diff --git a/...ilities/KafkaOptionsLoaderUtilsTest.scala → ...ilities/KafkaOptionsLoaderUtilsTest.scala b/...ilities/KafkaOptionsLoaderUtilsTest.scala → ...ilities/KafkaOptionsLoaderUtilsTest.scala
diff --git a/...kafka2/utilities/KafkaUtilitiesTest.scala → ...kafka2/utilities/KafkaUtilitiesTest.scala b/...kafka2/utilities/KafkaUtilitiesTest.scala → ...kafka2/utilities/KafkaUtilitiesTest.scala
diff --git a/...a2/utilities/MockKafkaoptionsLoader.scala → ...a2/utilities/MockKafkaoptionsLoader.scala b/...a2/utilities/MockKafkaoptionsLoader.scala → ...a2/utilities/MockKafkaoptionsLoader.scala
diff --git a/...afka2/writer/KafkaBatchProducerTest.scala → ...afka2/writer/KafkaBatchProducerTest.scala b/...afka2/writer/KafkaBatchProducerTest.scala → ...afka2/writer/KafkaBatchProducerTest.scala
diff --git a/gimel-dataapi/pom.xml b/gimel-dataapi/pom.xml
@@ -43,11 +43,11 @@ under the License.
         <module>gimel-connectors/gimel-hive</module>
         <module>gimel-connectors/gimel-restapi</module>
         <module>gimel-connectors/gimel-s3</module>
-        <module>gimel-connectors/gimel-jdbc</module>
+        <module>gimel-connectors/gimel-kafka</module>
         <module>gimel-connectors/gimel-hbase</module>
+        <module>gimel-connectors/gimel-jdbc</module>
         <module>gimel-connectors/gimel-cassandra-2.0</module>
         <module>gimel-connectors/gimel-aerospike-3.14</module>
-        <module>gimel-connectors/gimel-kafka-2.2</module>
         <module>gimel-connectors/gimel-druid-0.82</module>
         <module>gimel-core</module>
         <module>gimel-sql</module>
@@ -67,9 +67,9 @@ under the License.
         <jackson.version>2.6.7</jackson.version>
         <java.version>1.8</java.version>
         <jersey.version>1.19.4</jersey.version>
-        <kafka.paypal.monitoring.version>1.3.2</kafka.paypal.monitoring.version>
-        <kafka.version>2.2.1</kafka.version>
-        <kafka.binary.version>0-10</kafka.binary.version>
+<!--        <kafka.paypal.monitoring.version>1.3.2</kafka.paypal.monitoring.version>-->
+        <kafka.version>2.1.1</kafka.version>
+<!--        <kafka.binary.version>0-10</kafka.binary.version>-->
         <protobuf.javaformatter.version>1.4</protobuf.javaformatter.version>
         <gimel.logging.spark.binary.version>2.3</gimel.logging.spark.binary.version>
         <gimel.logging.version>0.4.3-SNAPSHOT</gimel.logging.version>
@@ -80,7 +80,7 @@ under the License.
         <teradata.version>15.10.00.22</teradata.version>
         <zookeeper.version>3.4.13</zookeeper.version>
         <curator.version>2.13.0</curator.version>
-        <spark.kafka.connector.version>0-10_2.11</spark.kafka.connector.version>
+        <spark.kafka.connector.version>0-10_2.12</spark.kafka.connector.version>
         <spark.sftp.version>1.1.3</spark.sftp.version>
         <!-- These are added because of conflict in hbase-testing-utility-->
         <netty.hadoop.version>3.9.9.Final</netty.hadoop.version>