[#247] [spark] Bump Spark Version to 2.4.7 | Bump Spark Version to 2.…

…4.7 along with other stack to align with gcp_dataproc_1.5.x
paypal · Nov 6, 2020 · 101ad94 · 101ad94
1 parent d78326b
commit 101ad94
Show file tree

Hide file tree

Showing 15 changed files with 122 additions and 120 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -29,4 +29,4 @@ before_install:
 
 # Build
 install: 
-- build/gimel -Dmaven.test.skip=true -pl gimel-dataapi/gimel-tools -am | egrep -v "Download|Copy|Including|WARNING"
+- sh build/gimel -Dmaven.test.skip=true -pl gimel-dataapi/gimel-tools -am | egrep -v "Download|Copy|Including|WARNING"
diff --git a/build/gimel b/build/gimel
@@ -17,7 +17,7 @@
 
 this_dir=`dirname build/install_dependencies`
 # Source reusable functions
-source ${this_dir}/gimel_functions
+. ${this_dir}/gimel_functions
 
 #--------------------------------Begin execution of Steps------------------------------------------------#
 

diff --git a/build/install_dependencies b/build/install_dependencies
@@ -19,7 +19,7 @@
 this_script=`pwd`/$BASH_SOURCE
 this_dir=`dirname $this_script`
 # Source reusable functions
-source ${this_dir}/gimel_functions
+. ${this_dir}/gimel_functions
 
 #--------------------------------Begin execution of Steps------------------------------------------------#
 

diff --git a/gimel-dataapi/gimel-common/pom.xml b/gimel-dataapi/gimel-common/pom.xml
@@ -55,9 +55,9 @@ under the License.
       <scope>${packaging.scope}</scope>
     </dependency>
     <dependency>
-      <groupId>org.scala-lang</groupId>
-      <artifactId>scala-xml</artifactId>
-      <version>2.11.0-M4</version>
+      <groupId>org.scala-lang.modules</groupId>
+      <artifactId>scala-xml_${scala.binary.version}</artifactId>
+      <version>${scala.xml.version}</version>
       <scope>${scala.packaging.scope}</scope>
     </dependency>
     <dependency>
@@ -82,12 +82,6 @@ under the License.
       <groupId>com.paypal.gimel</groupId>
       <artifactId>gimel-logger</artifactId>
       <version>${gimel.version}-SNAPSHOT</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.kafka</groupId>
-          <artifactId>kafka-clients</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.kafka</groupId>
@@ -97,7 +91,7 @@ under the License.
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>serde-common</artifactId>
-      <version>1.0-SNAPSHOT</version>
+      <version>2.4.7-SNAPSHOT</version>
       <scope>${packaging.scope}</scope>
     </dependency>
     <dependency>
@@ -106,12 +100,6 @@ under the License.
       <version>${kafka.version}</version>
       <scope>${packaging.scope}</scope>
     </dependency>
-    <dependency>
-      <groupId>com.databricks</groupId>
-      <artifactId>spark-avro_${scala.binary.version}</artifactId>
-      <version>4.0.0</version>
-      <scope>${packaging.scope}</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.curator</groupId>
       <artifactId>curator-framework</artifactId>
@@ -197,7 +185,7 @@ under the License.
       <version>${confluent.version}</version>
       <scope>test</scope>
     </dependency>
-    <!-- Kafka local testing utility needs Netty 3.x at test scope for the minicluster -->
+<!--     Kafka local testing utility needs Netty 3.x at test scope for the minicluster -->
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty</artifactId>
@@ -211,7 +199,7 @@ under the License.
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId> net.jpountz.lz4</groupId>
+      <groupId>net.jpountz.lz4</groupId>
       <artifactId>lz4</artifactId>
       <version>1.3.0</version>
       <scope>test</scope>
@@ -220,16 +208,19 @@ under the License.
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>
       <version>${jackson.version}</version>
+      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-annotations</artifactId>
       <version>${jackson.version}</version>
+      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
       <version>${jackson.version}</version>
+      <scope>compile</scope>
     </dependency>
   </dependencies>
 

diff --git a/gimel-dataapi/gimel-connectors/gimel-elasticsearch-6.2/pom.xml b/gimel-dataapi/gimel-connectors/gimel-elasticsearch-6.2/pom.xml
@@ -45,9 +45,23 @@ under the License.
     </dependency>
     <dependency>
       <groupId>org.elasticsearch</groupId>
-      <artifactId>elasticsearch-spark-20_${scala.binary.version}</artifactId>
+      <artifactId>elasticsearch-spark-20_${elastic.scala.binary.version}</artifactId>
       <version>${elasticsearch.version}</version>
       <scope>${packaging.scope}</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>log4j-over-slf4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.elasticsearch</groupId>
@@ -78,7 +92,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/gimel-dataapi/gimel-connectors/gimel-jdbc/pom.xml b/gimel-dataapi/gimel-connectors/gimel-jdbc/pom.xml
@@ -83,7 +83,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/gimel-dataapi/gimel-connectors/gimel-sftp/pom.xml b/gimel-dataapi/gimel-connectors/gimel-sftp/pom.xml
@@ -39,9 +39,19 @@ under the License.
     </dependency>
     <dependency>
       <groupId>com.springml</groupId>
-      <artifactId>spark-sftp_${scala.binary.version}</artifactId>
+      <artifactId>spark-sftp_${springml.scala.binary.version}</artifactId>
       <version>${spark.sftp.version}</version>
       <scope>${packaging.scope}</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.scala-lang</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.scalatest</groupId>
@@ -53,12 +63,11 @@ under the License.
 
   <build>
     <sourceDirectory>src/main/scala</sourceDirectory>
-    <testSourceDirectory>src/test/scala</testSourceDirectory>
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/gimel-dataapi/gimel-logger/pom.xml b/gimel-dataapi/gimel-logger/pom.xml
@@ -50,33 +50,6 @@ under the License.
       <version>${scala.version}</version>
       <scope>${scala.packaging.scope}</scope>
     </dependency>
-    <dependency>
-      <groupId>com.paypal.gimel</groupId>
-      <artifactId>gimel-logging_${gimel.logging.spark.binary.version}</artifactId>
-      <version>${gimel.logging.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-log4j12</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>log4j</groupId>
-          <artifactId>log4j</artifactId>
-        </exclusion>
-	<exclusion>
-	  <groupId>org.apache.kafka</groupId>
-	  <artifactId>*</artifactId>
-	</exclusion>
-        <exclusion>
-          <groupId>org.apache.kafka</groupId>
-          <artifactId>kafka-log4j-appender</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.googlecode.protobuf-java-format</groupId>
-          <artifactId>protobuf-java-format</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
   </dependencies>
   <build>
     <sourceDirectory>src/main/scala</sourceDirectory>

diff --git a/gimel-parser/pom.xml b/gimel-parser/pom.xml
@@ -15,9 +15,9 @@
     <dependencies>
         <!--Provided dependencies-->
         <dependency>
-            <groupId>org.scala-lang</groupId>
-            <artifactId>scala-xml</artifactId>
-            <version>2.11.0-M4</version>
+            <groupId>org.scala-lang.modules</groupId>
+            <artifactId>scala-xml_${scala.binary.version}</artifactId>
+            <version>${scala.xml.version}</version>
             <scope>${scala.packaging.scope}</scope>
         </dependency>
         <dependency>

diff --git a/gimel-serde/gimel-deserializers/generic-deserializers/pom.xml b/gimel-serde/gimel-deserializers/generic-deserializers/pom.xml
@@ -18,7 +18,7 @@
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>serde-common</artifactId>
-      <version>1.0-SNAPSHOT</version>
+      <version>2.4.7-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.scala-lang</groupId>
@@ -43,7 +43,7 @@
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>serde-common</artifactId>
-      <version>1.0-SNAPSHOT</version>
+      <version>2.4.7-SNAPSHOT</version>
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>

diff --git a/gimel-serde/gimel-serializers/generic-serializers/pom.xml b/gimel-serde/gimel-serializers/generic-serializers/pom.xml
@@ -18,7 +18,7 @@
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>serde-common</artifactId>
-      <version>1.0-SNAPSHOT</version>
+      <version>2.4.7-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.scala-lang</groupId>

diff --git a/gimel-serde/pom.xml b/gimel-serde/pom.xml
@@ -97,14 +97,14 @@
                     </execution>
                 </executions>
             </plugin>
-            <plugin>
-                <groupId>org.apache.avro</groupId>
-                <artifactId>avro-maven-plugin</artifactId>
-                <configuration>
-                    <!-- Avro string compiled to java.lang.String instead of CharSequence -->
-                    <stringType>String</stringType>
-                </configuration>
-            </plugin>
+<!--            <plugin>-->
+<!--                <groupId>org.apache.avro</groupId>-->
+<!--                <artifactId>avro-maven-plugin</artifactId>-->
+<!--                <configuration>-->
+<!--                    &lt;!&ndash; Avro string compiled to java.lang.String instead of CharSequence &ndash;&gt;-->
+<!--                    <stringType>String</stringType>-->
+<!--                </configuration>-->
+<!--            </plugin>-->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
@@ -164,7 +164,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.0.0</version>
+                <version>${maven.shade.plugin.version}</version>
                 <configuration>
                     <relocations>
                         <!--<relocation>-->

diff --git a/gimel-serde/serde-common/pom.xml b/gimel-serde/serde-common/pom.xml
@@ -50,12 +50,18 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <dependency>
-      <groupId>com.databricks</groupId>
-      <artifactId>spark-avro_${scala.binary.version}</artifactId>
-      <version>4.0.0</version>
-      <scope>${packaging.scope}</scope>
-    </dependency>
+<!--    <dependency>-->
+<!--      <groupId>com.databricks</groupId>-->
+<!--      <artifactId>spark-avro_2.11</artifactId>-->
+<!--      <version>4.0.0</version>-->
+<!--      <scope>${packaging.scope}</scope>-->
+<!--    </dependency>-->
+        <dependency>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>spark-avro_2.12</artifactId>
+          <version>${spark.version}</version>
+          <scope>${packaging.scope}</scope>
+        </dependency>
     <dependency>
       <groupId>io.spray</groupId>
       <artifactId>spray-json_${scala.binary.version}</artifactId>