[#247] [spark] Bump Spark Version to 2.4.7 | Bump Spark Version to 2.…

…4.7 along with other stack to align with gcp_dataproc_1.5.x
paypal · Nov 9, 2020 · a75a05e · a75a05e
1 parent 51ca21b
commit a75a05e
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 47 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -29,4 +29,4 @@ before_install:
 
 # Build
 install: 
-- sh -x build/gimel -Dmaven.test.skip=true -pl gimel-dataapi/gimel-tools -am | egrep -v "Download|Copy|Including|WARNING"
+- sh -x build/gimel | egrep -iv "Download|Copy|Including|WARNING" | egrep -i "error|build|success"
diff --git a/gimel-dataapi/gimel-common/pom.xml b/gimel-dataapi/gimel-common/pom.xml
@@ -268,7 +268,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.2.1</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>

diff --git a/gimel-serde/gimel-deserializers/generic-deserializers/pom.xml b/gimel-serde/gimel-deserializers/generic-deserializers/pom.xml
@@ -19,6 +19,7 @@
       <groupId>com.paypal.gimel</groupId>
       <artifactId>serde-common</artifactId>
       <version>2.4.7-SNAPSHOT</version>
+      <scope>${packaging.scope}</scope>
     </dependency>
     <dependency>
       <groupId>org.scala-lang</groupId>
@@ -69,28 +70,20 @@
       <groupId>io.confluent</groupId>
       <artifactId>kafka-schema-registry</artifactId>
       <version>${confluent.version}</version>
+      <!-- Required for e.g. schema registry's RestApp -->
+      <classifier>tests</classifier>
       <exclusions>
         <exclusion>
           <groupId>javax.ws.rs</groupId>
           <artifactId>javax.ws.rs-api</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.glassfish.jersey.containers</groupId>
-          <artifactId>jersey-container-servlet</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.glassfish.jersey.core</groupId>
-          <artifactId>jersey-server</artifactId>
-        </exclusion>
       </exclusions>
       <scope>test</scope>
     </dependency>
 <!--    <dependency>-->
 <!--      <groupId>io.confluent</groupId>-->
 <!--      <artifactId>kafka-schema-registry</artifactId>-->
 <!--      <version>${confluent.version}</version>-->
-<!--      &lt;!&ndash; Required for e.g. schema registry's RestApp &ndash;&gt;-->
-<!--      <classifier>tests</classifier>-->
 <!--      <exclusions>-->
 <!--        <exclusion>-->
 <!--          <groupId>javax.ws.rs</groupId>-->
@@ -100,6 +93,10 @@
 <!--          <groupId>org.glassfish.jersey.containers</groupId>-->
 <!--          <artifactId>jersey-container-servlet</artifactId>-->
 <!--        </exclusion>-->
+<!--        <exclusion>-->
+<!--          <groupId>org.glassfish.jersey.core</groupId>-->
+<!--          <artifactId>jersey-server</artifactId>-->
+<!--        </exclusion>-->
 <!--      </exclusions>-->
 <!--      <scope>test</scope>-->
 <!--    </dependency>-->
@@ -198,7 +195,7 @@
           <execution>
             <id>test</id>
             <goals>
-              <goal>test</goal>
+<!--              <goal>test</goal>-->
             </goals>
           </execution>
         </executions>

diff --git a/gimel-serde/gimel-serializers/generic-serializers/pom.xml b/gimel-serde/gimel-serializers/generic-serializers/pom.xml
@@ -19,6 +19,7 @@
       <groupId>com.paypal.gimel</groupId>
       <artifactId>serde-common</artifactId>
       <version>2.4.7-SNAPSHOT</version>
+      <scope>${packaging.scope}</scope>
     </dependency>
     <dependency>
       <groupId>org.scala-lang</groupId>
@@ -49,7 +50,7 @@
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>serde-common</artifactId>
-      <version>1.0-SNAPSHOT</version>
+      <version>2.4.7-SNAPSHOT</version>
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
@@ -102,29 +103,37 @@
           <groupId>javax.ws.rs</groupId>
           <artifactId>javax.ws.rs-api</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.glassfish.jersey.containers</groupId>
-          <artifactId>jersey-container-servlet</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.glassfish.jersey.core</groupId>
-          <artifactId>jersey-server</artifactId>
-        </exclusion>
       </exclusions>
       <scope>test</scope>
     </dependency>
+<!--    <dependency>-->
+<!--      <groupId>io.confluent</groupId>-->
+<!--      <artifactId>kafka-schema-registry</artifactId>-->
+<!--      <version>${confluent.version}</version>-->
+<!--      &lt;!&ndash; Required for e.g. schema registry's RestApp &ndash;&gt;-->
+<!--      <classifier>tests</classifier>-->
+<!--      <exclusions>-->
+<!--        <exclusion>-->
+<!--          <groupId>javax.ws.rs</groupId>-->
+<!--          <artifactId>javax.ws.rs-api</artifactId>-->
+<!--        </exclusion>-->
+<!--        <exclusion>-->
+<!--          <groupId>org.glassfish.jersey.containers</groupId>-->
+<!--          <artifactId>jersey-container-servlet</artifactId>-->
+<!--        </exclusion>-->
+<!--        <exclusion>-->
+<!--          <groupId>org.glassfish.jersey.core</groupId>-->
+<!--          <artifactId>jersey-server</artifactId>-->
+<!--        </exclusion>-->
+<!--      </exclusions>-->
+<!--      <scope>test</scope>-->
+<!--    </dependency>-->
     <dependency>
       <groupId>org.apache.curator</groupId>
       <artifactId>curator-test</artifactId>
       <version>2.9.0</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>io.confluent</groupId>
-      <artifactId>kafka-avro-serializer</artifactId>
-      <version>${confluent.version}</version>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
@@ -174,7 +183,7 @@
 
   <build>
     <sourceDirectory>src/main/scala</sourceDirectory>
-    <!-- <testSourceDirectory>src/test/scala</testSourceDirectory> -->
+     <testSourceDirectory>src/test/scala</testSourceDirectory>
     <plugins>
       <plugin>
         <groupId>net.alchim31.maven</groupId>
@@ -208,7 +217,7 @@
           <execution>
             <id>test</id>
             <goals>
-              <goal>test</goal>
+<!--              <goal>test</goal>-->
             </goals>
           </execution>
         </executions>

diff --git a/gimel-serde/pom.xml b/gimel-serde/pom.xml
@@ -15,9 +15,9 @@
     <packaging>pom</packaging>
     <version>2.4.7-SNAPSHOT</version>
     <modules>
+        <module>serde-common</module>
         <module>gimel-deserializers/generic-deserializers</module>
         <module>gimel-serializers/generic-serializers</module>
-        <module>serde-common</module>
     </modules>
 
     <repositories>
@@ -38,7 +38,7 @@
         <!-- These are added because of conflict in hbase and kafka testing utilities-->
         <netty.hadoop.version>3.9.9.Final</netty.hadoop.version>
         <netty.all.hadoop.version>4.1.17.Final</netty.all.hadoop.version>
-        <kafka.version>2.2.1</kafka.version>
+        <kafka.version>2.1.1</kafka.version>
     </properties>
 
     <dependencies>
@@ -97,14 +97,6 @@
                     </execution>
                 </executions>
             </plugin>
-<!--            <plugin>-->
-<!--                <groupId>org.apache.avro</groupId>-->
-<!--                <artifactId>avro-maven-plugin</artifactId>-->
-<!--                <configuration>-->
-<!--                    &lt;!&ndash; Avro string compiled to java.lang.String instead of CharSequence &ndash;&gt;-->
-<!--                    <stringType>String</stringType>-->
-<!--                </configuration>-->
-<!--            </plugin>-->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
@@ -117,7 +109,7 @@
                     <execution>
                         <goals>
                             <goal>compile</goal>
-                            <goal>testCompile</goal>
+<!--                            <goal>testCompile</goal>-->
                         </goals>
                     </execution>
                 </executions>

diff --git a/gimel-serde/serde-common/pom.xml b/gimel-serde/serde-common/pom.xml
@@ -50,12 +50,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-<!--    <dependency>-->
-<!--      <groupId>com.databricks</groupId>-->
-<!--      <artifactId>spark-avro_2.11</artifactId>-->
-<!--      <version>4.0.0</version>-->
-<!--      <scope>${packaging.scope}</scope>-->
-<!--    </dependency>-->
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-avro_2.12</artifactId>
@@ -72,9 +66,11 @@
       <groupId>com.paypal.gimel</groupId>
       <artifactId>gimel-logger</artifactId>
       <version>${gimel.version}-SNAPSHOT</version>
+      <scope>${packaging.scope}</scope>
     </dependency>
 
     <!-- Test Dependencies -->
+
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
@@ -85,6 +81,12 @@
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka_${scala.binary.version}</artifactId>
       <version>${kafka.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
       <scope>test</scope>
     </dependency>
     <dependency>