[CELEBORN-1443] Remove ratis dependencies from common module

### What changes were proposed in this pull request? Remove ratis dependencies from common module. ### Why are the changes needed? Ratis is only depended on by the master module. Removing ratis dependencies from the common module reduces the size of the Celeborn client package. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? GA. Closes apache#2538 from SteNicholas/CELEBORN-1443. Authored-by: SteNicholas <[email protected]> Signed-off-by: Shuang <[email protected]>
SteNicholas · Jun 3, 2024 · e5f09ce · e5f09ce
1 parent 142d0a0
commit e5f09ce
Show file tree

Hide file tree

Showing 16 changed files with 0 additions and 77 deletions.
diff --git a/common/pom.xml b/common/pom.xml
@@ -30,14 +30,6 @@
   <name>Celeborn Common</name>
 
   <dependencies>
-    <dependency>
-      <groupId>org.apache.ratis</groupId>
-      <artifactId>ratis-common</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.ratis</groupId>
-      <artifactId>ratis-client</artifactId>
-    </dependency>
     <dependency>
       <groupId>io.dropwizard.metrics</groupId>
       <artifactId>metrics-core</artifactId>

diff --git a/dev/deps/dependencies-client-flink-1.14 b/dev/deps/dependencies-client-flink-1.14
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.18//scala-library-2.12.18.jar
 scala-reflect/2.12.18//scala-reflect-2.12.18.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-flink-1.15 b/dev/deps/dependencies-client-flink-1.15
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.18//scala-library-2.12.18.jar
 scala-reflect/2.12.18//scala-reflect-2.12.18.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-flink-1.17 b/dev/deps/dependencies-client-flink-1.17
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.18//scala-library-2.12.18.jar
 scala-reflect/2.12.18//scala-reflect-2.12.18.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-flink-1.18 b/dev/deps/dependencies-client-flink-1.18
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.18//scala-library-2.12.18.jar
 scala-reflect/2.12.18//scala-reflect-2.12.18.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-flink-1.19 b/dev/deps/dependencies-client-flink-1.19
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.18//scala-library-2.12.18.jar
 scala-reflect/2.12.18//scala-reflect-2.12.18.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr
@@ -180,10 +180,6 @@ okhttp/4.9.3//okhttp-4.9.3.jar
 okio/2.8.0//okio-2.8.0.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 re2j/1.1//re2j-1.1.jar
 reload4j/1.2.22//reload4j-1.2.22.jar
 scala-library/2.12.18//scala-library-2.12.18.jar

diff --git a/dev/deps/dependencies-client-spark-2.4 b/dev/deps/dependencies-client-spark-2.4
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.11.12//scala-library-2.11.12.jar
 scala-reflect/2.11.12//scala-reflect-2.11.12.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-spark-3.0 b/dev/deps/dependencies-client-spark-3.0
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.10//scala-library-2.12.10.jar
 scala-reflect/2.12.10//scala-reflect-2.12.10.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-spark-3.1 b/dev/deps/dependencies-client-spark-3.1
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.10//scala-library-2.12.10.jar
 scala-reflect/2.12.10//scala-reflect-2.12.10.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-spark-3.2 b/dev/deps/dependencies-client-spark-3.2
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.15//scala-library-2.12.15.jar
 scala-reflect/2.12.15//scala-reflect-2.12.15.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-spark-3.3 b/dev/deps/dependencies-client-spark-3.3
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.15//scala-library-2.12.15.jar
 scala-reflect/2.12.15//scala-reflect-2.12.15.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-spark-3.4 b/dev/deps/dependencies-client-spark-3.4
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.17//scala-library-2.12.17.jar
 scala-reflect/2.12.17//scala-reflect-2.12.17.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/dev/deps/dependencies-client-spark-3.5 b/dev/deps/dependencies-client-spark-3.5
@@ -73,10 +73,6 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
 netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
 paranamer/2.8//paranamer-2.8.jar
 protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
 scala-library/2.12.18//scala-library-2.12.18.jar
 scala-reflect/2.12.18//scala-reflect-2.12.18.jar
 slf4j-api/1.7.36//slf4j-api-1.7.36.jar

diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
@@ -438,8 +438,6 @@ object CelebornCommon {
         Dependencies.hadoopClientApi,
         Dependencies.hadoopClientRuntime,
         Dependencies.jdkTools,
-        Dependencies.ratisClient,
-        Dependencies.ratisCommon,
         Dependencies.leveldbJniAll,
         Dependencies.roaringBitmap,
         Dependencies.scalaReflect,
@@ -569,11 +567,6 @@ object CelebornWorker {
     .dependsOn(CelebornMaster.master % "test->compile")
     .settings (
       commonSettings,
-      excludeDependencies ++= Seq(
-        // ratis-common/ratis-client are the transitive dependencies from celeborn-common
-        ExclusionRule("org.apache.ratis", "ratis-common"),
-        ExclusionRule("org.apache.ratis", "ratis-client")
-      ),
       libraryDependencies ++= Seq(
         Dependencies.apLoader,
         Dependencies.guava,

diff --git a/worker/pom.xml b/worker/pom.xml
@@ -32,16 +32,6 @@
       <groupId>org.apache.celeborn</groupId>
       <artifactId>celeborn-common_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.ratis</groupId>
-          <artifactId>ratis-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.ratis</groupId>
-          <artifactId>ratis-client</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.celeborn</groupId>