[apache#6165] feat(core): Use Gravitino cloud jar without hadoop pack…

…ages for Iceberg REST server (apache#6168) ### What changes were proposed in this pull request? 1. use Gravitino cloud jar without hadoop packages for Iceberg REST server credential vending in test and document 2. For OSS, use Gravitino Aliyun bundle jar in test and docker image because Iceberg doesn't provide Iceberg Aliyun bundle jar ### Why are the changes needed? Fix: apache#6165 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? test S3 OSS GCS ADLS Iceberg REST test in local
Abyss-lord · Jan 10, 2025 · 74ef660 · 74ef660
1 parent 20bfeba
commit 74ef660
Show file tree

Hide file tree

Showing 10 changed files with 41 additions and 23 deletions.
diff --git a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
@@ -35,17 +35,18 @@ tar xfz gravitino-iceberg-rest-server-*.tar.gz
 cp -r gravitino-iceberg-rest-server*-bin ${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server
 
 cd ${gravitino_home}
-./gradlew :bundles:gcp-bundle:jar
-./gradlew :bundles:aws-bundle:jar
-./gradlew :bundles:azure-bundle:jar
+./gradlew :bundles:gcp:jar
+./gradlew :bundles:aws:jar
+./gradlew :bundles:azure:jar
+## Iceberg doesn't provide Iceberg Aliyun bundle jar, so use Gravitino aliyun bundle to provide OSS packages.
 ./gradlew :bundles:aliyun-bundle:jar
 
 # prepare bundle jar
 cd ${iceberg_rest_server_dir}
 mkdir -p bundles
-cp ${gravitino_home}/bundles/gcp-bundle/build/libs/gravitino-gcp-bundle-*.jar bundles/
-cp ${gravitino_home}/bundles/aws-bundle/build/libs/gravitino-aws-bundle-*.jar bundles/
-cp ${gravitino_home}/bundles/azure-bundle/build/libs/gravitino-azure-bundle-*.jar bundles/
+cp ${gravitino_home}/bundles/gcp/build/libs/gravitino-gcp-*.jar bundles/
+cp ${gravitino_home}/bundles/aws/build/libs/gravitino-aws-*.jar bundles/
+cp ${gravitino_home}/bundles/azure/build/libs/gravitino-azure-*.jar bundles/
 cp ${gravitino_home}/bundles/aliyun-bundle/build/libs/gravitino-aliyun-bundle-*.jar bundles/
 
 iceberg_gcp_bundle="iceberg-gcp-bundle-1.5.2.jar"
@@ -63,11 +64,6 @@ if [ ! -f "bundles/${iceberg_azure_bundle}" ]; then
   curl -L -s -o bundles/${iceberg_azure_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/1.5.2/${iceberg_azure_bundle}
 fi
 
-iceberg_aliyun_bundle="iceberg-aliyun-bundle-1.5.2.jar"
-if [ ! -f "bundles/${iceberg_aliyun_bundle}" ]; then
-  curl -L -s -o bundles/${iceberg_aliyun_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aliyun-bundle/1.5.2/${iceberg_aliyun_bundle}
-fi
-
 # download jdbc driver
 curl -L -s -o bundles/sqlite-jdbc-3.42.0.0.jar https://repo1.maven.org/maven2/org/xerial/sqlite-jdbc/3.42.0.0/sqlite-jdbc-3.42.0.0.jar
 

diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
@@ -134,8 +134,14 @@ For other Iceberg OSS properties not managed by Gravitino like `client.security-
 
 Please refer to [OSS credentials](./security/credential-vending.md#oss-credentials) for credential related configurations.
 
+Additionally, Iceberg doesn't provide Iceberg Aliyun bundle jar which contains OSS packages, there are two alternatives to use OSS packages:
+1. Use [Gravitino Aliyun bundle jar with hadoop packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle).
+2. Use [Aliyun JAVA SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and extract `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` jars.
+
+Please place the above jars in the classpath of Iceberg REST server, please refer to [server management](#server-management) for classpath details.
+
 :::info
-Please set the `gravitino.iceberg-rest.warehouse` parameter to `oss://{bucket_name}/${prefix_name}`. Additionally, download the [Aliyun OSS SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and copy `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` in the classpath of Iceberg REST server, `iceberg-rest-server/libs` for the auxiliary server, `libs` for the standalone server.
+Please set the `gravitino.iceberg-rest.warehouse` parameter to `oss://{bucket_name}/${prefix_name}`. 
 :::
 
 #### GCS

diff --git a/docs/security/credential-vending.md b/docs/security/credential-vending.md
@@ -132,12 +132,23 @@ Gravitino supports custom credentials, you can implement the `org.apache.graviti
 
 Besides setting credentials related configuration, please download Gravitino cloud bundle jar and place it in the classpath of Iceberg REST server or Hadoop catalog.
 
-Gravitino cloud bundle jar:
+For Hadoop catalog, please use Gravitino cloud bundle jar with Hadoop and cloud packages:
 
-- [Gravitino AWS bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws-bundle)
-- [Gravitino Aliyun bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle)
-- [Gravitino GCP bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-gcp-bundle)
-- [Gravitino Azure bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-azure-bundle)
+- [Gravitino AWS bundle jar with Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws-bundle)
+- [Gravitino Aliyun bundle jar with Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle)
+- [Gravitino GCP bundle jar with Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-gcp-bundle)
+- [Gravitino Azure bundle jar with Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-azure-bundle)
+
+For Iceberg REST catalog server, please use Gravitino cloud bundle jar without Hadoop and cloud packages:
+
+- [Gravitino AWS bundle jar without Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws)
+- [Gravitino Aliyun bundle jar without Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun)
+- [Gravitino GCP bundle jar without Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-gcp)
+- [Gravitino Azure bundle jar without Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-azure)
+
+:::note
+For OSS, Iceberg doesn't provide Iceberg Aliyun bundle jar which contains OSS packages, you could provide the OSS jar by yourself or use [Gravitino Aliyun bundle jar with Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle), please refer to [OSS configuration](../iceberg-rest-service.md#oss-configuration) for more details.
+:::
 
 The classpath of the server:
 

diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts
@@ -62,7 +62,8 @@ dependencies {
   annotationProcessor(libs.lombok)
   compileOnly(libs.lombok)
 
-  testImplementation(project(":bundles:aliyun"))
+  // Iceberg doesn't provide Aliyun bundle jar, use Gravitino Aliyun bundle to provide OSS packages
+  testImplementation(project(":bundles:aliyun-bundle"))
   testImplementation(project(":bundles:aws"))
   testImplementation(project(":bundles:gcp", configuration = "shadow"))
   testImplementation(project(":bundles:azure", configuration = "shadow"))

diff --git a/...r/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java b/...r/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
@@ -130,7 +130,7 @@ private void downloadIcebergAzureBundleJar() throws IOException {
   private void copyAzureBundleJar() {
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
-    BaseIT.copyBundleJarsToDirectory("azure-bundle", targetDir);
+    BaseIT.copyBundleJarsToDirectory("azure", targetDir);
   }
 
   @Test

diff --git a/...test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java b/...test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
@@ -113,6 +113,6 @@ private void downloadIcebergAzureBundleJar() throws IOException {
   private void copyAzureBundleJar() {
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
-    BaseIT.copyBundleJarsToDirectory("azure-bundle", targetDir);
+    BaseIT.copyBundleJarsToDirectory("azure", targetDir);
   }
 }
diff --git a/...-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java b/...-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
@@ -88,7 +88,7 @@ private Map<String, String> getGCSConfig() {
   private void copyGCSBundleJar() {
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
-    BaseIT.copyBundleJarsToDirectory("gcp-bundle", targetDir);
+    BaseIT.copyBundleJarsToDirectory("gcp", targetDir);
   }
 
   private void downloadIcebergBundleJar() throws IOException {

diff --git a/...-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java b/...-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
@@ -126,6 +126,8 @@ private void downloadIcebergForAliyunJar() throws IOException {
   private void copyAliyunOSSJar() {
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
+    // Iceberg doesn't provide Iceberg Aliyun bundle jar, so use Gravitino aliyun bundle to provide
+    // OSS packages.
     BaseIT.copyBundleJarsToDirectory("aliyun-bundle", targetDir);
   }
 }
diff --git a/...r/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java b/...r/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
@@ -111,6 +111,8 @@ private void downloadIcebergForAliyunJar() throws IOException {
   private void copyAliyunOSSJar() {
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
+    // Iceberg doesn't provide Iceberg Aliyun bundle jar, so use Gravitino aliyun bundle to provide
+    // OSS packages.
     BaseIT.copyBundleJarsToDirectory("aliyun-bundle", targetDir);
   }
 }
diff --git a/...erg/integration/test/IcebergRESTS3IT.java → ...ntegration/test/IcebergRESTS3TokenIT.java b/...erg/integration/test/IcebergRESTS3IT.java → ...ntegration/test/IcebergRESTS3TokenIT.java
@@ -40,7 +40,7 @@
 
 @SuppressWarnings("FormatStringAnnotation")
 @EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true")
-public class IcebergRESTS3IT extends IcebergRESTJdbcCatalogIT {
+public class IcebergRESTS3TokenIT extends IcebergRESTJdbcCatalogIT {
 
   private String s3Warehouse;
   private String accessKey;
@@ -124,7 +124,7 @@ private void downloadIcebergAwsBundleJar() throws IOException {
   private void copyS3BundleJar() {
     String gravitinoHome = System.getenv("GRAVITINO_HOME");
     String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome);
-    BaseIT.copyBundleJarsToDirectory("aws-bundle", targetDir);
+    BaseIT.copyBundleJarsToDirectory("aws", targetDir);
   }
 
   /**