pass skip_credential_subscoping_indirection param to TaskFileIOSupplier

apache · Oct 23, 2024 · 09d3b09 · 09d3b09
1 parent 02468e0
commit 09d3b09
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 25 deletions.
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java
@@ -103,6 +103,22 @@ public static <T> Builder<T> builder() {
               .defaultValue(false)
               .build();
 
+  // Config key for whether to skip credential-subscoping indirection entirely whenever trying
+  // to obtain storage credentials for instantiating a FileIO. If 'true', no attempt is made
+  // to use StorageConfigs to generate table-specific storage credentials, but instead the default
+  // fallthrough of table-level credential properties or else provider-specific APPLICATION_DEFAULT
+  // credential-loading will be used for the FileIO.
+  // Typically this setting is used in single-tenant server deployments that don't rely on
+  // "credential-vending" and can use server-default environment variables or credential config
+  // files for all storage access, or in test/dev scenarios.
+  public static final Boolean SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT = false;
+  public static final PolarisConfiguration<Boolean> SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION =
+          PolarisConfiguration.<Boolean>builder()
+              .key("SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION")
+              .description("If set to true, skip credential-subscoping indirection and use the default credentials")
+              .defaultValue(SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT)
+              .build();
+
   public static final PolarisConfiguration<Boolean> ALLOW_TABLE_LOCATION_OVERLAP =
       PolarisConfiguration.<Boolean>builder()
           .key("ALLOW_TABLE_LOCATION_OVERLAP")

diff --git a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
@@ -71,6 +71,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.iceberg.rest.RESTSerializers;
+import org.apache.polaris.core.PolarisConfiguration;
 import org.apache.polaris.core.PolarisConfigurationStore;
 import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal;
 import org.apache.polaris.core.auth.PolarisAuthorizer;
@@ -207,11 +208,12 @@ public void run(PolarisApplicationConfig configuration, Environment environment)
       csAware.setConfigurationStore(configurationStore);
     }
 
+    Boolean skipCredentialSubscopingIndirection = configurationStore.getConfiguration(null, PolarisConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION.key);
     TaskHandlerConfiguration taskConfig = configuration.getTaskHandler();
     TaskExecutorImpl taskExecutor =
         new TaskExecutorImpl(taskConfig.executorService(), metaStoreManagerFactory);
     TaskFileIOSupplier fileIOSupplier =
-        new TaskFileIOSupplier(metaStoreManagerFactory, fileIOFactory);
+        new TaskFileIOSupplier(metaStoreManagerFactory, fileIOFactory, skipCredentialSubscopingIndirection);
     taskExecutor.addTaskHandler(
         new TableCleanupTaskHandler(taskExecutor, metaStoreManagerFactory, fileIOSupplier));
     taskExecutor.addTaskHandler(

diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
@@ -121,18 +121,6 @@ public class BasePolarisCatalog extends BaseMetastoreViewCatalog
   static final String ALLOW_SPECIFYING_FILE_IO_IMPL = "ALLOW_SPECIFYING_FILE_IO_IMPL";
   static final boolean ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT = false;
 
-  // Config key for whether to skip credential-subscoping indirection entirely whenever trying
-  // to obtain storage credentials for instantiating a FileIO. If 'true', no attempt is made
-  // to use StorageConfigs to generate table-specific storage credentials, but instead the default
-  // fallthrough of table-level credential properties or else provider-specific APPLICATION_DEFAULT
-  // credential-loading will be used for the FileIO.
-  // Typically this setting is used in single-tenant server deployments that don't rely on
-  // "credential-vending" and can use server-default environment variables or credential config
-  // files for all storage access, or in test/dev scenarios.
-  static final String SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION =
-      "SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION";
-  static final boolean SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT = false;
-
   // Config key for initializing a default "catalogFileIO" that is available either via getIo()
   // or for any TableOperations/ViewOperations instantiated, via ops.io() before entity-specific
   // FileIO initialization is triggered for any such operations.
@@ -868,7 +856,7 @@ private Map<String, String> refreshCredentials(
       PolarisEntity entity) {
     Boolean skipCredentialSubscopingIndirection =
         getBooleanContextConfiguration(
-            SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION, SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT);
+            PolarisConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION.key, PolarisConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT);
     if (Boolean.TRUE.equals(skipCredentialSubscopingIndirection)) {
       LOGGER
           .atInfo()

diff --git a/polaris-service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java b/polaris-service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java
@@ -34,11 +34,13 @@
 public class TaskFileIOSupplier implements Function<TaskEntity, FileIO> {
   private final MetaStoreManagerFactory metaStoreManagerFactory;
   private final FileIOFactory fileIOFactory;
+  private final Boolean skipCredentialSubscopingIndirection;
 
   public TaskFileIOSupplier(
-      MetaStoreManagerFactory metaStoreManagerFactory, FileIOFactory fileIOFactory) {
+      MetaStoreManagerFactory metaStoreManagerFactory, FileIOFactory fileIOFactory, Boolean skipCredentialSubscopingIndirection) {
     this.metaStoreManagerFactory = metaStoreManagerFactory;
     this.fileIOFactory = fileIOFactory;
+    this.skipCredentialSubscopingIndirection = skipCredentialSubscopingIndirection;
   }
 
   @Override
@@ -49,16 +51,18 @@ public FileIO apply(TaskEntity task) {
         metaStoreManagerFactory.getOrCreateMetaStoreManager(
             CallContext.getCurrentContext().getRealmContext());
     Map<String, String> properties = new HashMap<>(internalProperties);
-    properties.putAll(
-        metaStoreManagerFactory
-            .getOrCreateStorageCredentialCache(CallContext.getCurrentContext().getRealmContext())
-            .getOrGenerateSubScopeCreds(
-                metaStoreManager,
-                CallContext.getCurrentContext().getPolarisCallContext(),
-                task,
-                true,
-                Set.of(location),
-                Set.of(location)));
+    if (!skipCredentialSubscopingIndirection) {
+      properties.putAll(
+              metaStoreManagerFactory
+                      .getOrCreateStorageCredentialCache(CallContext.getCurrentContext().getRealmContext())
+                      .getOrGenerateSubScopeCreds(
+                              metaStoreManager,
+                              CallContext.getCurrentContext().getPolarisCallContext(),
+                              task,
+                              true,
+                              Set.of(location),
+                              Set.of(location)));
+    }
     String ioImpl =
         properties.getOrDefault(
             CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO");