From 09d3b0938a3570c6cb9fbd3aaf7968175f27d114 Mon Sep 17 00:00:00 2001 From: Alessandro Nori Date: Wed, 23 Oct 2024 10:47:37 +0200 Subject: [PATCH] pass skip_credential_subscoping_indirection param to TaskFileIOSupplier --- .../polaris/core/PolarisConfiguration.java | 16 ++++++++++++ .../polaris/service/PolarisApplication.java | 4 ++- .../service/catalog/BasePolarisCatalog.java | 14 +--------- .../service/task/TaskFileIOSupplier.java | 26 +++++++++++-------- 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 397c9afd9..ded078e67 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -103,6 +103,22 @@ public static Builder builder() { .defaultValue(false) .build(); + // Config key for whether to skip credential-subscoping indirection entirely whenever trying + // to obtain storage credentials for instantiating a FileIO. If 'true', no attempt is made + // to use StorageConfigs to generate table-specific storage credentials, but instead the default + // fallthrough of table-level credential properties or else provider-specific APPLICATION_DEFAULT + // credential-loading will be used for the FileIO. + // Typically this setting is used in single-tenant server deployments that don't rely on + // "credential-vending" and can use server-default environment variables or credential config + // files for all storage access, or in test/dev scenarios. + public static final Boolean SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT = false; + public static final PolarisConfiguration SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION = + PolarisConfiguration.builder() + .key("SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION") + .description("If set to true, skip credential-subscoping indirection and use the default credentials") + .defaultValue(SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT) + .build(); + public static final PolarisConfiguration ALLOW_TABLE_LOCATION_OVERLAP = PolarisConfiguration.builder() .key("ALLOW_TABLE_LOCATION_OVERLAP") diff --git a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java index 143d3dae1..d31578588 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java @@ -71,6 +71,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.iceberg.rest.RESTSerializers; +import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; @@ -207,11 +208,12 @@ public void run(PolarisApplicationConfig configuration, Environment environment) csAware.setConfigurationStore(configurationStore); } + Boolean skipCredentialSubscopingIndirection = configurationStore.getConfiguration(null, PolarisConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION.key); TaskHandlerConfiguration taskConfig = configuration.getTaskHandler(); TaskExecutorImpl taskExecutor = new TaskExecutorImpl(taskConfig.executorService(), metaStoreManagerFactory); TaskFileIOSupplier fileIOSupplier = - new TaskFileIOSupplier(metaStoreManagerFactory, fileIOFactory); + new TaskFileIOSupplier(metaStoreManagerFactory, fileIOFactory, skipCredentialSubscopingIndirection); taskExecutor.addTaskHandler( new TableCleanupTaskHandler(taskExecutor, metaStoreManagerFactory, fileIOSupplier)); taskExecutor.addTaskHandler( diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 1f08f6bb1..6e9771bc8 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -121,18 +121,6 @@ public class BasePolarisCatalog extends BaseMetastoreViewCatalog static final String ALLOW_SPECIFYING_FILE_IO_IMPL = "ALLOW_SPECIFYING_FILE_IO_IMPL"; static final boolean ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT = false; - // Config key for whether to skip credential-subscoping indirection entirely whenever trying - // to obtain storage credentials for instantiating a FileIO. If 'true', no attempt is made - // to use StorageConfigs to generate table-specific storage credentials, but instead the default - // fallthrough of table-level credential properties or else provider-specific APPLICATION_DEFAULT - // credential-loading will be used for the FileIO. - // Typically this setting is used in single-tenant server deployments that don't rely on - // "credential-vending" and can use server-default environment variables or credential config - // files for all storage access, or in test/dev scenarios. - static final String SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION = - "SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"; - static final boolean SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT = false; - // Config key for initializing a default "catalogFileIO" that is available either via getIo() // or for any TableOperations/ViewOperations instantiated, via ops.io() before entity-specific // FileIO initialization is triggered for any such operations. @@ -868,7 +856,7 @@ private Map refreshCredentials( PolarisEntity entity) { Boolean skipCredentialSubscopingIndirection = getBooleanContextConfiguration( - SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION, SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT); + PolarisConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION.key, PolarisConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION_DEFAULT); if (Boolean.TRUE.equals(skipCredentialSubscopingIndirection)) { LOGGER .atInfo() diff --git a/polaris-service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java b/polaris-service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java index c84eebd9f..c0881bf3d 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java @@ -34,11 +34,13 @@ public class TaskFileIOSupplier implements Function { private final MetaStoreManagerFactory metaStoreManagerFactory; private final FileIOFactory fileIOFactory; + private final Boolean skipCredentialSubscopingIndirection; public TaskFileIOSupplier( - MetaStoreManagerFactory metaStoreManagerFactory, FileIOFactory fileIOFactory) { + MetaStoreManagerFactory metaStoreManagerFactory, FileIOFactory fileIOFactory, Boolean skipCredentialSubscopingIndirection) { this.metaStoreManagerFactory = metaStoreManagerFactory; this.fileIOFactory = fileIOFactory; + this.skipCredentialSubscopingIndirection = skipCredentialSubscopingIndirection; } @Override @@ -49,16 +51,18 @@ public FileIO apply(TaskEntity task) { metaStoreManagerFactory.getOrCreateMetaStoreManager( CallContext.getCurrentContext().getRealmContext()); Map properties = new HashMap<>(internalProperties); - properties.putAll( - metaStoreManagerFactory - .getOrCreateStorageCredentialCache(CallContext.getCurrentContext().getRealmContext()) - .getOrGenerateSubScopeCreds( - metaStoreManager, - CallContext.getCurrentContext().getPolarisCallContext(), - task, - true, - Set.of(location), - Set.of(location))); + if (!skipCredentialSubscopingIndirection) { + properties.putAll( + metaStoreManagerFactory + .getOrCreateStorageCredentialCache(CallContext.getCurrentContext().getRealmContext()) + .getOrGenerateSubScopeCreds( + metaStoreManager, + CallContext.getCurrentContext().getPolarisCallContext(), + task, + true, + Set.of(location), + Set.of(location))); + } String ioImpl = properties.getOrDefault( CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO");