add gcp default credential handling for delta table format

dlt/common/configuration/specs/gcp_credentials.py

@@ -52,7 +52,7 @@ def __str__(self) -> str:
 
     def to_gcs_credentials(self) -> Dict[str, Any]:
         """
-        Dict of keyword arguments can be passed to gcsfs.
+        Dict of keyword arguments that can be passed to gcsfs.
         Delegates default GCS credential handling to gcsfs.
         """
         return {
@@ -64,6 +64,15 @@ def to_gcs_credentials(self) -> Dict[str, Any]:
             ),
         }
 
+    def to_object_store_rs_credentials(self) -> Dict[str, str]:
+        """
+        Dict of keyword arguments that can be passed to `object_store` Rust crate.
+        Delegates default GCS credential handling to `object_store` Rust crate.
+        """
+        if isinstance(self, CredentialsWithDefault) and self.has_default_credentials():
+            return {}
+        return {"service_account_key": json.dumps(dict(self))}
+
 
 @configspec
 class GcpServiceAccountCredentialsWithoutDefaults(GcpCredentials):
@@ -117,10 +126,6 @@ def to_native_credentials(self) -> Any:
         else:
             return ServiceAccountCredentials.from_service_account_info(self)
 
-    def to_object_store_rs_credentials(self) -> Dict[str, str]:
-        # https://docs.rs/object_store/latest/object_store/gcp
-        return {"service_account_key": json.dumps(dict(self))}
-
     def __str__(self) -> str:
         return f"{self.client_email}@{self.project_id}"
 

tests/load/filesystem/test_object_store_rs_credentials.py

@@ -2,6 +2,7 @@
 
 from typing import Any, Dict
 
+import json  # noqa: I251
 import pytest
 from deltalake import DeltaTable
 from deltalake.exceptions import TableNotFoundError
@@ -14,9 +15,13 @@
     AzureCredentialsWithoutDefaults,
     AwsCredentials,
     AwsCredentialsWithoutDefaults,
+    GcpCredentials,
     GcpServiceAccountCredentialsWithoutDefaults,
     GcpOAuthCredentialsWithoutDefaults,
 )
+from dlt.common.utils import custom_environ
+from dlt.common.configuration.resolve import resolve_configuration
+from dlt.common.configuration.specs.gcp_credentials import GcpDefaultCredentials
 from dlt.common.configuration.specs.exceptions import ObjectStoreRsCredentialsException
 
 from tests.load.utils import (
@@ -169,6 +174,9 @@ def test_aws_object_store_rs_credentials(driver: str) -> None:
     "driver", [driver for driver in ALL_FILESYSTEM_DRIVERS if driver in ("gs")]
 )
 def test_gcp_object_store_rs_credentials(driver) -> None:
+    creds: GcpCredentials
+
+    # GcpServiceAccountCredentialsWithoutDefaults
     creds = GcpServiceAccountCredentialsWithoutDefaults(
         project_id=FS_CREDS["project_id"],
         private_key=FS_CREDS["private_key"],
@@ -178,6 +186,17 @@ def test_gcp_object_store_rs_credentials(driver) -> None:
     )
     assert can_connect(GCS_BUCKET, creds.to_object_store_rs_credentials())
 
+    # GcpDefaultCredentials
+    # write service account key to JSON file
+    service_json = json.loads(creds.to_object_store_rs_credentials()["service_account_key"])
+    path = "_secrets/service.json"
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(service_json, f)
+    with custom_environ({"GOOGLE_APPLICATION_CREDENTIALS": path}):
+        creds = GcpDefaultCredentials()
+        resolve_configuration(creds)
+        can_connect(GCS_BUCKET, creds.to_object_store_rs_credentials())
+
     # GcpOAuthCredentialsWithoutDefaults is currently not supported
     with pytest.raises(NotImplementedError):
         GcpOAuthCredentialsWithoutDefaults().to_object_store_rs_credentials()