/vsigs/: make GetFileMetadata('/vsigs/bucket', NULL, NULL) work if using OAuth2 auth

autotest/gcore/vsigs.py

@@ -692,6 +692,107 @@ def test_vsigs_headers(gs_test_config, webserver_port):
             )
 
 
+###############################################################################
+# Test GetFileMetadata() on root of bucket with OAuth2
+
+
+@gdaltest.enable_exceptions()
+def test_vsigs_GetFileMetadatabucket_root_oauth2(
+    gs_test_config, webserver_port, tmp_vsimem
+):
+
+    gdal.VSICurlClearCache()
+
+    service_account_filename = str(tmp_vsimem / "service_account.json")
+    gdal.FileFromMemBuffer(
+        service_account_filename,
+        """{
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk\nVNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm\ni2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ\nDVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy\nTabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I\n2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd\nDrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+\nbpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg\nWz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3\n4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5\nKvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj\nj+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI\nyzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq\ngwE6fxOLyJDxuWRf\n-----END PRIVATE KEY-----\n",
+  "client_email": "CLIENT_EMAIL",
+  "type": "service_account"
+                           }""",
+    )
+
+    gdal.SetPathSpecificOption(
+        "/vsigs/gs_fake_bucket",
+        "GOOGLE_APPLICATION_CREDENTIALS",
+        service_account_filename,
+    )
+
+    try:
+        with gdaltest.config_options(
+            {
+                "GO2A_AUD": "http://localhost:%d/oauth2/v4/token" % webserver_port,
+                "GOA2_NOW": "123456",
+            },
+            thread_local=False,
+        ):
+
+            gdal.VSICurlClearCache()
+
+            handler = webserver.SequentialHandler()
+
+            def method(request):
+                request.send_response(200)
+                request.send_header("Content-type", "text/plain")
+                content = """{
+                        "access_token" : "ACCESS_TOKEN",
+                        "token_type" : "Bearer",
+                        "expires_in" : 3600,
+                        }"""
+                request.send_header("Content-Length", len(content))
+                request.end_headers()
+                request.wfile.write(content.encode("ascii"))
+
+            handler.add("POST", "/oauth2/v4/token", custom_method=method)
+
+            handler.add(
+                "GET",
+                "/storage/v1/b/gs_fake_bucket",
+                200,
+                {"Content-type": "application/json"},
+                '{"foo":"bar"}',
+            )
+            try:
+                with webserver.install_http_handler(handler):
+                    md = gdal.GetFileMetadata("/vsigs/gs_fake_bucket/", None)
+
+            except Exception:
+                if (
+                    gdal.GetLastErrorMsg().find("CPLRSASHA256Sign() not implemented")
+                    >= 0
+                ):
+                    pytest.skip("CPLRSASHA256Sign() not implemented")
+
+            assert md == {"foo": "bar"}
+    finally:
+        gdal.SetPathSpecificOption(
+            "/vsigs/gs_fake_bucket", "GOOGLE_APPLICATION_CREDENTIALS", None
+        )
+
+
+###############################################################################
+# Test GetFileMetadata() on root of bucket with non-OAuth2 (does not work)
+
+
+def test_vsigs_GetFileMetadatabucket_root_not_oauth2(gs_test_config, webserver_port):
+
+    gdal.VSICurlClearCache()
+
+    with gdaltest.config_options(
+        {
+            "GS_SECRET_ACCESS_KEY": "GS_SECRET_ACCESS_KEY",
+            "GS_ACCESS_KEY_ID": "GS_ACCESS_KEY_ID",
+        },
+        thread_local=False,
+    ):
+
+        handler = webserver.SequentialHandler()
+        with webserver.install_http_handler(handler):
+            md = gdal.GetFileMetadata("/vsigs/gs_fake_bucket/", None)
+        assert md == {}
+
+
 ###############################################################################
 # Read credentials with OAuth2 refresh_token
 

port/cpl_vsil.cpp

@@ -1231,24 +1231,35 @@ int VSIStatExL(const char *pszFilename, VSIStatBufL *psStatBuf, int nFlags)
  * Implemented currently only for network-like filesystems, or starting
  * with GDAL 3.7 for /vsizip/
  *
+ * Starting with GDAL 3.11, calling it with pszFilename being the root of a
+ * /vsigs/ bucket and pszDomain == nullptr, and when authenticated through
+ * OAuth2, will result in returning the result of a "Buckets: get"
+ * operation (https://cloud.google.com/storage/docs/json_api/v1/buckets/get),
+ * with the keys of the top-level JSON document as keys of the key=value pairs
+ * returned by this function.
+ *
  * @param pszFilename the path of the filesystem object to be queried.
  * UTF-8 encoded.
  * @param pszDomain Metadata domain to query. Depends on the file system.
- * The following are supported:
+ * The following ones are supported:
  * <ul>
  * <li>HEADERS: to get HTTP headers for network-like filesystems (/vsicurl/,
- * /vsis3/, /vsgis/, etc)</li> <li>TAGS: <ul> <li>/vsis3/: to get S3 Object
- * tagging information</li> <li>/vsiaz/: to get blob tags. Refer to
- * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags</li>
- *    </ul>
+ * /vsis3/, /vsgis/, etc)</li>
+ * <li>TAGS:
+ *   <ul>
+ *     <li>/vsis3/: to get S3 Object tagging information</li>
+ *     <li>/vsiaz/: to get blob tags. Refer to
+ *     https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags
+ *     </li>
+ *   </ul>
  * </li>
  * <li>STATUS: specific to /vsiadls/: returns all system defined properties for
  * a path (seems in practice to be a subset of HEADERS)</li> <li>ACL: specific
  * to /vsiadls/ and /vsigs/: returns the access control list for a path. For
  * /vsigs/, a single XML=xml_content string is returned. Refer to
  * https://cloud.google.com/storage/docs/xml-api/get-object-acls
  * </li>
- * <li>METADATA: specific to /vsiaz/: to set blob metadata. Refer to
+ * <li>METADATA: specific to /vsiaz/: to get blob metadata. Refer to
  * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-metadata.
  * Note: this will be a subset of what pszDomain=HEADERS returns</li>
  * <li>ZIP: specific to /vsizip/: to obtain ZIP specific metadata, in particular

port/cpl_vsil_gs.cpp

@@ -13,6 +13,7 @@
 #include "cpl_port.h"
 #include "cpl_http.h"
 #include "cpl_minixml.h"
+#include "cpl_json.h"
 #include "cpl_vsil_curl_priv.h"
 #include "cpl_vsil_curl_class.h"
 
@@ -331,6 +332,124 @@ char **VSIGSFSHandler::GetFileMetadata(const char *pszFilename,
     if (!STARTS_WITH_CI(pszFilename, GetFSPrefix().c_str()))
         return nullptr;
 
+    if (pszDomain == nullptr)
+    {
+        // Handle case of requesting GetFileMetadata() on the bucket root
+        std::string osFilename(pszFilename);
+        if (osFilename.back() == '/')
+            osFilename.pop_back();
+        if (osFilename.find('/', GetFSPrefix().size()) == std::string::npos)
+        {
+            const std::string osBucket =
+                osFilename.substr(GetFSPrefix().size());
+            const std::string osResource =
+                std::string("storage/v1/b/").append(osBucket);
+
+            auto poHandleHelper = std::unique_ptr<VSIGSHandleHelper>(
+                VSIGSHandleHelper::BuildFromURI(osResource.c_str(),
+                                                GetFSPrefix().c_str(),
+                                                osBucket.c_str()));
+            if (!poHandleHelper)
+                return nullptr;
+
+            // The JSON API cannot be used with HMAC keys
+            if (poHandleHelper->UsesHMACKey())
+            {
+                CPLDebug(GetDebugKey(),
+                         "GetFileMetadata() on bucket "
+                         "only available for OAuth2 authentication");
+                return VSICurlFilesystemHandlerBase::GetFileMetadata(
+                    pszFilename, pszDomain, papszOptions);
+            }
+
+            NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str());
+            NetworkStatisticsAction oContextAction("GetFileMetadata");
+
+            const CPLStringList aosHTTPOptions(
+                CPLHTTPGetOptionsFromEnv(pszFilename));
+            const CPLHTTPRetryParameters oRetryParameters(aosHTTPOptions);
+            CPLHTTPRetryContext oRetryContext(oRetryParameters);
+
+            bool bRetry;
+            CPLStringList aosResult;
+            do
+            {
+                bRetry = false;
+                CURL *hCurlHandle = curl_easy_init();
+
+                struct curl_slist *headers =
+                    static_cast<struct curl_slist *>(CPLHTTPSetOptions(
+                        hCurlHandle, poHandleHelper->GetURL().c_str(),
+                        aosHTTPOptions.List()));
+                headers = VSICurlMergeHeaders(
+                    headers, poHandleHelper->GetCurlHeaders("GET", headers));
+
+                CurlRequestHelper requestHelper;
+                const long response_code = requestHelper.perform(
+                    hCurlHandle, headers, this, poHandleHelper.get());
+
+                NetworkStatisticsLogger::LogGET(
+                    requestHelper.sWriteFuncData.nSize);
+
+                if (response_code != 200 ||
+                    requestHelper.sWriteFuncData.pBuffer == nullptr)
+                {
+                    // Look if we should attempt a retry
+                    if (oRetryContext.CanRetry(
+                            static_cast<int>(response_code),
+                            requestHelper.sWriteFuncHeaderData.pBuffer,
+                            requestHelper.szCurlErrBuf))
+                    {
+                        CPLError(CE_Warning, CPLE_AppDefined,
+                                 "HTTP error code: %d - %s. "
+                                 "Retrying again in %.1f secs",
+                                 static_cast<int>(response_code),
+                                 poHandleHelper->GetURL().c_str(),
+                                 oRetryContext.GetCurrentDelay());
+                        CPLSleep(oRetryContext.GetCurrentDelay());
+                        bRetry = true;
+                    }
+                    else
+                    {
+                        CPLDebug(GetDebugKey(), "%s",
+                                 requestHelper.sWriteFuncData.pBuffer
+                                     ? requestHelper.sWriteFuncData.pBuffer
+                                     : "(null)");
+                        CPLError(CE_Failure, CPLE_AppDefined,
+                                 "GetFileMetadata failed");
+                    }
+                }
+                else
+                {
+                    CPLJSONDocument oDoc;
+                    if (oDoc.LoadMemory(
+                            reinterpret_cast<const GByte *>(
+                                requestHelper.sWriteFuncData.pBuffer),
+                            static_cast<int>(
+                                requestHelper.sWriteFuncData.nSize)) &&
+                        oDoc.GetRoot().GetType() == CPLJSONObject::Type::Object)
+                    {
+                        for (const auto &oObj : oDoc.GetRoot().GetChildren())
+                        {
+                            aosResult.SetNameValue(oObj.GetName().c_str(),
+                                                   oObj.ToString().c_str());
+                        }
+                    }
+                    else
+                    {
+                        // Shouldn't happen normally
+                        aosResult.SetNameValue(
+                            "DATA", requestHelper.sWriteFuncData.pBuffer);
+                    }
+                }
+
+                curl_easy_cleanup(hCurlHandle);
+            } while (bRetry);
+
+            return aosResult.StealList();
+        }
+    }
+
     if (pszDomain == nullptr || !EQUAL(pszDomain, "ACL"))
     {
         return VSICurlFilesystemHandlerBase::GetFileMetadata(
@@ -404,7 +523,7 @@ char **VSIGSFSHandler::GetFileMetadata(const char *pszFilename,
 
         curl_easy_cleanup(hCurlHandle);
     } while (bRetry);
-    return CSLDuplicate(aosResult.List());
+    return aosResult.StealList();
 }
 
 /************************************************************************/