diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 57295d9..7960ac0 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,20 +14,20 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.1.1 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.1.2 with: extension_name: azure - duckdb_version: v1.1.1 - ci_tools_version: v1.1.1 + duckdb_version: v1.1.2 + ci_tools_version: v1.1.2 exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' duckdb-stable-deploy: name: Deploy extension binaries needs: duckdb-stable-build - uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.1.1 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.1.2 secrets: inherit with: extension_name: azure - duckdb_version: v1.1.1 + duckdb_version: v1.1.2 exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} diff --git a/duckdb b/duckdb index af39bd0..f680b7d 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit af39bd0dcf66876e09ac2a7c3baa28fe1b301151 +Subproject commit f680b7d08f56183391b581077d4baf589e1cc8bd diff --git a/extension-ci-tools b/extension-ci-tools index 00831df..f5594c6 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit 00831df06713072df217d3fb2f6b5e0fae78742f +Subproject commit f5594c61803daee122a5245afb817966e1a4545c diff --git a/src/azure_blob_filesystem.cpp b/src/azure_blob_filesystem.cpp index 4050960..9aec06b 100644 --- a/src/azure_blob_filesystem.cpp +++ b/src/azure_blob_filesystem.cpp @@ -90,7 +90,9 @@ unique_ptr AzureBlobStorageFileSystem::CreateHandle(const strin auto handle = make_uniq(*this, path, flags, storage_context->read_options, std::move(blob_client)); - handle->PostConstruct(); + if (!handle->PostConstruct()) { + return nullptr; + } return std::move(handle); } diff --git a/src/azure_dfs_filesystem.cpp b/src/azure_dfs_filesystem.cpp index 27966e3..8cde847 100644 --- a/src/azure_dfs_filesystem.cpp +++ b/src/azure_dfs_filesystem.cpp @@ -108,7 +108,9 @@ unique_ptr AzureDfsStorageFileSystem::CreateHandle(const string auto handle = make_uniq(*this, path, flags, storage_context->read_options, file_system_client.GetFileClient(parsed_url.path)); - handle->PostConstruct(); + if (!handle->PostConstruct()) { + return nullptr; + } return std::move(handle); } diff --git a/src/azure_filesystem.cpp b/src/azure_filesystem.cpp index 8a6fcaa..c3d2de1 100644 --- a/src/azure_filesystem.cpp +++ b/src/azure_filesystem.cpp @@ -31,17 +31,25 @@ AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, string path, FileOp if (flags.OpenForReading()) { read_buffer = duckdb::unique_ptr(new data_t[read_options.buffer_size]); } + if (flags.RequireParallelAccess()) { + // use direct i/o if parallel access is required + flags |= FileOpenFlags(FileOpenFlags::FILE_FLAGS_DIRECT_IO); + } } -void AzureFileHandle::PostConstruct() { - static_cast(file_system).LoadFileInfo(*this); +bool AzureFileHandle::PostConstruct() { + return static_cast(file_system).LoadFileInfo(*this); } -void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { +bool AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { if (handle.flags.OpenForReading()) { try { LoadRemoteFileInfo(handle); } catch (const Azure::Storage::StorageException &e) { + auto status_code = int(e.StatusCode); + if (status_code == 404 && handle.flags.ReturnNullIfNotExists()) { + return false; + } throw IOException( "AzureBlobStorageFileSystem open file '%s' failed with code'%s', Reason Phrase: '%s', Message: '%s'", handle.path, e.ErrorCode, e.ReasonPhrase, e.Message); @@ -52,6 +60,7 @@ void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { handle.path, e.what()); } } + return true; } unique_ptr AzureStorageFileSystem::OpenFile(const string &path, FileOpenFlags flags, @@ -162,11 +171,10 @@ shared_ptr AzureStorageFileSystem::GetOrCreateStorageContext( if (FileOpener::TryGetCurrentSetting(opener, "azure_context_caching", value)) { azure_context_caching = value.GetValue(); } + auto client_context = FileOpener::TryGetClientContext(opener); shared_ptr result; - if (azure_context_caching) { - auto client_context = FileOpener::TryGetClientContext(opener); - + if (azure_context_caching && client_context) { auto context_key = GetContextPrefix() + parsed_url.storage_account_name; auto ®istered_state = client_context->registered_state; diff --git a/src/azure_storage_account_client.cpp b/src/azure_storage_account_client.cpp index 7ea6373..6318c7b 100644 --- a/src/azure_storage_account_client.cpp +++ b/src/azure_storage_account_client.cpp @@ -559,11 +559,11 @@ static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(opti } const SecretMatch LookupSecret(optional_ptr opener, const std::string &path) { - auto context = opener->TryGetClientContext(); + auto secret_manager = FileOpener::TryGetSecretManager(opener); + auto transaction = FileOpener::TryGetCatalogTransaction(opener); - if (context) { - auto transaction = CatalogTransaction::GetSystemCatalogTransaction(*context); - return context->db->config.secret_manager->LookupSecret(transaction, path, "azure"); + if (secret_manager && transaction) { + return secret_manager->LookupSecret(*transaction, path, "azure"); } return {}; diff --git a/src/include/azure_filesystem.hpp b/src/include/azure_filesystem.hpp index 02ed44b..7450576 100644 --- a/src/include/azure_filesystem.hpp +++ b/src/include/azure_filesystem.hpp @@ -48,7 +48,7 @@ class AzureStorageFileSystem; class AzureFileHandle : public FileHandle { public: - virtual void PostConstruct(); + virtual bool PostConstruct(); void Close() override { } @@ -56,7 +56,7 @@ class AzureFileHandle : public FileHandle { AzureFileHandle(AzureStorageFileSystem &fs, string path, FileOpenFlags flags, const AzureReadOptions &read_options); public: - const FileOpenFlags flags; + FileOpenFlags flags; // File info idx_t length; @@ -96,7 +96,7 @@ class AzureStorageFileSystem : public FileSystem { void Seek(FileHandle &handle, idx_t location) override; void FileSync(FileHandle &handle) override; - void LoadFileInfo(AzureFileHandle &handle); + bool LoadFileInfo(AzureFileHandle &handle); protected: virtual duckdb::unique_ptr CreateHandle(const string &path, FileOpenFlags flags,