Merge pull request #86 from Mytherin/attach

Make ATTACH work over Azure

.github/workflows/MainDistributionPipeline.yml

@@ -14,20 +14,20 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/[email protected].1
+    uses: duckdb/extension-ci-tools/.github/workflows/[email protected].2
     with:
       extension_name: azure
-      duckdb_version: v1.1.1
-      ci_tools_version: v1.1.1
+      duckdb_version: v1.1.2
+      ci_tools_version: v1.1.2
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
 
   duckdb-stable-deploy:
     name: Deploy extension binaries
     needs: duckdb-stable-build
-    uses: duckdb/extension-ci-tools/.github/workflows/[email protected].1
+    uses: duckdb/extension-ci-tools/.github/workflows/[email protected].2
     secrets: inherit
     with:
       extension_name: azure
-      duckdb_version: v1.1.1
+      duckdb_version: v1.1.2
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

src/azure_blob_filesystem.cpp

@@ -90,7 +90,9 @@ unique_ptr<AzureFileHandle> AzureBlobStorageFileSystem::CreateHandle(const strin
 
 	auto handle = make_uniq<AzureBlobStorageFileHandle>(*this, path, flags, storage_context->read_options,
 	                                                    std::move(blob_client));
-	handle->PostConstruct();
+	if (!handle->PostConstruct()) {
+		return nullptr;
+	}
 	return std::move(handle);
 }
 

src/azure_dfs_filesystem.cpp

@@ -108,7 +108,9 @@ unique_ptr<AzureFileHandle> AzureDfsStorageFileSystem::CreateHandle(const string
 
 	auto handle = make_uniq<AzureDfsStorageFileHandle>(*this, path, flags, storage_context->read_options,
 	                                                   file_system_client.GetFileClient(parsed_url.path));
-	handle->PostConstruct();
+	if (!handle->PostConstruct()) {
+		return nullptr;
+	}
 	return std::move(handle);
 }
 

src/azure_filesystem.cpp

@@ -31,17 +31,25 @@ AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, string path, FileOp
 	if (flags.OpenForReading()) {
 		read_buffer = duckdb::unique_ptr<data_t[]>(new data_t[read_options.buffer_size]);
 	}
+	if (flags.RequireParallelAccess()) {
+		// use direct i/o if parallel access is required
+		flags |= FileOpenFlags(FileOpenFlags::FILE_FLAGS_DIRECT_IO);
+	}
 }
 
-void AzureFileHandle::PostConstruct() {
-	static_cast<AzureStorageFileSystem &>(file_system).LoadFileInfo(*this);
+bool AzureFileHandle::PostConstruct() {
+	return static_cast<AzureStorageFileSystem &>(file_system).LoadFileInfo(*this);
 }
 
-void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) {
+bool AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) {
 	if (handle.flags.OpenForReading()) {
 		try {
 			LoadRemoteFileInfo(handle);
 		} catch (const Azure::Storage::StorageException &e) {
+			auto status_code = int(e.StatusCode);
+			if (status_code == 404 && handle.flags.ReturnNullIfNotExists()) {
+				return false;
+			}
 			throw IOException(
 			    "AzureBlobStorageFileSystem open file '%s' failed with code'%s', Reason Phrase: '%s', Message: '%s'",
 			    handle.path, e.ErrorCode, e.ReasonPhrase, e.Message);
@@ -52,6 +60,7 @@ void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) {
 			    handle.path, e.what());
 		}
 	}
+	return true;
 }
 
 unique_ptr<FileHandle> AzureStorageFileSystem::OpenFile(const string &path, FileOpenFlags flags,
@@ -162,11 +171,10 @@ shared_ptr<AzureContextState> AzureStorageFileSystem::GetOrCreateStorageContext(
 	if (FileOpener::TryGetCurrentSetting(opener, "azure_context_caching", value)) {
 		azure_context_caching = value.GetValue<bool>();
 	}
+	auto client_context = FileOpener::TryGetClientContext(opener);
 
 	shared_ptr<AzureContextState> result;
-	if (azure_context_caching) {
-		auto client_context = FileOpener::TryGetClientContext(opener);
-
+	if (azure_context_caching && client_context) {
 		auto context_key = GetContextPrefix() + parsed_url.storage_account_name;
 
 		auto &registered_state = client_context->registered_state;

src/azure_storage_account_client.cpp

@@ -559,11 +559,11 @@ static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(opti
 }
 
 const SecretMatch LookupSecret(optional_ptr<FileOpener> opener, const std::string &path) {
-	auto context = opener->TryGetClientContext();
+	auto secret_manager = FileOpener::TryGetSecretManager(opener);
+	auto transaction = FileOpener::TryGetCatalogTransaction(opener);
 
-	if (context) {
-		auto transaction = CatalogTransaction::GetSystemCatalogTransaction(*context);
-		return context->db->config.secret_manager->LookupSecret(transaction, path, "azure");
+	if (secret_manager && transaction) {
+		return secret_manager->LookupSecret(*transaction, path, "azure");
 	}
 
 	return {};

src/include/azure_filesystem.hpp

@@ -48,15 +48,15 @@ class AzureStorageFileSystem;
 
 class AzureFileHandle : public FileHandle {
 public:
-	virtual void PostConstruct();
+	virtual bool PostConstruct();
 	void Close() override {
 	}
 
 protected:
 	AzureFileHandle(AzureStorageFileSystem &fs, string path, FileOpenFlags flags, const AzureReadOptions &read_options);
 
 public:
-	const FileOpenFlags flags;
+	FileOpenFlags flags;
 
 	// File info
 	idx_t length;
@@ -96,7 +96,7 @@ class AzureStorageFileSystem : public FileSystem {
 	void Seek(FileHandle &handle, idx_t location) override;
 	void FileSync(FileHandle &handle) override;
 
-	void LoadFileInfo(AzureFileHandle &handle);
+	bool LoadFileInfo(AzureFileHandle &handle);
 
 protected:
 	virtual duckdb::unique_ptr<AzureFileHandle> CreateHandle(const string &path, FileOpenFlags flags,