diff --git a/src/azure_blob_filesystem.cpp b/src/azure_blob_filesystem.cpp index 4050960..9aec06b 100644 --- a/src/azure_blob_filesystem.cpp +++ b/src/azure_blob_filesystem.cpp @@ -90,7 +90,9 @@ unique_ptr AzureBlobStorageFileSystem::CreateHandle(const strin auto handle = make_uniq(*this, path, flags, storage_context->read_options, std::move(blob_client)); - handle->PostConstruct(); + if (!handle->PostConstruct()) { + return nullptr; + } return std::move(handle); } diff --git a/src/azure_dfs_filesystem.cpp b/src/azure_dfs_filesystem.cpp index 27966e3..8cde847 100644 --- a/src/azure_dfs_filesystem.cpp +++ b/src/azure_dfs_filesystem.cpp @@ -108,7 +108,9 @@ unique_ptr AzureDfsStorageFileSystem::CreateHandle(const string auto handle = make_uniq(*this, path, flags, storage_context->read_options, file_system_client.GetFileClient(parsed_url.path)); - handle->PostConstruct(); + if (!handle->PostConstruct()) { + return nullptr; + } return std::move(handle); } diff --git a/src/azure_filesystem.cpp b/src/azure_filesystem.cpp index 8a6fcaa..c3d2de1 100644 --- a/src/azure_filesystem.cpp +++ b/src/azure_filesystem.cpp @@ -31,17 +31,25 @@ AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, string path, FileOp if (flags.OpenForReading()) { read_buffer = duckdb::unique_ptr(new data_t[read_options.buffer_size]); } + if (flags.RequireParallelAccess()) { + // use direct i/o if parallel access is required + flags |= FileOpenFlags(FileOpenFlags::FILE_FLAGS_DIRECT_IO); + } } -void AzureFileHandle::PostConstruct() { - static_cast(file_system).LoadFileInfo(*this); +bool AzureFileHandle::PostConstruct() { + return static_cast(file_system).LoadFileInfo(*this); } -void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { +bool AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { if (handle.flags.OpenForReading()) { try { LoadRemoteFileInfo(handle); } catch (const Azure::Storage::StorageException &e) { + auto status_code = int(e.StatusCode); + if (status_code == 404 && handle.flags.ReturnNullIfNotExists()) { + return false; + } throw IOException( "AzureBlobStorageFileSystem open file '%s' failed with code'%s', Reason Phrase: '%s', Message: '%s'", handle.path, e.ErrorCode, e.ReasonPhrase, e.Message); @@ -52,6 +60,7 @@ void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { handle.path, e.what()); } } + return true; } unique_ptr AzureStorageFileSystem::OpenFile(const string &path, FileOpenFlags flags, @@ -162,11 +171,10 @@ shared_ptr AzureStorageFileSystem::GetOrCreateStorageContext( if (FileOpener::TryGetCurrentSetting(opener, "azure_context_caching", value)) { azure_context_caching = value.GetValue(); } + auto client_context = FileOpener::TryGetClientContext(opener); shared_ptr result; - if (azure_context_caching) { - auto client_context = FileOpener::TryGetClientContext(opener); - + if (azure_context_caching && client_context) { auto context_key = GetContextPrefix() + parsed_url.storage_account_name; auto ®istered_state = client_context->registered_state; diff --git a/src/azure_storage_account_client.cpp b/src/azure_storage_account_client.cpp index 7ea6373..6318c7b 100644 --- a/src/azure_storage_account_client.cpp +++ b/src/azure_storage_account_client.cpp @@ -559,11 +559,11 @@ static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(opti } const SecretMatch LookupSecret(optional_ptr opener, const std::string &path) { - auto context = opener->TryGetClientContext(); + auto secret_manager = FileOpener::TryGetSecretManager(opener); + auto transaction = FileOpener::TryGetCatalogTransaction(opener); - if (context) { - auto transaction = CatalogTransaction::GetSystemCatalogTransaction(*context); - return context->db->config.secret_manager->LookupSecret(transaction, path, "azure"); + if (secret_manager && transaction) { + return secret_manager->LookupSecret(*transaction, path, "azure"); } return {}; diff --git a/src/include/azure_filesystem.hpp b/src/include/azure_filesystem.hpp index 02ed44b..7450576 100644 --- a/src/include/azure_filesystem.hpp +++ b/src/include/azure_filesystem.hpp @@ -48,7 +48,7 @@ class AzureStorageFileSystem; class AzureFileHandle : public FileHandle { public: - virtual void PostConstruct(); + virtual bool PostConstruct(); void Close() override { } @@ -56,7 +56,7 @@ class AzureFileHandle : public FileHandle { AzureFileHandle(AzureStorageFileSystem &fs, string path, FileOpenFlags flags, const AzureReadOptions &read_options); public: - const FileOpenFlags flags; + FileOpenFlags flags; // File info idx_t length; @@ -96,7 +96,7 @@ class AzureStorageFileSystem : public FileSystem { void Seek(FileHandle &handle, idx_t location) override; void FileSync(FileHandle &handle) override; - void LoadFileInfo(AzureFileHandle &handle); + bool LoadFileInfo(AzureFileHandle &handle); protected: virtual duckdb::unique_ptr CreateHandle(const string &path, FileOpenFlags flags,