Skip to content

Commit

Permalink
Merge pull request #57 from carlopi/patch
Browse files Browse the repository at this point in the history 
Apply duckdb's open_file.patch and bump duckdb
  • Loading branch information
@samansmink
samansmink authored Apr 15, 2024
2 parents 4512a65 + 0bacbab commit 0962377
Show file tree
Hide file tree
Showing 10 changed files with 66 additions and 69 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/MainDistributionPipeline.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
uses: duckdb/extension-ci-tools/.github/workflows/[email protected]
with:
extension_name: azure
duckdb_version: 'v0.10.1'
duckdb_version: main
exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'

duckdb-stable-deploy:
Expand All @@ -27,6 +27,6 @@ jobs:
secrets: inherit
with:
extension_name: azure
duckdb_version: 'v0.10.1'
duckdb_version: main
exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
2 changes: 1 addition & 1 deletion duckdb
Submodule duckdb updated 1044 files
17 changes: 8 additions & 9 deletions src/azure_blob_filesystem.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,21 +66,20 @@ AzureBlobContextState::GetBlobContainerClient(const std::string &blobContainerNa
}

//////// AzureBlobStorageFileHandle ////////
AzureBlobStorageFileHandle::AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, string path, uint8_t flags,
AzureBlobStorageFileHandle::AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, string path, FileOpenFlags flags,
const AzureReadOptions &read_options,
Azure::Storage::Blobs::BlobClient blob_client)
: AzureFileHandle(fs, std::move(path), flags, read_options), blob_client(std::move(blob_client)) {
}

//////// AzureBlobStorageFileSystem ////////
unique_ptr<AzureFileHandle> AzureBlobStorageFileSystem::CreateHandle(const string &path, uint8_t flags,
FileLockType lock, FileCompressionType compression,
FileOpener *opener) {
if (opener == nullptr) {
unique_ptr<AzureFileHandle> AzureBlobStorageFileSystem::CreateHandle(const string &path, FileOpenFlags flags,
optional_ptr<FileOpener> opener) {
if (!opener) {
throw InternalException("Cannot do Azure storage CreateHandle without FileOpener");
}

D_ASSERT(compression == FileCompressionType::UNCOMPRESSED);
D_ASSERT(flags.Compression() == FileCompressionType::UNCOMPRESSED);

auto parsed_url = ParseUrl(path);
auto storage_context = GetOrCreateStorageContext(opener, path, parsed_url);
Expand Down Expand Up @@ -167,9 +166,9 @@ void AzureBlobStorageFileSystem::LoadRemoteFileInfo(AzureFileHandle &handle) {
hfh.last_modified = ToTimeT(res.Value.LastModified);
}

bool AzureBlobStorageFileSystem::FileExists(const string &filename) {
bool AzureBlobStorageFileSystem::FileExists(const string &filename, optional_ptr<FileOpener> opener) {
try {
auto handle = OpenFile(filename, FileFlags::FILE_FLAGS_READ);
auto handle = OpenFile(filename, FileFlags::FILE_FLAGS_READ, opener);
auto &sfh = handle->Cast<AzureBlobStorageFileHandle>();
if (sfh.length == 0) {
return false;
Expand Down Expand Up @@ -202,7 +201,7 @@ void AzureBlobStorageFileSystem::ReadRange(AzureFileHandle &handle, idx_t file_o
}
}

std::shared_ptr<AzureContextState> AzureBlobStorageFileSystem::CreateStorageContext(FileOpener *opener,
std::shared_ptr<AzureContextState> AzureBlobStorageFileSystem::CreateStorageContext(optional_ptr<FileOpener> opener,
const string &path,
const AzureParsedUrl &parsed_url) {
auto azure_read_options = ParseAzureReadOptions(opener);
Expand Down
11 changes: 5 additions & 6 deletions src/azure_dfs_filesystem.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,21 +83,20 @@ AzureDfsContextState::GetDfsFileSystemClient(const std::string &file_system_name
}

//////// AzureDfsContextState ////////
AzureDfsStorageFileHandle::AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, string path, uint8_t flags,
AzureDfsStorageFileHandle::AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, string path, FileOpenFlags flags,
const AzureReadOptions &read_options,
Azure::Storage::Files::DataLake::DataLakeFileClient client)
: AzureFileHandle(fs, std::move(path), flags, read_options), file_client(std::move(client)) {
}

//////// AzureDfsStorageFileSystem ////////
unique_ptr<AzureFileHandle> AzureDfsStorageFileSystem::CreateHandle(const string &path, uint8_t flags,
FileLockType lock, FileCompressionType compression,
FileOpener *opener) {
unique_ptr<AzureFileHandle> AzureDfsStorageFileSystem::CreateHandle(const string &path, FileOpenFlags flags,
optional_ptr<FileOpener> opener) {
if (opener == nullptr) {
throw InternalException("Cannot do Azure storage CreateHandle without FileOpener");
}

D_ASSERT(compression == FileCompressionType::UNCOMPRESSED);
D_ASSERT(flags.Compression() == FileCompressionType::UNCOMPRESSED);

auto parsed_url = ParseUrl(path);
auto storage_context = GetOrCreateStorageContext(opener, path, parsed_url);
Expand Down Expand Up @@ -186,7 +185,7 @@ void AzureDfsStorageFileSystem::ReadRange(AzureFileHandle &handle, idx_t file_of
}
}

std::shared_ptr<AzureContextState> AzureDfsStorageFileSystem::CreateStorageContext(FileOpener *opener,
std::shared_ptr<AzureContextState> AzureDfsStorageFileSystem::CreateStorageContext(optional_ptr<FileOpener> opener,
const string &path,
const AzureParsedUrl &parsed_url) {
auto azure_read_options = ParseAzureReadOptions(opener);
Expand Down
24 changes: 12 additions & 12 deletions src/azure_filesystem.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ void AzureContextState::QueryEnd() {
is_valid = false;
}

AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, string path, uint8_t flags,
AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, string path, FileOpenFlags flags,
const AzureReadOptions &read_options)
: FileHandle(fs, std::move(path)), flags(flags),
// File info
Expand All @@ -27,7 +27,7 @@ AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, string path, uint8_
buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0),
// Options
read_options(read_options) {
if (flags & FileFlags::FILE_FLAGS_READ) {
if (flags.OpenForReading()) {
read_buffer = duckdb::unique_ptr<data_t[]>(new data_t[read_options.buffer_size]);
}
}
Expand All @@ -37,7 +37,7 @@ void AzureFileHandle::PostConstruct() {
}

void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) {
if (handle.flags & FileFlags::FILE_FLAGS_READ) {
if (handle.flags.OpenForReading()) {
try {
LoadRemoteFileInfo(handle);
} catch (const Azure::Storage::StorageException &e) {
Expand All @@ -53,15 +53,15 @@ void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) {
}
}

unique_ptr<FileHandle> AzureStorageFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock,
FileCompressionType compression, FileOpener *opener) {
D_ASSERT(compression == FileCompressionType::UNCOMPRESSED);
unique_ptr<FileHandle> AzureStorageFileSystem::OpenFile(const string &path,FileOpenFlags flags,
optional_ptr<FileOpener> opener) {
D_ASSERT(flags.Compression() == FileCompressionType::UNCOMPRESSED);

if (flags & FileFlags::FILE_FLAGS_WRITE) {
if (flags.OpenForWriting()) {
throw NotImplementedException("Writing to Azure containers is currently not supported");
}

auto handle = CreateHandle(path, flags, lock, compression, opener);
auto handle = CreateHandle(path, flags, opener);
return std::move(handle);
}

Expand Down Expand Up @@ -92,7 +92,7 @@ void AzureStorageFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_b
idx_t buffer_offset = 0;

// Don't buffer when DirectIO is set.
if (hfh.flags & FileFlags::FILE_FLAGS_DIRECT_IO && to_read > 0) {
if (hfh.flags.DirectIO() && to_read > 0) {
ReadRange(hfh, location, (char *)buffer, to_read);
hfh.buffer_available = 0;
hfh.buffer_idx = 0;
Expand Down Expand Up @@ -153,7 +153,7 @@ int64_t AzureStorageFileSystem::Read(FileHandle &handle, void *buffer, int64_t n
return nr_bytes;
}

std::shared_ptr<AzureContextState> AzureStorageFileSystem::GetOrCreateStorageContext(FileOpener *opener,
std::shared_ptr<AzureContextState> AzureStorageFileSystem::GetOrCreateStorageContext(optional_ptr<FileOpener> opener,
const string &path,
const AzureParsedUrl &parsed_url) {
Value value;
Expand All @@ -164,7 +164,7 @@ std::shared_ptr<AzureContextState> AzureStorageFileSystem::GetOrCreateStorageCon

std::shared_ptr<AzureContextState> result;
if (azure_context_caching) {
auto *client_context = FileOpener::TryGetClientContext(opener);
auto client_context = FileOpener::TryGetClientContext(opener);

auto context_key = GetContextPrefix() + parsed_url.storage_account_name;

Expand Down Expand Up @@ -192,7 +192,7 @@ std::shared_ptr<AzureContextState> AzureStorageFileSystem::GetOrCreateStorageCon
return result;
}

AzureReadOptions AzureStorageFileSystem::ParseAzureReadOptions(FileOpener *opener) {
AzureReadOptions AzureStorageFileSystem::ParseAzureReadOptions(optional_ptr<FileOpener> opener) {
AzureReadOptions options;

Value concurrency_val;
Expand Down
32 changes: 16 additions & 16 deletions src/azure_storage_account_client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ namespace duckdb {
const static std::string DEFAULT_BLOB_ENDPOINT = "blob.core.windows.net";
const static std::string DEFAULT_DFS_ENDPOINT = "dfs.core.windows.net";

static std::string TryGetCurrentSetting(FileOpener *opener, const std::string &name) {
static std::string TryGetCurrentSetting(optional_ptr<FileOpener> opener, const std::string &name) {
Value val;
if (FileOpener::TryGetCurrentSetting(opener, name, val)) {
return val.ToString();
Expand Down Expand Up @@ -110,7 +110,7 @@ ToTokenCredentialOptions(const Azure::Core::Http::Policies::TransportOptions &tr
return options;
}

static std::shared_ptr<HTTPState> GetHttpState(FileOpener *opener) {
static std::shared_ptr<HTTPState> GetHttpState(optional_ptr<FileOpener> opener) {
Value value;
bool enable_http_stats = false;
if (FileOpener::TryGetCurrentSetting(opener, "azure_http_stats", value)) {
Expand Down Expand Up @@ -270,7 +270,7 @@ static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(const s
return transport_options;
}

static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(FileOpener *opener,
static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(optional_ptr<FileOpener> opener,
const KeyValueSecret &secret) {
auto transport_option_type = TryGetCurrentSetting(opener, "azure_transport_option_type");

Expand Down Expand Up @@ -302,7 +302,7 @@ static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(FileOpe
}

static Azure::Storage::Blobs::BlobServiceClient
GetBlobStorageAccountClientFromConfigProvider(FileOpener *opener, const KeyValueSecret &secret,
GetBlobStorageAccountClientFromConfigProvider(optional_ptr<FileOpener> opener, const KeyValueSecret &secret,
const AzureParsedUrl &azure_parsed_url) {
auto transport_options = GetTransportOptions(opener, secret);

Expand All @@ -328,7 +328,7 @@ GetBlobStorageAccountClientFromConfigProvider(FileOpener *opener, const KeyValue
}

static Azure::Storage::Files::DataLake::DataLakeServiceClient
GetDfsStorageAccountClientFromConfigProvider(FileOpener *opener, const KeyValueSecret &secret,
GetDfsStorageAccountClientFromConfigProvider(optional_ptr<FileOpener> opener, const KeyValueSecret &secret,
const AzureParsedUrl &azure_parsed_url) {
auto transport_options = GetTransportOptions(opener, secret);

Expand All @@ -355,7 +355,7 @@ GetDfsStorageAccountClientFromConfigProvider(FileOpener *opener, const KeyValueS
}

static Azure::Storage::Blobs::BlobServiceClient
GetBlobStorageAccountClientFromCredentialChainProvider(FileOpener *opener, const KeyValueSecret &secret,
GetBlobStorageAccountClientFromCredentialChainProvider(optional_ptr<FileOpener> opener, const KeyValueSecret &secret,
const AzureParsedUrl &azure_parsed_url) {
auto transport_options = GetTransportOptions(opener, secret);
// Create credential chain
Expand All @@ -369,7 +369,7 @@ GetBlobStorageAccountClientFromCredentialChainProvider(FileOpener *opener, const
}

static Azure::Storage::Files::DataLake::DataLakeServiceClient
GetDfsStorageAccountClientFromCredentialChainProvider(FileOpener *opener, const KeyValueSecret &secret,
GetDfsStorageAccountClientFromCredentialChainProvider(optional_ptr<FileOpener> opener, const KeyValueSecret &secret,
const AzureParsedUrl &azure_parsed_url) {
auto transport_options = GetTransportOptions(opener, secret);
// Create credential chain
Expand All @@ -383,7 +383,7 @@ GetDfsStorageAccountClientFromCredentialChainProvider(FileOpener *opener, const
}

static Azure::Storage::Blobs::BlobServiceClient
GetBlobStorageAccountClientFromServicePrincipalProvider(FileOpener *opener, const KeyValueSecret &secret,
GetBlobStorageAccountClientFromServicePrincipalProvider(optional_ptr<FileOpener> opener, const KeyValueSecret &secret,
const AzureParsedUrl &azure_parsed_url) {
auto transport_options = GetTransportOptions(opener, secret);
auto token_credential = CreateClientCredential(secret, transport_options);
Expand All @@ -396,7 +396,7 @@ GetBlobStorageAccountClientFromServicePrincipalProvider(FileOpener *opener, cons
}

static Azure::Storage::Files::DataLake::DataLakeServiceClient
GetDfsStorageAccountClientFromServicePrincipalProvider(FileOpener *opener, const KeyValueSecret &secret,
GetDfsStorageAccountClientFromServicePrincipalProvider(optional_ptr<FileOpener> opener, const KeyValueSecret &secret,
const AzureParsedUrl &azure_parsed_url) {
auto transport_options = GetTransportOptions(opener, secret);
auto token_credential = CreateClientCredential(secret, transport_options);
Expand All @@ -409,7 +409,7 @@ GetDfsStorageAccountClientFromServicePrincipalProvider(FileOpener *opener, const
}

static Azure::Storage::Blobs::BlobServiceClient
GetBlobStorageAccountClient(FileOpener *opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) {
GetBlobStorageAccountClient(optional_ptr<FileOpener> opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) {
auto &provider = secret.GetProvider();
// default provider
if (provider == "config") {
Expand All @@ -424,7 +424,7 @@ GetBlobStorageAccountClient(FileOpener *opener, const KeyValueSecret &secret, co
}

static Azure::Storage::Files::DataLake::DataLakeServiceClient
GetDfsStorageAccountClient(FileOpener *opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) {
GetDfsStorageAccountClient(optional_ptr<FileOpener> opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) {
auto &provider = secret.GetProvider();
// default provider
if (provider == "config") {
Expand All @@ -438,7 +438,7 @@ GetDfsStorageAccountClient(FileOpener *opener, const KeyValueSecret &secret, con
throw InvalidInputException("Unsupported provider type %s for azure", provider);
}

static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(FileOpener *opener) {
static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(optional_ptr<FileOpener> opener) {
auto azure_transport_option_type = TryGetCurrentSetting(opener, "azure_transport_option_type");

// Load proxy options
Expand All @@ -449,7 +449,7 @@ static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(FileOpe
return GetTransportOptions(azure_transport_option_type, http_proxy, http_proxy_user_name, http_proxy_password);
}

static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(FileOpener *opener,
static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(optional_ptr<FileOpener> opener,
const std::string &provided_storage_account,
const std::string &provided_endpoint) {
auto transport_options = GetTransportOptions(opener);
Expand Down Expand Up @@ -494,7 +494,7 @@ static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(File
return Azure::Storage::Blobs::BlobServiceClient {account_url, blob_options};
}

const SecretMatch LookupSecret(FileOpener *opener, const std::string &path) {
const SecretMatch LookupSecret(optional_ptr<FileOpener> opener, const std::string &path) {
auto context = opener->TryGetClientContext();

if (context) {
Expand All @@ -505,7 +505,7 @@ const SecretMatch LookupSecret(FileOpener *opener, const std::string &path) {
return {};
}

Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(FileOpener *opener, const std::string &path,
Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(optional_ptr<FileOpener> opener, const std::string &path,
const AzureParsedUrl &azure_parsed_url) {

auto secret_match = LookupSecret(opener, path);
Expand All @@ -519,7 +519,7 @@ Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(FileOpener
}

Azure::Storage::Files::DataLake::DataLakeServiceClient
ConnectToDfsStorageAccount(FileOpener *opener, const std::string &path, const AzureParsedUrl &azure_parsed_url) {
ConnectToDfsStorageAccount(optional_ptr<FileOpener> opener, const std::string &path, const AzureParsedUrl &azure_parsed_url) {
auto secret_match = LookupSecret(opener, path);
if (secret_match.HasMatch()) {
const auto &base_secret = secret_match.GetSecret();
Expand Down
10 changes: 5 additions & 5 deletions src/include/azure_blob_filesystem.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class AzureBlobStorageFileSystem;

class AzureBlobStorageFileHandle : public AzureFileHandle {
public:
AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, string path, uint8_t flags,
AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, string path, FileOpenFlags flags,
const AzureReadOptions &read_options, Azure::Storage::Blobs::BlobClient blob_client);
~AzureBlobStorageFileHandle() override = default;

Expand All @@ -36,7 +36,7 @@ class AzureBlobStorageFileSystem : public AzureStorageFileSystem {
vector<string> Glob(const string &path, FileOpener *opener = nullptr) override;

// FS methods
bool FileExists(const string &filename) override;
bool FileExists(const string &filename, optional_ptr<FileOpener> opener = nullptr) override;
bool CanHandleFile(const string &fpath) override;
string GetName() const override {
return "AzureBlobStorageFileSystem";
Expand All @@ -57,10 +57,10 @@ class AzureBlobStorageFileSystem : public AzureStorageFileSystem {
const string &GetContextPrefix() const override {
return PATH_PREFIX;
}
std::shared_ptr<AzureContextState> CreateStorageContext(FileOpener *opener, const string &path,
std::shared_ptr<AzureContextState> CreateStorageContext(optional_ptr<FileOpener> opener, const string &path,
const AzureParsedUrl &parsed_url) override;
duckdb::unique_ptr<AzureFileHandle> CreateHandle(const string &path, uint8_t flags, FileLockType lock,
FileCompressionType compression, FileOpener *opener) override;
duckdb::unique_ptr<AzureFileHandle> CreateHandle(const string &path, FileOpenFlags flags,
optional_ptr<FileOpener> opener) override;

void ReadRange(AzureFileHandle &handle, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) override;
};
Expand Down
Loading

0 comments on commit 0962377

Please sign in to comment.