Skip to content

Commit

Permalink
Add download_from_bucketfs_to_fileobj to BucketFSLocation
Browse files Browse the repository at this point in the history
  • Loading branch information
@tkilias
tkilias committed Feb 25, 2024
1 parent 96fc64b commit 58b8d45
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 20 deletions.
20 changes: 12 additions & 8 deletions exasol_bucketfs_utils_python/abstract_bucketfs_location.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,39 +27,43 @@ class AbstractBucketFSLocation(ABC, metaclass=ABCMeta):

@abstractmethod
def generate_bucket_udf_path(
self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None
self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None
) -> PurePosixPath:
pass

@abstractmethod
def get_complete_file_path_in_bucket(
self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None
self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None
) -> str:
pass

@abstractmethod
def download_from_bucketfs_to_string(self, bucket_file_path: str) -> str:
pass

@abstractmethod
def download_from_bucketfs_to_fileobj(self, bucket_file_path: str, fileobj: IO):
pass

@abstractmethod
def download_object_from_bucketfs_via_joblib(self, bucket_file_path: str) -> Any:
pass

@abstractmethod
def upload_string_to_bucketfs(
self, bucket_file_path: str, string: str
self, bucket_file_path: str, string: str
) -> Tuple[ParseResult, PurePosixPath]:
pass

@abstractmethod
def upload_object_to_bucketfs_via_joblib(
self, object: Any, bucket_file_path: str, **kwargs
self, object: Any, bucket_file_path: str, **kwargs
) -> Tuple[ParseResult, PurePosixPath]:
pass

@abstractmethod
def upload_fileobj_to_bucketfs(
self, fileobj: IO, bucket_file_path: str
self, fileobj: IO, bucket_file_path: str
) -> Tuple[ParseResult, PurePosixPath]:
pass

Expand All @@ -71,13 +75,13 @@ def read_file_from_bucketfs_to_string(self, bucket_file_path: str) -> str:

@abstractmethod
def read_file_from_bucketfs_to_file(
self, bucket_file_path: str, local_file_path: Path
self, bucket_file_path: str, local_file_path: Path
) -> None:
pass

@abstractmethod
def read_file_from_bucketfs_to_fileobj(
self, bucket_file_path: str, fileobj: IO
self, bucket_file_path: str, fileobj: IO
) -> None:
pass

Expand All @@ -95,6 +99,6 @@ def delete_file_in_bucketfs(self, bucket_file_path: str) -> None:

@abstractmethod
def joinpath(
self, *others: Union[str, PurePosixPath]
self, *others: Union[str, PurePosixPath]
) -> "AbstractBucketFSLocation":
pass
19 changes: 12 additions & 7 deletions exasol_bucketfs_utils_python/bucketfs_location.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,14 @@ def __init__(self, bucket_config: BucketConfig, base_path: Optional[PurePosixPat
self.bucket_config = bucket_config

def generate_bucket_udf_path(
self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None
self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None
) -> PurePosixPath:
return bucketfs_utils.generate_bucket_udf_path(
self.bucket_config, self.get_complete_file_path_in_bucket(path_in_bucket)
)

def get_complete_file_path_in_bucket(
self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None
self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None
) -> str:

if bucket_file_path is not None:
Expand All @@ -58,6 +58,11 @@ def get_complete_file_path_in_bucket(
bucket_file_path = ""
return str(PurePosixPath(self.base_path, bucket_file_path))

def download_from_bucketfs_to_fileobj(self, bucket_file_path: str, fileobj: IO):
download.download_from_bucketfs_to_fileobj(fileobj=fileobj,
bucket_file_path=bucket_file_path,
bucket_config=self.bucket_config)

def download_from_bucketfs_to_string(self, bucket_file_path: str) -> str:
return download.download_from_bucketfs_to_string(
self.bucket_config, self.get_complete_file_path_in_bucket(bucket_file_path)
Expand All @@ -69,7 +74,7 @@ def download_object_from_bucketfs_via_joblib(self, bucket_file_path: str) -> Any
)

def upload_string_to_bucketfs(
self, bucket_file_path: str, string: str
self, bucket_file_path: str, string: str
) -> Tuple[ParseResult, PurePosixPath]:
return upload.upload_string_to_bucketfs(
self.bucket_config,
Expand All @@ -78,7 +83,7 @@ def upload_string_to_bucketfs(
)

def upload_object_to_bucketfs_via_joblib(
self, object: Any, bucket_file_path: str, **kwargs
self, object: Any, bucket_file_path: str, **kwargs
) -> Tuple[ParseResult, PurePosixPath]:
return upload.upload_object_to_bucketfs_via_joblib(
object,
Expand All @@ -88,7 +93,7 @@ def upload_object_to_bucketfs_via_joblib(
)

def upload_fileobj_to_bucketfs(
self, fileobj: IO, bucket_file_path: str
self, fileobj: IO, bucket_file_path: str
) -> Tuple[ParseResult, PurePosixPath]:
return upload.upload_fileobj_to_bucketfs(
self.bucket_config,
Expand All @@ -102,7 +107,7 @@ def read_file_from_bucketfs_to_string(self, bucket_file_path: str) -> str:
)

def read_file_from_bucketfs_to_file(
self, bucket_file_path: str, local_file_path: Path
self, bucket_file_path: str, local_file_path: Path
) -> None:
from_BFS.read_file_from_bucketfs_to_file(
self.get_complete_file_path_in_bucket(bucket_file_path),
Expand All @@ -111,7 +116,7 @@ def read_file_from_bucketfs_to_file(
)

def read_file_from_bucketfs_to_fileobj(
self, bucket_file_path: str, fileobj: IO
self, bucket_file_path: str, fileobj: IO
) -> None:
from_BFS.read_file_from_bucketfs_to_fileobj(
self.get_complete_file_path_in_bucket(bucket_file_path),
Expand Down
14 changes: 9 additions & 5 deletions exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def __init__(self, base_path: Optional[PurePosixPath]):
self.base_path = "" if base_path is None else base_path

def get_complete_file_path_in_bucket(
self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None
self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None
) -> str:
if bucket_file_path is not None:
bucket_file_path = bucketfs_utils.make_path_relative(bucket_file_path)
Expand All @@ -39,7 +39,7 @@ def get_complete_file_path_in_bucket(
return str(PurePosixPath(self.base_path, bucket_file_path))

def generate_bucket_udf_path(
self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None
self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None
) -> PurePosixPath:

if path_in_bucket is not None:
Expand All @@ -49,6 +49,10 @@ def generate_bucket_udf_path(
path = PurePosixPath(self.base_path, path_in_bucket)
return path

def download_from_bucketfs_to_fileobj(self, bucket_file_path: str, fileobj: IO):
with open(self.get_complete_file_path_in_bucket(bucket_file_path)) as f:
fileobj.write(f.read())

def download_from_bucketfs_to_string(self, bucket_file_path: str) -> str:
with open(self.get_complete_file_path_in_bucket(bucket_file_path)) as f:
result = f.read()
Expand All @@ -65,7 +69,7 @@ def upload_string_to_bucketfs(self, bucket_file_path: str, string: str) -> None:
f.write(string)

def upload_object_to_bucketfs_via_joblib(
self, object_: Any, bucket_file_path: str, **kwargs
self, object_: Any, bucket_file_path: str, **kwargs
) -> None:
path = self.get_complete_file_path_in_bucket(bucket_file_path)
Path(path).parent.mkdir(parents=True, exist_ok=True)
Expand All @@ -79,15 +83,15 @@ def upload_fileobj_to_bucketfs(self, fileobj: IO, bucket_file_path: str) -> None
f.write(chunk)

def read_file_from_bucketfs_to_fileobj(
self, bucket_file_path: str, fileobj: IO
self, bucket_file_path: str, fileobj: IO
) -> None:
bucket_path = self.get_complete_file_path_in_bucket(bucket_file_path)
with open(bucket_path, "rb") as read_file:
read_file.seek(0)
fileobj.write(read_file.read())

def read_file_from_bucketfs_to_file(
self, bucket_file_path: str, local_file_path: Path
self, bucket_file_path: str, local_file_path: Path
) -> None:
with open(local_file_path, "wb") as fileobj:
self.read_file_from_bucketfs_to_fileobj(bucket_file_path, fileobj)
Expand Down

0 comments on commit 58b8d45

Please sign in to comment.