diff --git a/exasol_bucketfs_utils_python/abstract_bucketfs_location.py b/exasol_bucketfs_utils_python/abstract_bucketfs_location.py index cb57e4d7..e0a26e32 100644 --- a/exasol_bucketfs_utils_python/abstract_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/abstract_bucketfs_location.py @@ -27,13 +27,13 @@ class AbstractBucketFSLocation(ABC, metaclass=ABCMeta): @abstractmethod def generate_bucket_udf_path( - self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None + self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None ) -> PurePosixPath: pass @abstractmethod def get_complete_file_path_in_bucket( - self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None + self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None ) -> str: pass @@ -41,25 +41,29 @@ def get_complete_file_path_in_bucket( def download_from_bucketfs_to_string(self, bucket_file_path: str) -> str: pass + @abstractmethod + def download_from_bucketfs_to_fileobj(self, bucket_file_path: str, fileobj: IO): + pass + @abstractmethod def download_object_from_bucketfs_via_joblib(self, bucket_file_path: str) -> Any: pass @abstractmethod def upload_string_to_bucketfs( - self, bucket_file_path: str, string: str + self, bucket_file_path: str, string: str ) -> Tuple[ParseResult, PurePosixPath]: pass @abstractmethod def upload_object_to_bucketfs_via_joblib( - self, object: Any, bucket_file_path: str, **kwargs + self, object: Any, bucket_file_path: str, **kwargs ) -> Tuple[ParseResult, PurePosixPath]: pass @abstractmethod def upload_fileobj_to_bucketfs( - self, fileobj: IO, bucket_file_path: str + self, fileobj: IO, bucket_file_path: str ) -> Tuple[ParseResult, PurePosixPath]: pass @@ -71,13 +75,13 @@ def read_file_from_bucketfs_to_string(self, bucket_file_path: str) -> str: @abstractmethod def read_file_from_bucketfs_to_file( - self, bucket_file_path: str, local_file_path: Path + self, bucket_file_path: str, local_file_path: Path ) -> None: pass @abstractmethod def read_file_from_bucketfs_to_fileobj( - self, bucket_file_path: str, fileobj: IO + self, bucket_file_path: str, fileobj: IO ) -> None: pass @@ -95,6 +99,6 @@ def delete_file_in_bucketfs(self, bucket_file_path: str) -> None: @abstractmethod def joinpath( - self, *others: Union[str, PurePosixPath] + self, *others: Union[str, PurePosixPath] ) -> "AbstractBucketFSLocation": pass diff --git a/exasol_bucketfs_utils_python/bucketfs_location.py b/exasol_bucketfs_utils_python/bucketfs_location.py index 45d6038d..cb6f769b 100644 --- a/exasol_bucketfs_utils_python/bucketfs_location.py +++ b/exasol_bucketfs_utils_python/bucketfs_location.py @@ -42,14 +42,14 @@ def __init__(self, bucket_config: BucketConfig, base_path: Optional[PurePosixPat self.bucket_config = bucket_config def generate_bucket_udf_path( - self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None + self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None ) -> PurePosixPath: return bucketfs_utils.generate_bucket_udf_path( self.bucket_config, self.get_complete_file_path_in_bucket(path_in_bucket) ) def get_complete_file_path_in_bucket( - self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None + self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None ) -> str: if bucket_file_path is not None: @@ -58,6 +58,11 @@ def get_complete_file_path_in_bucket( bucket_file_path = "" return str(PurePosixPath(self.base_path, bucket_file_path)) + def download_from_bucketfs_to_fileobj(self, bucket_file_path: str, fileobj: IO): + download.download_from_bucketfs_to_fileobj(fileobj=fileobj, + bucket_file_path=bucket_file_path, + bucket_config=self.bucket_config) + def download_from_bucketfs_to_string(self, bucket_file_path: str) -> str: return download.download_from_bucketfs_to_string( self.bucket_config, self.get_complete_file_path_in_bucket(bucket_file_path) @@ -69,7 +74,7 @@ def download_object_from_bucketfs_via_joblib(self, bucket_file_path: str) -> Any ) def upload_string_to_bucketfs( - self, bucket_file_path: str, string: str + self, bucket_file_path: str, string: str ) -> Tuple[ParseResult, PurePosixPath]: return upload.upload_string_to_bucketfs( self.bucket_config, @@ -78,7 +83,7 @@ def upload_string_to_bucketfs( ) def upload_object_to_bucketfs_via_joblib( - self, object: Any, bucket_file_path: str, **kwargs + self, object: Any, bucket_file_path: str, **kwargs ) -> Tuple[ParseResult, PurePosixPath]: return upload.upload_object_to_bucketfs_via_joblib( object, @@ -88,7 +93,7 @@ def upload_object_to_bucketfs_via_joblib( ) def upload_fileobj_to_bucketfs( - self, fileobj: IO, bucket_file_path: str + self, fileobj: IO, bucket_file_path: str ) -> Tuple[ParseResult, PurePosixPath]: return upload.upload_fileobj_to_bucketfs( self.bucket_config, @@ -102,7 +107,7 @@ def read_file_from_bucketfs_to_string(self, bucket_file_path: str) -> str: ) def read_file_from_bucketfs_to_file( - self, bucket_file_path: str, local_file_path: Path + self, bucket_file_path: str, local_file_path: Path ) -> None: from_BFS.read_file_from_bucketfs_to_file( self.get_complete_file_path_in_bucket(bucket_file_path), @@ -111,7 +116,7 @@ def read_file_from_bucketfs_to_file( ) def read_file_from_bucketfs_to_fileobj( - self, bucket_file_path: str, fileobj: IO + self, bucket_file_path: str, fileobj: IO ) -> None: from_BFS.read_file_from_bucketfs_to_fileobj( self.get_complete_file_path_in_bucket(bucket_file_path), diff --git a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py index b2f3eabf..d8b2687c 100644 --- a/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py +++ b/exasol_bucketfs_utils_python/localfs_mock_bucketfs_location.py @@ -30,7 +30,7 @@ def __init__(self, base_path: Optional[PurePosixPath]): self.base_path = "" if base_path is None else base_path def get_complete_file_path_in_bucket( - self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None + self, bucket_file_path: Optional[Union[str, PurePosixPath]] = None ) -> str: if bucket_file_path is not None: bucket_file_path = bucketfs_utils.make_path_relative(bucket_file_path) @@ -39,7 +39,7 @@ def get_complete_file_path_in_bucket( return str(PurePosixPath(self.base_path, bucket_file_path)) def generate_bucket_udf_path( - self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None + self, path_in_bucket: Optional[Union[str, PurePosixPath]] = None ) -> PurePosixPath: if path_in_bucket is not None: @@ -49,6 +49,10 @@ def generate_bucket_udf_path( path = PurePosixPath(self.base_path, path_in_bucket) return path + def download_from_bucketfs_to_fileobj(self, bucket_file_path: str, fileobj: IO): + with open(self.get_complete_file_path_in_bucket(bucket_file_path)) as f: + fileobj.write(f.read()) + def download_from_bucketfs_to_string(self, bucket_file_path: str) -> str: with open(self.get_complete_file_path_in_bucket(bucket_file_path)) as f: result = f.read() @@ -65,7 +69,7 @@ def upload_string_to_bucketfs(self, bucket_file_path: str, string: str) -> None: f.write(string) def upload_object_to_bucketfs_via_joblib( - self, object_: Any, bucket_file_path: str, **kwargs + self, object_: Any, bucket_file_path: str, **kwargs ) -> None: path = self.get_complete_file_path_in_bucket(bucket_file_path) Path(path).parent.mkdir(parents=True, exist_ok=True) @@ -79,7 +83,7 @@ def upload_fileobj_to_bucketfs(self, fileobj: IO, bucket_file_path: str) -> None f.write(chunk) def read_file_from_bucketfs_to_fileobj( - self, bucket_file_path: str, fileobj: IO + self, bucket_file_path: str, fileobj: IO ) -> None: bucket_path = self.get_complete_file_path_in_bucket(bucket_file_path) with open(bucket_path, "rb") as read_file: @@ -87,7 +91,7 @@ def read_file_from_bucketfs_to_fileobj( fileobj.write(read_file.read()) def read_file_from_bucketfs_to_file( - self, bucket_file_path: str, local_file_path: Path + self, bucket_file_path: str, local_file_path: Path ) -> None: with open(local_file_path, "wb") as fileobj: self.read_file_from_bucketfs_to_fileobj(bucket_file_path, fileobj)