Skip to content

Commit

Permalink
#113 Addressed issued found in the review.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ahsimb
ahsimb committed May 2, 2024
1 parent f8edf86 commit a630c23
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 38 deletions.
26 changes: 12 additions & 14 deletions exasol/bucketfs/_buckets.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import requests
from requests import HTTPError
from requests.auth import HTTPBasicAuth
from urllib.parse import quote_plus

from exasol.saas.client.openapi.client import AuthenticatedClient as SaasAuthenticatedClient
from exasol.saas.client.openapi.models.file import File as SaasFile
Expand Down Expand Up @@ -184,7 +185,7 @@ def _auth(self) -> HTTPBasicAuth:
@property
def files(self) -> Iterable[str]:
url = _build_url(service_url=self._service, bucket=self.name)
LOGGER.info(f"Retrieving bucket listing for {self.name}.")
LOGGER.info("Retrieving bucket listing for {0}.", self.name)
response = requests.get(url, auth=self._auth, verify=self._verify)
try:
response.raise_for_status()
Expand All @@ -208,7 +209,7 @@ def upload(
data: raw content of the file.
"""
url = _build_url(service_url=self._service, bucket=self.name, path=path)
LOGGER.info(f"Uploading {path} to bucket {self.name}.")
LOGGER.info("Uploading {0} to bucket {1}.", path, self.name)
response = requests.put(url, data=data, auth=self._auth, verify=self._verify)
try:
response.raise_for_status()
Expand All @@ -226,7 +227,7 @@ def delete(self, path) -> None:
A BucketFsError if the operation couldn't be executed successfully.
"""
url = _build_url(service_url=self._service, bucket=self.name, path=path)
LOGGER.info(f"Deleting {path} from bucket {self.name}.")
LOGGER.info("Deleting {0} from bucket {1}.", path, self.name)
response = requests.delete(url, auth=self._auth, verify=self._verify)

try:
Expand All @@ -247,7 +248,8 @@ def download(self, path: str, chunk_size: int = 8192) -> Iterable[ByteString]:
"""
url = _build_url(service_url=self._service, bucket=self.name, path=path)
LOGGER.info(
f"Downloading {path} using a chunk size of {chunk_size} bytes from bucket {self.name}."
"Downloading {0} using a chunk size of {1} bytes from bucket {2}.",
path, chunk_size, self.name
)
with requests.get(
url, stream=True, auth=self._auth, verify=self._verify
Expand All @@ -260,10 +262,6 @@ def download(self, path: str, chunk_size: int = 8192) -> Iterable[ByteString]:
yield from response.iter_content(chunk_size=chunk_size)


def _to_path_in_url(path: str) -> str:
return path.replace('/', '%2F')


class SaaSBucket:

def __init__(self, url: str, account_id: str, database_id: str, pat: str) -> None:
Expand Down Expand Up @@ -305,17 +303,17 @@ def recursive_file_collector(node: SaasFile) -> None:
return file_list

def delete(self, path: str) -> None:
LOGGER.info(f"Deleting {path} from the bucket.")
LOGGER.info("Deleting {0} from the bucket.", path)
with SaasAuthenticatedClient(base_url=self._url,
token=self._pat,
raise_on_unexpected_status=True) as client:
saas_delete_file(account_id=self._account_id,
database_id=self._database_id,
key=_to_path_in_url(path),
key=quote_plus(path),
client=client)

def upload(self, path: str, data: ByteString | BinaryIO) -> None:
LOGGER.info(f"Uploading {path} to the bucket.")
LOGGER.info("Uploading {0} to the bucket.", path)
# Q. The service can handle any characters in the path.
# Do we need to check this path for presence of characters deemed
# invalid in the BucketLike protocol?
Expand All @@ -324,7 +322,7 @@ def upload(self, path: str, data: ByteString | BinaryIO) -> None:
raise_on_unexpected_status=False) as client:
response = saas_upload_file(account_id=self._account_id,
database_id=self._database_id,
key=_to_path_in_url(path),
key=quote_plus(path),
client=client)
if response.status_code >= 400:
# Q. Is it the right type of exception?
Expand All @@ -336,13 +334,13 @@ def upload(self, path: str, data: ByteString | BinaryIO) -> None:
response.raise_for_status()

def download(self, path: str, chunk_size: int = 8192) -> Iterable[ByteString]:
LOGGER.info(f"Downloading {path} from the bucket.")
LOGGER.info("Downloading {0} from the bucket.", path)
with SaasAuthenticatedClient(base_url=self._url,
token=self._pat,
raise_on_unexpected_status=False) as client:
response = saas_download_file(account_id=self._account_id,
database_id=self._database_id,
key=_to_path_in_url(path),
key=quote_plus(path),
client=client)
if response.status_code == 404:
raise BucketFsError("The file {path} doesn't exist in the SaaS BucketFs.")
Expand Down
26 changes: 14 additions & 12 deletions test/integration/test_bucket_path.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,30 +7,32 @@

@pytest.fixture
def children_poem() -> ByteString:
poem_text = \
b"Twinkle twinkle little star." \
b"How I wonder what you are." \
b"Up above the world so high." \
poem_text = (
b"Twinkle twinkle little star."
b"How I wonder what you are."
b"Up above the world so high."
b"Like a diamond in the sky."
)
return poem_text


@pytest.fixture
def classic_poem() -> ByteString:
poem_text = \
b"My heart's in the Highlands, my heart is not here," \
b"My heart's in the Highlands, a-chasing the deer;" \
b"Chasing the wild-deer, and following the roe," \
poem_text = (
b"My heart's in the Highlands, my heart is not here,"
b"My heart's in the Highlands, a-chasing the deer;"
b"Chasing the wild-deer, and following the roe,"
b"My heart's in the Highlands, wherever I go."
)
return poem_text


def _collect_all_names(path: bfs.path.PathLike) -> set[str]:
all_names = []
all_names = set()
for _, dirs, files in path.walk():
all_names.extend(dirs)
all_names.extend(files)
return set(all_names)
all_names.update(dirs)
all_names.update(files)
return all_names


def test_write_read_back_onprem(test_config, children_poem):
Expand Down
26 changes: 14 additions & 12 deletions test_saas/integration/test_path.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,30 +6,32 @@

@pytest.fixture
def children_poem() -> ByteString:
poem_text = \
b"Twinkle twinkle little star." \
b"How I wonder what you are." \
b"Up above the world so high." \
poem_text = (
b"Twinkle twinkle little star."
b"How I wonder what you are."
b"Up above the world so high."
b"Like a diamond in the sky."
)
return poem_text


@pytest.fixture
def classic_poem() -> ByteString:
poem_text = \
b"My heart's in the Highlands, my heart is not here," \
b"My heart's in the Highlands, a-chasing the deer;" \
b"Chasing the wild-deer, and following the roe," \
poem_text = (
b"My heart's in the Highlands, my heart is not here,"
b"My heart's in the Highlands, a-chasing the deer;"
b"Chasing the wild-deer, and following the roe,"
b"My heart's in the Highlands, wherever I go."
)
return poem_text


def _collect_all_names(path: bfs.path.PathLike) -> set[str]:
all_names = []
all_names = set()
for _, dirs, files in path.walk():
all_names.extend(dirs)
all_names.extend(files)
return set(all_names)
all_names.update(dirs)
all_names.update(files)
return all_names


def test_write_read_back_saas(saas_test_service_url, saas_test_token,
Expand Down

0 comments on commit a630c23

Please sign in to comment.