Skip to content

Commit

Permalink
Deprecated get_s3fs_session
Browse files Browse the repository at this point in the history
  • Loading branch information
Sherwin-14 committed Jul 20, 2024
1 parent 7a79ea5 commit 1a57e19
Showing 1 changed file with 27 additions and 6 deletions.
33 changes: 27 additions & 6 deletions earthaccess/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import s3fs
from multimethod import multimethod as singledispatchmethod
from pqdm.threads import pqdm
from typing_extensions import deprecated

import earthaccess

Expand Down Expand Up @@ -197,6 +198,7 @@ def set_requests_session(
else:
resp.raise_for_status()

@deprecated("Use get_s3_filesystem instead")
def get_s3fs_session(
self,
daac: Optional[str] = None,
Expand All @@ -206,6 +208,25 @@ def get_s3fs_session(
) -> s3fs.S3FileSystem:
"""Returns a s3fs instance for a given cloud provider / DAAC.
Parameters:
daac: any of the DAACs, e.g. NSIDC, PODAAC
provider: a data provider if we know them, e.g. PODAAC -> POCLOUD
endpoint: pass the URL for the credentials directly
Returns:
a s3fs file instance
"""
return self.get_s3_filesystem(daac, concept_id, provider, endpoint)

def get_s3fs_filesystem(
self,
daac: Optional[str] = None,
concept_id: Optional[str] = None,
provider: Optional[str] = None,
endpoint: Optional[str] = None,
) -> s3fs.S3FileSystem:
"""Returns a s3fs instance for a given cloud provider / DAAC.
Parameters:
daac: any of the DAACs, e.g. NSIDC, PODAAC
provider: a data provider if we know them, e.g. PODAAC -> POCLOUD
Expand Down Expand Up @@ -360,10 +381,10 @@ def _open_granules(
endpoint = self._own_s3_credentials(granules[0]["umm"]["RelatedUrls"])
if endpoint is not None:
logger.info(f"using endpoint: {endpoint}")
s3_fs = self.get_s3fs_session(endpoint=endpoint)
s3_fs = self.get_s3fs_filesystem(endpoint=endpoint)
else:
logger.info(f"using provider: {provider}")
s3_fs = self.get_s3fs_session(provider=provider)
s3_fs = self.get_s3fs_filesystem(provider=provider)
else:
access = "on_prem"
s3_fs = None
Expand Down Expand Up @@ -416,7 +437,7 @@ def _open_urls(
url_mapping: Mapping[str, None] = {url: None for url in granules}
if self.in_region and granules[0].startswith("s3"):
if provider is not None:
s3_fs = self.get_s3fs_session(provider=provider)
s3_fs = self.get_s3fs_filesystem(provider=provider)
if s3_fs is not None:
try:
fileset = _open_files(
Expand Down Expand Up @@ -530,7 +551,7 @@ def _get_urls(
)
if self.in_region and data_links[0].startswith("s3"):
logger.info(f"Accessing cloud dataset using provider: {provider}")
s3_fs = self.get_s3fs_session(provider=provider)
s3_fs = self.get_s3fs_filesystem(provider=provider)
# TODO: make this parallel or concurrent
for file in data_links:
s3_fs.get(file, str(local_path))
Expand Down Expand Up @@ -573,10 +594,10 @@ def _get_granules(
logger.info(
f"Accessing cloud dataset using dataset endpoint credentials: {endpoint}"
)
s3_fs = self.get_s3fs_session(endpoint=endpoint)
s3_fs = self.get_s3fs_filesystem(endpoint=endpoint)
else:
logger.info(f"Accessing cloud dataset using provider: {provider}")
s3_fs = self.get_s3fs_session(provider=provider)
s3_fs = self.get_s3fs_filesystem(provider=provider)

local_path.mkdir(parents=True, exist_ok=True)

Expand Down

0 comments on commit 1a57e19

Please sign in to comment.