Skip to content

Commit

Permalink
ignored the D1* errors
Browse files Browse the repository at this point in the history
  • Loading branch information
Sherwin-14 committed May 23, 2024
1 parent 6518335 commit dc87dd7
Show file tree
Hide file tree
Showing 13 changed files with 461 additions and 260 deletions.
3 changes: 1 addition & 2 deletions earthaccess/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,7 @@


def __getattr__(name): # type: ignore
"""
Module-level getattr to handle automatic authentication when accessing
"""Module-level getattr to handle automatic authentication when accessing
`earthaccess.__auth__` and `earthaccess.__store__`.
Other unhandled attributes raise as `AttributeError` as expected.
Expand Down
94 changes: 67 additions & 27 deletions earthaccess/api.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@

import logging

import requests
Expand Down Expand Up @@ -37,7 +38,8 @@ def search_datasets(count: int = -1, **kwargs: Any) -> List[DataCollection]:
[https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html](https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html)
Parameters:
Parameters
----------
count: Number of records to get, -1 = all
kwargs (Dict):
arguments to CMR:
Expand All @@ -52,20 +54,24 @@ def search_datasets(count: int = -1, **kwargs: Any) -> List[DataCollection]:
* **bounding_box**: a tuple representing spatial bounds in the form
`(lower_left_lon, lower_left_lat, upper_right_lon, upper_right_lat)`
Returns:
Returns
-------
A list of DataCollection results that can be used to get information about a
dataset, e.g. concept_id, doi, etc.
Raises:
Raises
------
RuntimeError: The CMR query failed.
Examples:
Examples
--------
```python
datasets = earthaccess.search_datasets(
keyword="sea surface anomaly",
cloud_hosted=True
)
```
"""
if not validate.valid_dataset_parameters(**kwargs):
logger.warn("A valid set of parameters is needed to search for datasets on CMR")
Expand All @@ -86,7 +92,8 @@ def search_data(count: int = -1, **kwargs: Any) -> List[DataGranule]:
[https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html](https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html)
Parameters:
Parameters
----------
count: Number of records to get, -1 = all
kwargs (Dict):
arguments to CMR:
Expand All @@ -101,21 +108,25 @@ def search_data(count: int = -1, **kwargs: Any) -> List[DataGranule]:
* **bounding_box**: a tuple representing spatial bounds in the form
`(lower_left_lon, lower_left_lat, upper_right_lon, upper_right_lat)`
Returns:
Returns
-------
a list of DataGranules that can be used to access the granule files by using
`download()` or `open()`.
Raises:
Raises
------
RuntimeError: The CMR query failed.
Examples:
Examples
--------
```python
datasets = earthaccess.search_data(
doi="10.5067/SLREF-CDRV2",
cloud_hosted=True,
temporal=("2002-01-01", "2002-12-31")
)
```
"""
if earthaccess.__auth__.authenticated:
query = DataGranules(earthaccess.__auth__).parameters(**kwargs)
Expand All @@ -131,7 +142,8 @@ def search_data(count: int = -1, **kwargs: Any) -> List[DataGranule]:
def login(strategy: str = "all", persist: bool = False, system: System = PROD) -> Auth:
"""Authenticate with Earthdata login (https://urs.earthdata.nasa.gov/).
Parameters:
Parameters
----------
strategy:
An authentication method.
Expand All @@ -142,8 +154,10 @@ def login(strategy: str = "all", persist: bool = False, system: System = PROD) -
persist: will persist credentials in a .netrc file
system: the Earthdata system to access, defaults to PROD
Returns:
Returns
-------
An instance of Auth.
"""
# Set the underlying Auth object's earthdata system,
# before triggering the getattr function for `__auth__`.
Expand Down Expand Up @@ -181,17 +195,21 @@ def download(
* If we run it outside AWS (us-west-2 region) and the dataset is cloud hosted,
we'll use HTTP links.
Parameters:
Parameters
----------
granules: a granule, list of granules, a granule link (HTTP), or a list of granule links (HTTP)
local_path: local directory to store the remote data granules
provider: if we download a list of URLs, we need to specify the provider.
threads: parallel number of threads to use to download the files, adjust as necessary, default = 8
Returns:
Returns
-------
List of downloaded files
Raises:
Raises
------
Exception: A file download failed.
"""
provider = _normalize_location(provider)
if isinstance(granules, DataGranule):
Expand All @@ -205,6 +223,7 @@ def download(
f"{err}: You must call earthaccess.login() before you can download data"
)
return []

return results


Expand All @@ -215,13 +234,16 @@ def open(
"""Returns a list of fsspec file-like objects that can be used to access files
hosted on S3 or HTTPS by third party libraries like xarray.
Parameters:
Parameters
----------
granules: a list of granule instances **or** list of URLs, e.g. `s3://some-granule`.
If a list of URLs is passed, we need to specify the data provider.
provider: e.g. POCLOUD, NSIDC_CPRD, etc.
Returns:
Returns
-------
a list of s3fs "file pointers" to s3 files.
"""
provider = _normalize_location(provider)
results = earthaccess.__store__.open(granules=granules, provider=provider)
Expand All @@ -238,13 +260,16 @@ def get_s3_credentials(
If we use results, earthaccess will use the metadata on the response to get the credentials,
which is useful for missions that do not use the same endpoint as their DAACs, e.g. SWOT.
Parameters:
Parameters
----------
daac: a DAAC short_name like NSIDC or PODAAC, etc.
provider: if we know the provider for the DAAC, e.g. POCLOUD, LPCLOUD etc.
results: List of results from search_data()
Returns:
Returns
-------
a dictionary with S3 credentials for the DAAC or provider
"""
daac = _normalize_location(daac)
provider = _normalize_location(provider)
Expand All @@ -257,8 +282,10 @@ def get_s3_credentials(
def collection_query() -> CollectionQuery:
"""Returns a query builder instance for NASA collections (datasets).
Returns:
Returns
-------
a query builder instance for data collections.
"""
if earthaccess.__auth__.authenticated:
query_builder = DataCollections(earthaccess.__auth__)
Expand All @@ -268,10 +295,12 @@ def collection_query() -> CollectionQuery:


def granule_query() -> GranuleQuery:
"""Returns a query builder instance for data granules
"""Returns a query builder instance for data granules.
Returns:
Returns
-------
a query builder instance for data granules.
"""
if earthaccess.__auth__.authenticated:
query_builder = DataGranules(earthaccess.__auth__)
Expand All @@ -283,10 +312,12 @@ def granule_query() -> GranuleQuery:
def get_fsspec_https_session() -> AbstractFileSystem:
"""Returns a fsspec session that can be used to access datafiles across many different DAACs.
Returns:
Returns
-------
An fsspec instance able to access data across DAACs.
Examples:
Examples
--------
```python
import earthaccess
Expand All @@ -295,6 +326,7 @@ def get_fsspec_https_session() -> AbstractFileSystem:
with fs.open(DAAC_GRANULE) as f:
f.read(10)
```
"""
session = earthaccess.__store__.get_fsspec_session()
return session
Expand All @@ -305,10 +337,12 @@ def get_requests_https_session() -> requests.Session:
This is useful for making requests to restricted URLs, such as data granules or services that
require authentication with NASA EDL.
Returns:
Returns
-------
An authenticated requests Session instance.
Examples:
Examples
--------
```python
import earthaccess
Expand All @@ -318,6 +352,7 @@ def get_requests_https_session() -> requests.Session:
data = req_session.get(granule_url, headers = {"Range": "bytes=0-100"})
```
"""
session = earthaccess.__store__.get_requests_session()
return session
Expand All @@ -330,15 +365,18 @@ def get_s3fs_session(
) -> s3fs.S3FileSystem:
"""Returns a fsspec s3fs file session for direct access when we are in us-west-2.
Parameters:
Parameters
----------
daac: Any DAAC short name e.g. NSIDC, GES_DISC
provider: Each DAAC can have a cloud provider.
If the DAAC is specified, there is no need to use provider.
results: A list of results from search_data().
`earthaccess` will use the metadata from CMR to obtain the S3 Endpoint.
Returns:
Returns
-------
An authenticated s3fs session valid for 1 hour.
"""
daac = _normalize_location(daac)
provider = _normalize_location(provider)
Expand All @@ -354,8 +392,10 @@ def get_s3fs_session(
def get_edl_token() -> str:
"""Returns the current token used for EDL.
Returns:
Returns
-------
EDL token
"""
token = earthaccess.__auth__.token
return token
Expand Down
24 changes: 16 additions & 8 deletions earthaccess/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@


class SessionWithHeaderRedirection(requests.Session):
"""
Requests removes auth headers if the redirect happens outside the
"""Requests removes auth headers if the redirect happens outside the
original req domain.
"""

Expand Down Expand Up @@ -85,7 +84,8 @@ def login(
) -> Any:
"""Authenticate with Earthdata login.
Parameters:
Parameters
----------
strategy:
The authentication method.
Expand All @@ -96,8 +96,10 @@ def login(
persist: Will persist credentials in a `.netrc` file.
system (Env): the EDL endpoint to log in to Earthdata, defaults to PROD
Returns:
Returns
-------
An instance of Auth.
"""
if system is not None:
self._set_earthdata_system(system)
Expand Down Expand Up @@ -191,13 +193,16 @@ def get_s3_credentials(
"""Gets AWS S3 credentials for a given NASA cloud provider.
The easier way is to use the DAAC short name; provider is optional if we know it.
Parameters:
Parameters
----------
daac: The name of a NASA DAAC, e.g. NSIDC or PODAAC.
provider: A valid cloud provider. Each DAAC has a provider code for their cloud distributions.
endpoint: Getting the credentials directly from the S3Credentials URL.
Returns:
Returns
-------
A Python dictionary with the temporary AWS S3 credentials.
"""
if self.authenticated:
session = SessionWithHeaderRedirection(self.username, self.password)
Expand Down Expand Up @@ -246,11 +251,14 @@ def get_s3_credentials(
def get_session(self, bearer_token: bool = True) -> requests.Session:
"""Returns a new request session instance.
Parameters:
Parameters
----------
bearer_token: whether to include bearer token
Returns:
Returns
-------
class Session instance with Auth and bearer token headers
"""
session = SessionWithHeaderRedirection()
if bearer_token and self.authenticated:
Expand Down
2 changes: 1 addition & 1 deletion earthaccess/formatters.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@


def _load_static_files() -> List[str]:
"""Load styles"""
"""Load styles."""
return [
importlib_resources.files("earthaccess.css").joinpath(fname).read_text("utf8")
for fname in STATIC_FILES
Expand Down
Loading

0 comments on commit dc87dd7

Please sign in to comment.