From 228972f33d1323a86191fc207940a4a0005b35fc Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Mon, 8 Jan 2024 13:45:17 -0500 Subject: [PATCH 01/16] add auth_info property --- earthaccess/auth.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/earthaccess/auth.py b/earthaccess/auth.py index 9a3b22cb..a49fc200 100644 --- a/earthaccess/auth.py +++ b/earthaccess/auth.py @@ -62,6 +62,33 @@ def __init__(self) -> None: self.EDL_GENERATE_TOKENS_URL = "https://urs.earthdata.nasa.gov/api/users/token" self.EDL_REVOKE_TOKEN = "https://urs.earthdata.nasa.gov/api/users/revoke_token" + def __str__(self) -> str: + print_str = "Authentication Info\n" + "----------\n" + for k, v in self.auth_info: + print_str += str("{}: {}\n".format(k, v)) + + return print_str + + @property + def auth_info(self) -> Dict: + """Get information about the authentication session + + Returns: + Dict: information about the auth object + """ + summary_dict: Dict[str, Any] + summary_dict = { + "authenticated?": self.authenticated, + "tokens": self.tokens, + } + + #modify this to get the region check if in s3 + # add a separate in uswest2 access point to api? + if "Region" in self.s3_bucket(): + summary_dict["cloud-info"] = self.s3_bucket() + + return summary_dict + def login(self, strategy: str = "netrc", persist: bool = False) -> Any: """Authenticate with Earthdata login @@ -154,7 +181,7 @@ def get_s3_credentials( daac: the name of a NASA DAAC, i.e. NSIDC or PODAAC endpoint: getting the credentials directly from the S3Credentials URL - Rreturns: + Returns: A Python dictionary with the temporary AWS S3 credentials """ From 4b38af7a31b79a69e2823e8188cea291af30c084 Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Mon, 8 Jan 2024 14:35:15 -0500 Subject: [PATCH 02/16] update function lists --- earthaccess/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/earthaccess/__init__.py b/earthaccess/__init__.py index 4977bf38..050b5495 100644 --- a/earthaccess/__init__.py +++ b/earthaccess/__init__.py @@ -10,11 +10,13 @@ get_requests_https_session, get_s3_credentials, get_s3fs_session, + get_edl_token, granule_query, login, open, search_data, search_datasets, + in_us_west_2, ) from .auth import Auth from .kerchunk import consolidate_metadata @@ -24,6 +26,7 @@ logger = logging.getLogger(__name__) __all__ = [ + # api.py "login", "search_datasets", "search_data", @@ -31,15 +34,21 @@ "get_fsspec_https_session", "get_s3fs_session", "get_s3_credentials", + "get_edl_token" "granule_query", "collection_query", "open", "download", + "auth_environ", + "in_us_west_2", + # search.py "DataGranules", "DataCollections", + # auth.py "Auth", + # store.py "Store", - "auth_environ", + # kerchunk "consolidate_metadata", ] From 967794d229010167e3df2271e6069720f1e6c14c Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Mon, 8 Jan 2024 14:38:44 -0500 Subject: [PATCH 03/16] add in_us_west_2 check to API --- earthaccess/api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/earthaccess/api.py b/earthaccess/api.py index 8e518912..e237ea34 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -362,3 +362,12 @@ def auth_environ() -> Dict[str, str]: "`auth_environ()` requires you to first authenticate with `earthaccess.login()`" ) return {"EARTHDATA_USERNAME": auth.username, "EARTHDATA_PASSWORD": auth.password} + + +def in_us_west_2() -> bool: + """Returns true if the user is in AWS region us-west-2 + + Returns: + bool: boolean indicating if the user is in AWS region us-west-2 + """ + return earthaccess.__store__._running_in_us_west_2() From 032880df3b807667798d21f5649ef9912d518bda Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Mon, 8 Jan 2024 14:39:25 -0500 Subject: [PATCH 04/16] switch to repr for auth --- earthaccess/auth.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/earthaccess/auth.py b/earthaccess/auth.py index a49fc200..60fe342c 100644 --- a/earthaccess/auth.py +++ b/earthaccess/auth.py @@ -62,9 +62,9 @@ def __init__(self) -> None: self.EDL_GENERATE_TOKENS_URL = "https://urs.earthdata.nasa.gov/api/users/token" self.EDL_REVOKE_TOKEN = "https://urs.earthdata.nasa.gov/api/users/revoke_token" - def __str__(self) -> str: - print_str = "Authentication Info\n" + "----------\n" - for k, v in self.auth_info: + def __repr__(self) -> str: + print_str = "Authentication Info\n" + "-------------------\n" + for k, v in self.auth_info.items(): print_str += str("{}: {}\n".format(k, v)) return print_str @@ -82,11 +82,6 @@ def auth_info(self) -> Dict: "tokens": self.tokens, } - #modify this to get the region check if in s3 - # add a separate in uswest2 access point to api? - if "Region" in self.s3_bucket(): - summary_dict["cloud-info"] = self.s3_bucket() - return summary_dict def login(self, strategy: str = "netrc", persist: bool = False) -> Any: From 467689c5c9954508ba487ac170683acce68e6217 Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Mon, 8 Jan 2024 14:43:26 -0500 Subject: [PATCH 05/16] return string instead of boolean --- earthaccess/api.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/earthaccess/api.py b/earthaccess/api.py index e237ea34..1cea0abb 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -364,10 +364,14 @@ def auth_environ() -> Dict[str, str]: return {"EARTHDATA_USERNAME": auth.username, "EARTHDATA_PASSWORD": auth.password} -def in_us_west_2() -> bool: +def in_us_west_2() -> str: """Returns true if the user is in AWS region us-west-2 Returns: - bool: boolean indicating if the user is in AWS region us-west-2 + str: string indicating if the user is in AWS region us-west-2 """ - return earthaccess.__store__._running_in_us_west_2() + if earthaccess.__store__._running_in_us_west_2() is True: + msg = "You are running in AWS region 'us-west-2'" + else: + msg = "You are not running in AWS region 'us-west-2'" + return msg From 1a32effa3f49c814be0858506fab0a3f8b4c0db1 Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Wed, 10 Jan 2024 15:11:10 -0500 Subject: [PATCH 06/16] run linter --- earthaccess/__init__.py | 7 ++----- earthaccess/api.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/earthaccess/__init__.py b/earthaccess/__init__.py index 050b5495..c8f859f1 100644 --- a/earthaccess/__init__.py +++ b/earthaccess/__init__.py @@ -10,13 +10,11 @@ get_requests_https_session, get_s3_credentials, get_s3fs_session, - get_edl_token, - granule_query, + in_us_west_2, login, open, search_data, search_datasets, - in_us_west_2, ) from .auth import Auth from .kerchunk import consolidate_metadata @@ -34,8 +32,7 @@ "get_fsspec_https_session", "get_s3fs_session", "get_s3_credentials", - "get_edl_token" - "granule_query", + "get_edl_token" "granule_query", "collection_query", "open", "download", diff --git a/earthaccess/api.py b/earthaccess/api.py index 1cea0abb..ca57a820 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -372,6 +372,6 @@ def in_us_west_2() -> str: """ if earthaccess.__store__._running_in_us_west_2() is True: msg = "You are running in AWS region 'us-west-2'" - else: + else: msg = "You are not running in AWS region 'us-west-2'" return msg From 0b88005a4d251b7eecd5dfab8d9f4fc06fe5f7ea Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Tue, 6 Feb 2024 13:41:59 -0500 Subject: [PATCH 07/16] replace us-west-2 check with boto3 functionality --- earthaccess/store.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/earthaccess/store.py b/earthaccess/store.py index 645721ad..3d693bf8 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Mapping, Optional, Tuple, Union from uuid import uuid4 +import boto3 import fsspec import requests import s3fs @@ -140,26 +141,12 @@ def _own_s3_credentials(self, links: List[Dict[str, Any]]) -> Union[str, None]: return None def _running_in_us_west_2(self) -> bool: - session = self.auth.get_session() - try: - # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html - token_ = session.put( - "http://169.254.169.254/latest/api/token", - headers={"X-aws-ec2-metadata-token-ttl-seconds": "21600"}, - timeout=1, - ) - resp = session.get( - "http://169.254.169.254/latest/meta-data/placement/region", - timeout=1, - headers={"X-aws-ec2-metadata-token": token_.text}, - ) - except Exception: - return False - - if resp.status_code == 200 and b"us-west-2" == resp.content: - # On AWS in region us-west-2 + if (boto3.client('s3').meta.region_name == 'us-west-2'): return True - return False + else: + raise ValueError('Your instance is not running inside the' + ' AWS us-west-2 region.' + ' You will not be able to directly access NASA Earthdata S3 buckets') def set_requests_session( self, url: str, method: str = "get", bearer_token: bool = False From 7c5e027d7bc9888f93c2cb8a0c7b211554bb59f8 Mon Sep 17 00:00:00 2001 From: jessicas11 Date: Tue, 6 Feb 2024 19:17:21 +0000 Subject: [PATCH 08/16] change api to error --- binder/environment.yml | 1 + earthaccess/api.py | 7 ++++--- earthaccess/store.py | 4 +--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/binder/environment.yml b/binder/environment.yml index 2bc59969..c59ef68d 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -2,6 +2,7 @@ name: earthaccess channels: - conda-forge dependencies: + - boto3 - python=3.9 - xarray>=0.19 - dask>=2022.1 diff --git a/earthaccess/api.py b/earthaccess/api.py index ca57a820..678698b0 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -371,7 +371,8 @@ def in_us_west_2() -> str: str: string indicating if the user is in AWS region us-west-2 """ if earthaccess.__store__._running_in_us_west_2() is True: - msg = "You are running in AWS region 'us-west-2'" + return "You are running in AWS region 'us-west-2'" else: - msg = "You are not running in AWS region 'us-west-2'" - return msg + raise ValueError('Your instance is not running inside the' + ' AWS us-west-2 region.' + ' You will not be able to directly access NASA Earthdata S3 buckets') diff --git a/earthaccess/store.py b/earthaccess/store.py index 3d693bf8..37c4c77a 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -144,9 +144,7 @@ def _running_in_us_west_2(self) -> bool: if (boto3.client('s3').meta.region_name == 'us-west-2'): return True else: - raise ValueError('Your instance is not running inside the' - ' AWS us-west-2 region.' - ' You will not be able to directly access NASA Earthdata S3 buckets') + return False def set_requests_session( self, url: str, method: str = "get", bearer_token: bool = False From f9115faeb77b0fe3e30330654345fde6cc3876f0 Mon Sep 17 00:00:00 2001 From: Chris Battisto <34325676+battistowx@users.noreply.github.com> Date: Tue, 6 Feb 2024 13:37:01 -0600 Subject: [PATCH 09/16] Change boto3 region check to use botocore --- earthaccess/store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/earthaccess/store.py b/earthaccess/store.py index 37c4c77a..e0a3624c 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List, Mapping, Optional, Tuple, Union from uuid import uuid4 -import boto3 +import botocore.session import fsspec import requests import s3fs @@ -141,7 +141,7 @@ def _own_s3_credentials(self, links: List[Dict[str, Any]]) -> Union[str, None]: return None def _running_in_us_west_2(self) -> bool: - if (boto3.client('s3').meta.region_name == 'us-west-2'): + if (botocore.session.get_session().get_config_variable('region') == 'us-west-2'): return True else: return False From c4135d77958d74c027c560c65a3da62dc2d4500d Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Tue, 6 Feb 2024 14:53:08 -0500 Subject: [PATCH 10/16] use double quotes for ruff --- earthaccess/api.py | 6 +++--- earthaccess/store.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/earthaccess/api.py b/earthaccess/api.py index 678698b0..3ce302eb 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -373,6 +373,6 @@ def in_us_west_2() -> str: if earthaccess.__store__._running_in_us_west_2() is True: return "You are running in AWS region 'us-west-2'" else: - raise ValueError('Your instance is not running inside the' - ' AWS us-west-2 region.' - ' You will not be able to directly access NASA Earthdata S3 buckets') + raise ValueError("Your instance is not running inside the" + " AWS us-west-2 region." + " You will not be able to directly access NASA Earthdata S3 buckets.") diff --git a/earthaccess/store.py b/earthaccess/store.py index e0a3624c..534896b9 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -141,7 +141,7 @@ def _own_s3_credentials(self, links: List[Dict[str, Any]]) -> Union[str, None]: return None def _running_in_us_west_2(self) -> bool: - if (botocore.session.get_session().get_config_variable('region') == 'us-west-2'): + if (botocore.session.get_session().get_config_variable("region") == "us-west-2"): return True else: return False From 45e2ea12cdaf74670968a12854b5714e3ce016ab Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Tue, 6 Feb 2024 14:55:42 -0500 Subject: [PATCH 11/16] fix more formatting --- earthaccess/api.py | 8 +++++--- earthaccess/store.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/earthaccess/api.py b/earthaccess/api.py index 3ce302eb..146a0279 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -373,6 +373,8 @@ def in_us_west_2() -> str: if earthaccess.__store__._running_in_us_west_2() is True: return "You are running in AWS region 'us-west-2'" else: - raise ValueError("Your instance is not running inside the" - " AWS us-west-2 region." - " You will not be able to directly access NASA Earthdata S3 buckets.") + raise ValueError( + "Your instance is not running inside the" + " AWS us-west-2 region." + " You will not be able to directly access NASA Earthdata S3 buckets." + ) \ No newline at end of file diff --git a/earthaccess/store.py b/earthaccess/store.py index 534896b9..a8363a5d 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -141,7 +141,7 @@ def _own_s3_credentials(self, links: List[Dict[str, Any]]) -> Union[str, None]: return None def _running_in_us_west_2(self) -> bool: - if (botocore.session.get_session().get_config_variable("region") == "us-west-2"): + if botocore.session.get_session().get_config_variable("region") == "us-west-2": return True else: return False From b1a1f2737cf7f18e65d099f0a8b7cf9a6d91ae03 Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Tue, 6 Feb 2024 14:56:45 -0500 Subject: [PATCH 12/16] add newline --- earthaccess/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthaccess/api.py b/earthaccess/api.py index 146a0279..1dd6cee0 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -377,4 +377,4 @@ def in_us_west_2() -> str: "Your instance is not running inside the" " AWS us-west-2 region." " You will not be able to directly access NASA Earthdata S3 buckets." - ) \ No newline at end of file + ) From d0a25d7cef392d439ec911b8459d4137a13e8238 Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Tue, 6 Feb 2024 14:57:47 -0500 Subject: [PATCH 13/16] remove tab --- earthaccess/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthaccess/api.py b/earthaccess/api.py index 1dd6cee0..ef0721e1 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -377,4 +377,4 @@ def in_us_west_2() -> str: "Your instance is not running inside the" " AWS us-west-2 region." " You will not be able to directly access NASA Earthdata S3 buckets." - ) + ) From 4c8837d5a2b6434e2179f9c02ddae74027548c30 Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Tue, 6 Feb 2024 15:02:29 -0500 Subject: [PATCH 14/16] remove more tabs --- earthaccess/api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/earthaccess/api.py b/earthaccess/api.py index ef0721e1..38358868 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -374,7 +374,7 @@ def in_us_west_2() -> str: return "You are running in AWS region 'us-west-2'" else: raise ValueError( - "Your instance is not running inside the" - " AWS us-west-2 region." - " You will not be able to directly access NASA Earthdata S3 buckets." - ) + "Your instance is not running inside the" + " AWS us-west-2 region." + " You will not be able to directly access NASA Earthdata S3 buckets." + ) From 5a28e74f46848bed29c294cde7e0f5d6a041b787 Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Tue, 6 Feb 2024 15:03:26 -0500 Subject: [PATCH 15/16] remove more tabs --- earthaccess/api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/earthaccess/api.py b/earthaccess/api.py index 38358868..fac05a9e 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -374,7 +374,7 @@ def in_us_west_2() -> str: return "You are running in AWS region 'us-west-2'" else: raise ValueError( - "Your instance is not running inside the" - " AWS us-west-2 region." - " You will not be able to directly access NASA Earthdata S3 buckets." - ) + "Your instance is not running inside the" + " AWS us-west-2 region." + " You will not be able to directly access NASA Earthdata S3 buckets." + ) From aaf884d336191f38cec49c07d6048cfd45f76784 Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Tue, 6 Feb 2024 15:05:47 -0500 Subject: [PATCH 16/16] update docstring --- earthaccess/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthaccess/api.py b/earthaccess/api.py index fac05a9e..35048745 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -365,7 +365,7 @@ def auth_environ() -> Dict[str, str]: def in_us_west_2() -> str: - """Returns true if the user is in AWS region us-west-2 + """Returns a message indicating if the user is in AWS region us-west-2 Returns: str: string indicating if the user is in AWS region us-west-2