From c821f12ce13349c5fa5b43aebbbb6ee9c9a603fe Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 15 Sep 2023 15:00:12 -0500 Subject: [PATCH 1/5] Update pickle logic --- earthaccess/store.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/earthaccess/store.py b/earthaccess/store.py index dd0f1369..3ea78ea7 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Union from uuid import uuid4 +from pickle import loads import fsspec import requests @@ -36,7 +37,7 @@ def __reduce__(self) -> Any: type(self.f), self.granule, earthaccess.__auth__, - self.f.__reduce__(), + dumps(self.f), ) def __repr__(self) -> str: @@ -65,7 +66,7 @@ def multi_thread_open(data: tuple) -> EarthAccessFile: def make_instance( - cls: Any, granule: DataGranule, auth: Auth, _reduce: Any + cls: Any, granule: DataGranule, auth: Auth, data: Any ) -> EarthAccessFile: # Attempt to re-authenticate if not earthaccess.__auth__.authenticated: @@ -78,9 +79,7 @@ def make_instance( # guaranteed to be the right one. return EarthAccessFile(earthaccess.open([granule])[0], granule) else: - func = _reduce[0] - args = _reduce[1] - return func(*args) + return EarthAccessFile(loads(data), granule) class Store(object): From a48f727579f93c279f56d5961631a031ead1f4aa Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 15 Sep 2023 15:07:57 -0500 Subject: [PATCH 2/5] dumps --- earthaccess/store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthaccess/store.py b/earthaccess/store.py index 3ea78ea7..3070bb07 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Union from uuid import uuid4 -from pickle import loads +from pickle import loads, dumps import fsspec import requests From 6823cdb74e7ddcaeaecdf64582af63bd146edbd9 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 15 Sep 2023 16:10:47 -0500 Subject: [PATCH 3/5] more --- earthaccess/store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthaccess/store.py b/earthaccess/store.py index 3070bb07..9828c18b 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -73,7 +73,7 @@ def make_instance( earthaccess.__auth__ = auth earthaccess.login() - if earthaccess.__store__.running_in_aws and cls is not s3fs.S3File: + if (earthaccess.__store__.running_in_aws and cls is not s3fs.S3File) or (not earthaccess.__store__.running_in_aws and cls is s3fs.S3File): # On AWS but not using a S3File. Reopen the file in this case for direct S3 access. # NOTE: This uses the first data_link listed in the granule. That's not # guaranteed to be the right one. From c684117ac1fdaae81779851ad38ed100fdca17df Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 18 Sep 2023 12:28:25 -0500 Subject: [PATCH 4/5] Lint --- earthaccess/api.py | 3 +-- earthaccess/store.py | 13 +++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/earthaccess/api.py b/earthaccess/api.py index ccb62200..0dc6961d 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -1,11 +1,10 @@ from typing import Any, Dict, List, Optional, Type, Union +import earthaccess import requests import s3fs from fsspec import AbstractFileSystem -import earthaccess - from .auth import Auth from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery from .store import Store diff --git a/earthaccess/store.py b/earthaccess/store.py index 9828c18b..3b5776e6 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -6,22 +6,21 @@ from functools import lru_cache from itertools import chain from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Union +from pickle import dumps, loads +from typing import Any, Dict, List, Optional, Union from uuid import uuid4 -from pickle import loads, dumps +import earthaccess import fsspec import requests import s3fs from multimethod import multimethod as singledispatchmethod from pqdm.threads import pqdm -import earthaccess - +from .auth import Auth from .daac import DAAC_TEST_URLS, find_provider from .results import DataGranule from .search import DataCollections -from .auth import Auth class EarthAccessFile(fsspec.spec.AbstractBufferedFile): @@ -73,7 +72,9 @@ def make_instance( earthaccess.__auth__ = auth earthaccess.login() - if (earthaccess.__store__.running_in_aws and cls is not s3fs.S3File) or (not earthaccess.__store__.running_in_aws and cls is s3fs.S3File): + if (earthaccess.__store__.running_in_aws and cls is not s3fs.S3File) or ( + not earthaccess.__store__.running_in_aws and cls is s3fs.S3File + ): # On AWS but not using a S3File. Reopen the file in this case for direct S3 access. # NOTE: This uses the first data_link listed in the granule. That's not # guaranteed to be the right one. From 9e3460f14ceecd52e56f9ce525ea914b139b1712 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 18 Sep 2023 14:39:59 -0500 Subject: [PATCH 5/5] Update comment --- earthaccess/store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/earthaccess/store.py b/earthaccess/store.py index 610c224c..4905be82 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -72,10 +72,11 @@ def make_instance( earthaccess.__auth__ = auth earthaccess.login() + # When sending EarthAccessFiles between processes, it's possible that + # we will need to switch between s3 <--> https protocols. if (earthaccess.__store__.running_in_aws and cls is not s3fs.S3File) or ( not earthaccess.__store__.running_in_aws and cls is s3fs.S3File ): - # On AWS but not using a S3File. Reopen the file in this case for direct S3 access. # NOTE: This uses the first data_link listed in the granule. That's not # guaranteed to be the right one. return EarthAccessFile(earthaccess.open([granule])[0], granule)