Fix CMR-related type hints (nsidc#510)

There were a number of type hints in `search.py` and `api.py` related to
CMR queries that were incorrect. These were fixed. In addition, there
were a number of other static type errors that were masked because of
ignored `cmr` imports. Added type stubs for `python_cmr` library to
unmask and address these additional type errors.

In addition:

- Aligned vcrpy usage with VCRTestCase as per
  https://vcrpy.readthedocs.io/en/latest/usage.html#unittest-integration
- Restored use of session for CMR paged queries, which was accidentally
  removed with the introduction of Search-After functionality.
- Wrapped a number of docstrings at 88 characters per ruff configuration

Fixes nsidc#508

.gitignore

@@ -20,8 +20,78 @@ docs/tutorials/data
 tests/integration/data
 .ruff_cache
 
-# OS X
+notebooks/data/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Created by https://www.toptal.com/developers/gitignore/api/macos
+# Edit at https://www.toptal.com/developers/gitignore?templates=macos
+
+### macOS ###
+# General
 .DS_Store
+.AppleDouble
+.LSOverride
 
-notebooks/data/
-.vscode
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### macOS Patch ###
+# iCloud generated files
+*.icloud
+
+# End of https://www.toptal.com/developers/gitignore/api/macos
+
+# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode
+# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode
+
+### VisualStudioCode ###
+.vscode/
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode
+
+# Created by https://www.toptal.com/developers/gitignore/api/direnv
+# Edit at https://www.toptal.com/developers/gitignore?templates=direnv
+
+### direnv ###
+.direnv
+.envrc
+
+# End of https://www.toptal.com/developers/gitignore/api/direnv

CHANGELOG.md

@@ -5,6 +5,10 @@
     * fixed 483 by extracting a common CMR query method for collections and granules using SearchAfter header
     * Added VCR support for verifying the API call to CMR and the parsing of returned results without relying on CMR availability post development
 
+* Enhancements:
+  * Corrected and enhanced static type hints for functions and methods that make
+    CMR queries or handle CMR query results (#508)
+
 ## [v0.9.0] 2024-02-28
 
 * Bug fixes:

CONTRIBUTING.md

@@ -84,6 +84,15 @@ Finally, for _development dependencies only_, you must add an entry to
 make format lint
 ```
 
+We attempt to provide comprehensive type annotations within this repository.  If
+you do not provide fully annotated functions or methods, the `lint` command will
+fail.  Over time, we plan to increase type-checking strictness in order to
+ensure more precise, beneficial type annotations.
+
+We have included type stubs for the untyped `python-cmr` library, which we
+intend to eventually upstream.  Since `python-cmr` exposes the `cmr` package,
+the stubs appear under `stubs/cmr`.
+
 ### Requirements to merge code (Pull Request Process)
 
 - you must include test coverage

ci/environment-mindeps.yaml

@@ -17,11 +17,14 @@ dependencies:
   - multimethod=1.8
   - python-dateutil=2.8.2
   - importlib-resources=6.3.2
+  - typing-extensions=4.10.0
   # test dependencies
   - responses
   - pytest
   - pytest-cov
+  - python-magic
   - mypy
+  - types-python-dateutil
   - types-requests
   - types-setuptools
   - ruff

earthaccess/api.py

@@ -1,13 +1,12 @@
-from typing import Any, Dict, List, Optional, Type, Union
-
 import requests
 import s3fs
 from fsspec import AbstractFileSystem
+from typing_extensions import Any, Dict, List, Optional, Union
 
 import earthaccess
 
 from .auth import Auth
-from .results import DataGranule
+from .results import DataCollection, DataGranule
 from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery
 from .store import Store
 from .utils import _validation as validate
@@ -28,9 +27,7 @@ def _normalize_location(location: Optional[str]) -> Optional[str]:
     return location
 
 
-def search_datasets(
-    count: int = -1, **kwargs: Any
-) -> List[earthaccess.results.DataCollection]:
+def search_datasets(count: int = -1, **kwargs: Any) -> List[DataCollection]:
     """Search datasets using NASA's CMR.
 
     [https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html](https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html)
@@ -54,6 +51,9 @@ def search_datasets(
         A list of DataCollection results that can be used to get information about a
             dataset, e.g. concept_id, doi, etc.
 
+    Raises:
+        RuntimeError: The CMR query failed.
+
     Examples:
         ```python
         datasets = earthaccess.search_datasets(
@@ -78,9 +78,7 @@ def search_datasets(
     return query.get_all()
 
 
-def search_data(
-    count: int = -1, **kwargs: Any
-) -> List[earthaccess.results.DataGranule]:
+def search_data(count: int = -1, **kwargs: Any) -> List[DataGranule]:
     """Search dataset granules using NASA's CMR.
 
     [https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html](https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html)
@@ -104,6 +102,9 @@ def search_data(
         a list of DataGranules that can be used to access the granule files by using
             `download()` or `open()`.
 
+    Raises:
+        RuntimeError: The CMR query failed.
+
     Examples:
         ```python
         datasets = earthaccess.search_data(
@@ -178,6 +179,9 @@ def download(
 
     Returns:
         List of downloaded files
+
+    Raises:
+        Exception: A file download failed.
     """
     provider = _normalize_location(provider)
     if isinstance(granules, DataGranule):
@@ -194,7 +198,7 @@ def download(
 
 
 def open(
-    granules: Union[List[str], List[earthaccess.results.DataGranule]],
+    granules: Union[List[str], List[DataGranule]],
     provider: Optional[str] = None,
 ) -> List[AbstractFileSystem]:
     """Returns a list of fsspec file-like objects that can be used to access files
@@ -216,7 +220,7 @@ def open(
 def get_s3_credentials(
     daac: Optional[str] = None,
     provider: Optional[str] = None,
-    results: Optional[List[earthaccess.results.DataGranule]] = None,
+    results: Optional[List[DataGranule]] = None,
 ) -> Dict[str, Any]:
     """Returns temporary (1 hour) credentials for direct access to NASA S3 buckets. We can
     use the daac name, the provider, or a list of results from earthaccess.search_data().
@@ -239,7 +243,7 @@ def get_s3_credentials(
     return earthaccess.__auth__.get_s3_credentials(daac=daac, provider=provider)
 
 
-def collection_query() -> Type[CollectionQuery]:
+def collection_query() -> CollectionQuery:
     """Returns a query builder instance for NASA collections (datasets).
 
     Returns:
@@ -252,7 +256,7 @@ def collection_query() -> Type[CollectionQuery]:
     return query_builder
 
 
-def granule_query() -> Type[GranuleQuery]:
+def granule_query() -> GranuleQuery:
     """Returns a query builder instance for data granules
 
     Returns:
@@ -311,7 +315,7 @@ def get_requests_https_session() -> requests.Session:
 def get_s3fs_session(
     daac: Optional[str] = None,
     provider: Optional[str] = None,
-    results: Optional[earthaccess.results.DataGranule] = None,
+    results: Optional[DataGranule] = None,
 ) -> s3fs.S3FileSystem:
     """Returns a fsspec s3fs file session for direct access when we are in us-west-2.