[WIP] Order data with Harmony instead of ECS

.github/workflows/test.yml

@@ -0,0 +1,50 @@
+name: Test
+
+on:
+  pull_request:
+    branches:
+      - v2  # TODO: Remove!
+      - development
+      - main
+  push:
+    branches:
+      - main
+      - development
+
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+
+      - name: Install package and test dependencies
+        run: |
+          # TODO: Once we have expressed dependencies in a more standard way:
+          # python -m pip install .[test]
+          python -m pip install .
+          python -m pip install -r requirements-dev.txt
+
+      - name: Type-check package
+        run: mypy
+
+      - name: Unit test
+        # TODO: Test behind EDL
+        run: pytest icepyx/ --verbose --cov app --ignore=icepyx/tests/test_behind_NSIDC_API_login.py --ignore=icepyx/tests/test_auth.py
+
+      - name: Upload coverage report
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

icepyx/core/APIformatting.py

@@ -1,6 +1,9 @@
-# Generate and format information for submitting to API (CMR and NSIDC)
+"""Generate and format information for submitting to API (CMR and NSIDC)."""
 
 import datetime as dt
+from typing import Any, Generic, Literal, TypeVar, overload
+
+from icepyx.core.types import CMRParams, EGISpecificParams, EGISpecificParamsSubset
 
 
 # ----------------------------------------------------------------------
@@ -122,14 +125,17 @@ def combine_params(*param_dicts):
     """
     Combine multiple dictionaries into one.
 
+    Merging is performed in sequence using `dict.update()`; dictionaries later in the
+    list overwrite those earlier.
+
     Parameters
     ----------
     params : dictionaries
         Unlimited number of dictionaries to combine
 
     Returns
     -------
-    single dictionary of all input dictionaries combined
+    A single dictionary of all input dictionaries combined
 
     Examples
     --------
@@ -181,12 +187,56 @@ def to_string(params):
     return "&".join(param_list)
 
 
+ParameterType = Literal["CMR", "required", "subset"]
+# DevGoal: When Python 3.12 is minimum supported version, migrate to PEP695 style
+T = TypeVar("T", contravariant=False, bound=ParameterType)
+
+
+class _FmtedKeysDescriptor:
+    """Enable the Parameters class' fmted_keys property to be typechecked correctly.
+
+    See: https://github.com/microsoft/pyright/issues/3071#issuecomment-1043978070
+    """
+
+    @overload
+    def __get__(  # type: ignore
+        self,
+        instance: 'Parameters[Literal["CMR"]]',
+        owner: Any,
+    ) -> CMRParams: ...
+
+    @overload
+    def __get__(
+        self,
+        instance: 'Parameters[Literal["required"]]',
+        owner: Any,
+    ) -> EGISpecificParams: ...
+
+    @overload
+    def __get__(
+        self,
+        instance: 'Parameters[Literal["subset"]]',
+        owner: Any,
+    ) -> EGISpecificParamsSubset: ...
+
+    def __get__(
+        self,
+        instance: "Parameters",
+        owner: Any,
+    ) -> CMRParams | EGISpecificParams:
+        """
+        Returns the dictionary of formatted keys associated with the
+        parameter object.
+        """
+        return instance._fmted_keys
+
+
 # ----------------------------------------------------------------------
 # DevNote: Currently, this class is not tested!!
 # DevGoal: this could be expanded, similar to the variables class, to provide users with valid options if need be
 # DevGoal: currently this does not do much by way of checking/formatting of other subsetting options (reprojection or formats)
 # it would be great to incorporate that so that people can't just feed any keywords in...
-class Parameters:
+class Parameters(Generic[T]):
     """
     Build and update the parameter lists needed to submit a data order
 
@@ -204,12 +254,19 @@ class Parameters:
         on the type of query. Must be one of ['search','download']
     """
 
-    def __init__(self, partype, values=None, reqtype=None):
+    fmted_keys = _FmtedKeysDescriptor()
+
+    def __init__(
+        self,
+        partype: T,
+        values=None,
+        reqtype=None,
+    ):
         assert partype in [
             "CMR",
             "required",
             "subset",
-        ], "You need to submit a valid parametery type."
+        ], "You need to submit a valid parameter type."
         self.partype = partype
 
         if partype == "required":
@@ -240,15 +297,7 @@ def poss_keys(self):
 
     #     return self._wanted
 
-    @property
-    def fmted_keys(self):
-        """
-        Returns the dictionary of formatted keys associated with the
-        parameter object.
-        """
-        return self._fmted_keys
-
-    def _get_possible_keys(self):
+    def _get_possible_keys(self) -> dict[str, list[str]]:
         """
         Use the parameter type to get a list of possible parameter keys.
         """
@@ -345,7 +394,7 @@ def check_values(self):
         else:
             return False
 
-    def build_params(self, **kwargs):
+    def build_params(self, **kwargs) -> None:
         """
         Build the parameter dictionary of formatted key:value pairs for submission to NSIDC
         in the data request.
@@ -393,8 +442,6 @@ def build_params(self, **kwargs):
                             self._fmted_keys.update({key: kwargs[key]})
                         except KeyError:
                             self._fmted_keys.update({key: kwargs["product"]})
-                    elif key == "version":
-                        self._fmted_keys.update({key: kwargs["version"]})
                     elif key in kwargs:
                         self._fmted_keys.update({key: kwargs[key]})
                     elif key in defaults:
@@ -411,7 +458,10 @@ def build_params(self, **kwargs):
 
                 for key in opt_keys:
                     if key == "Coverage" and key in kwargs:
-                        # DevGoal: make there be an option along the lines of Coverage=default, which will get the default variables for that product without the user having to input is2obj.build_wanted_wanted_var_list as their input value for using the Coverage kwarg
+                        # DevGoal: make an option along the lines of Coverage=default,
+                        # which will get the default variables for that product without
+                        # the user having to input is2obj.build_wanted_wanted_var_list
+                        # as their input value for using the Coverage kwarg
                         self._fmted_keys.update(
                             {key: _fmt_var_subset_list(kwargs[key])}
                         )
@@ -438,3 +488,8 @@ def build_params(self, **kwargs):
                             k = "Boundingshape"
 
                     self._fmted_keys.update({k: kwargs["spatial_extent"]})
+
+
+CMRParameters = Parameters[Literal["CMR"]]
+RequiredParameters = Parameters[Literal["required"]]
+SubsetParameters = Parameters[Literal["subset"]]

icepyx/core/auth.py

@@ -55,7 +55,7 @@ def __init__(self, auth=None):
         self._s3login_credentials = None
         self._s3_initial_ts = None  # timer for 1h expiration on s3 credentials
 
-    def __str__(self):
+    def __str__(self) -> str:
         if self.session:
             repr_string = "EarthdataAuth obj with session initialized"
         else:

icepyx/core/granules.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import datetime
 import requests
 import time
@@ -7,12 +9,15 @@
 import numpy as np
 import os
 import pprint
-from xml.etree import ElementTree as ET
 import zipfile
+from requests.compat import unquote
+from xml.etree import ElementTree as ET
 
 import icepyx.core.APIformatting as apifmt
-from icepyx.core.auth import EarthdataAuthMixin
 import icepyx.core.exceptions
+from icepyx.core.auth import EarthdataAuthMixin
+from icepyx.core.types import CMRParams, EGISpecificParams
+from icepyx.core.urls import DOWNLOAD_BASE_URL, ORDER_BASE_URL, GRANULE_SEARCH_BASE_URL
 
 
 def info(grans):
@@ -168,14 +173,19 @@
     # ----------------------------------------------------------------------
     # Methods
 
-    def get_avail(self, CMRparams, reqparams, cloud=False):
+    def get_avail(
+        self,
+        CMRparams: CMRParams | None,
+        reqparams: EGISpecificParams | None,
+        cloud=False,
+    ):
         """
         Get a list of available granules for the query object's parameters.
         Generates the `avail` attribute of the granules object.
 
         Parameters
         ----------
-        CMRparams : dictionary
+        CMRparams :
             Dictionary of properly formatted CMR search parameters.
         reqparams : dictionary
             Dictionary of properly formatted parameters required for searching, ordering,
@@ -201,8 +211,6 @@
         # if not hasattr(self, 'avail'):
         self.avail = []
 
-        granule_search_url = "https://cmr.earthdata.nasa.gov/search/granules"
-
         headers = {"Accept": "application/json", "Client-Id": "icepyx"}
         # note we should also check for errors whenever we ping NSIDC-API -
         # make a function to check for errors
@@ -220,7 +228,7 @@
                 headers["CMR-Search-After"] = cmr_search_after
 
             response = requests.get(
-                granule_search_url,
+                GRANULE_SEARCH_BASE_URL,
                 headers=headers,
                 params=apifmt.to_string(params),
             )
@@ -261,13 +269,13 @@
     # DevGoal: add kwargs to allow subsetting and more control over request options.
     def place_order(
         self,
-        CMRparams,
-        reqparams,
+        CMRparams: CMRParams,
+        reqparams: EGISpecificParams,
         subsetparams,
         verbose,
         subset=True,
         geom_filepath=None,
-    ):  # , **kwargs):
+    ):
         """
         Place an order for the available granules for the query object.
         Adds the list of zipped files (orders) to the granules data object (which is
@@ -276,11 +284,11 @@
 
         Parameters
         ----------
-        CMRparams : dictionary
+        CMRparams :
             Dictionary of properly formatted CMR search parameters.
-        reqparams : dictionary
+        reqparams :
             Dictionary of properly formatted parameters required for searching, ordering,
-            or downloading from NSIDC.
+            or downloading from EGI.
         subsetparams : dictionary
             Dictionary of properly formatted subsetting parameters. An empty dictionary
             is passed as input here when subsetting is set to False in query methods.
@@ -308,8 +316,6 @@
         query.Query.order_granules
         """
 
-        base_url = "https://n5eil02u.ecs.nsidc.org/egi/request"
-
         self.get_avail(CMRparams, reqparams)
 
         if subset is False:
@@ -345,7 +351,7 @@
             )
             request_params.update({"page_num": page_num})
 
-            request = self.session.get(base_url, params=request_params)
+            request = self.session.get(ORDER_BASE_URL, params=request_params)
 
             # DevGoal: use the request response/number to do some error handling/
             # give the user better messaging for failures
@@ -361,7 +367,7 @@
             request.raise_for_status()
             esir_root = ET.fromstring(request.content)
             if verbose is True:
-                print("Order request URL: ", requests.utils.unquote(request.url))
+                print("Order request URL: ", unquote(request.url))
                 print(
                     "Order request response XML content: ",
                     request.content.decode("utf-8"),
@@ -377,7 +383,7 @@
             print("order ID: ", orderID)
 
             # Create status URL
-            statusURL = base_url + "/" + orderID
+            statusURL = ORDER_BASE_URL + "/" + orderID
             if verbose is True:
                 print("status URL: ", statusURL)
 
@@ -399,6 +405,7 @@
             print("Initial status of your order request at NSIDC is: ", status)
 
             # Continue loop while request is still processing
+            loop_root = None
             while status == "pending" or status == "processing":
                 print(
                     "Your order status is still ",
@@ -422,6 +429,13 @@
                 if status == "pending" or status == "processing":
                     continue
 
+            if not isinstance(loop_root, ET.Element):
+                # The typechecker determined that loop_root could be unbound at this
+                # point. We know for sure this shouldn't be possible, though, because
+                # the while loop should run once.
+                # See: https://github.com/microsoft/pyright/discussions/2033
+                raise RuntimeError("Programmer error!")
+
             # Order can either complete, complete_with_errors, or fail:
             # Provide complete_with_errors error message:
             if status == "complete_with_errors" or status == "failed":
@@ -522,7 +536,7 @@
                 i_order = self.orderIDs.index(order_start) + 1
 
         for order in self.orderIDs[i_order:]:
-            downloadURL = "https://n5eil02u.ecs.nsidc.org/esir/" + order + ".zip"
+            downloadURL = f"{DOWNLOAD_BASE_URL}/{order}.zip"
             # DevGoal: get the download_url from the granules
 
             if verbose is True: