diff --git a/icepyx/core/APIformatting.py b/icepyx/core/APIformatting.py index dc20c6878..73047e05b 100644 --- a/icepyx/core/APIformatting.py +++ b/icepyx/core/APIformatting.py @@ -1,13 +1,11 @@ """Generate and format information for submitting to API (CMR and NSIDC).""" import datetime as dt -from typing import Any, Generic, Literal, Optional, TypeVar, Union, overload +from typing import Any, Generic, Literal, Optional, TypeVar, overload from icepyx.core.exceptions import ExhaustiveTypeGuardException, TypeGuardException -from icepyx.core.types import ( +from icepyx.core.types.api import ( CMRParams, - EGIParamsSubset, - EGIRequiredParams, ) # ---------------------------------------------------------------------- @@ -212,20 +210,22 @@ def __get__( self, instance: 'Parameters[Literal["required"]]', owner: Any, - ) -> EGIRequiredParams: ... + ): # -> EGIRequiredParams: ... + ... @overload def __get__( self, instance: 'Parameters[Literal["subset"]]', owner: Any, - ) -> EGIParamsSubset: ... + ): # -> EGIParamsSubset: ... + ... def __get__( self, instance: "Parameters", owner: Any, - ) -> Union[CMRParams, EGIRequiredParams, EGIParamsSubset]: + ) -> CMRParams: """ Returns the dictionary of formatted keys associated with the parameter object. diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index d6a519048..32ab7a048 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -17,15 +17,17 @@ import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin -from icepyx.core.cmr import CMR_PROVIDER +from icepyx.core.cmr import CMR_PROVIDER, get_concept_id import icepyx.core.exceptions -from icepyx.core.types import ( +from icepyx.core.harmony import HarmonyApi +from icepyx.core.types.api import ( CMRParams, - EGIRequiredParamsDownload, - EGIRequiredParamsSearch, ) from icepyx.core.urls import DOWNLOAD_BASE_URL, GRANULE_SEARCH_BASE_URL, ORDER_BASE_URL -from icepyx.uat import EDL_ACCESS_TOKEN + +# TODO: mix this into existing classes rather than declaring as a global +# variable. +HARMONY_API = HarmonyApi() def info(grans: list[dict]) -> dict[str, Union[int, float]]: @@ -191,7 +193,6 @@ def __init__( def get_avail( self, CMRparams: CMRParams, - reqparams: EGIRequiredParamsSearch, cloud: bool = False, ): """ @@ -222,9 +223,7 @@ def get_avail( query.Query.avail_granules """ - assert ( - CMRparams is not None and reqparams is not None - ), "Missing required input parameter dictionaries" + assert CMRparams is not None, "Missing required input parameter dictionary" # if not hasattr(self, 'avail'): self.avail = [] @@ -232,14 +231,12 @@ def get_avail( headers = { "Accept": "application/json", "Client-Id": "icepyx", - "Authorization": f"Bearer {EDL_ACCESS_TOKEN}", } # note we should also check for errors whenever we ping NSIDC-API - # make a function to check for errors params = apifmt.combine_params( CMRparams, - {k: reqparams[k] for k in ["short_name", "version", "page_size"]}, {"provider": CMR_PROVIDER}, ) @@ -292,7 +289,7 @@ def get_avail( def place_order( self, CMRparams: CMRParams, - reqparams: EGIRequiredParamsDownload, + reqparams, # : EGIRequiredParamsDownload, subsetparams, verbose, subset=True, @@ -337,7 +334,7 @@ def place_order( -------- query.Query.order_granules """ - raise icepyx.core.exceptions.RefactoringException + # raise icepyx.core.exceptions.RefactoringException self.get_avail(CMRparams, reqparams) @@ -348,6 +345,29 @@ def place_order( else: request_params = apifmt.combine_params(CMRparams, reqparams, subsetparams) + concept_id = get_concept_id( + product=request_params["short_name"], version=request_params["version"] + ) + + # TODO: At this point, the request parameters have been formatted into + # strings. `harmony-py` expects python objects (e.g., `dt.datetime` for + # temporal values) + + # Place the order. + # TODO: there are probably other options we want to more generically + # expose here. E.g., instead of just accepting a `bounding_box` of a + # particular flavor, we want to be able to pass in a polygon? + HARMONY_API.place_order( + concept_id=concept_id, + bounding_box=request_params["bounding_box"], + temporal=request_params["temporal"], + ) + + ######################################################################## + # Most of what exists after this point will go away. `harmony-py` deals + # with submitting orders and tracking status via a single job ID. + ######################################################################## + order_fn = ".order_restart" total_pages = int(np.ceil(len(self.avail) / reqparams["page_size"])) diff --git a/icepyx/core/harmony.py b/icepyx/core/harmony.py index 33274a5ac..4d978dae7 100644 --- a/icepyx/core/harmony.py +++ b/icepyx/core/harmony.py @@ -21,3 +21,29 @@ def get_capabilities(self, concept_id: str) -> dict[str, Any]: response = self.harmony_client.submit(capabilities_request) return response + + def place_order(self, concept_id: str, bounding_box, temporal) -> str: + """Places a Harmony order with the given parameters. + + Return a string representing a job ID. + """ + collection = harmony.Collection(id=concept_id) + request = harmony.Request( + collection=collection, + # TODO: add spatial bounding box is a `harmony.BBox`. + spatial=bounding_box, + # TODO: temporal should be a dict {"start": dt.datetime, "end": dt.datetime} + temporal=temporal, + ) + + if not request.is_valid(): + # TODO: consider more specific error class & message + raise RuntimeError("Failed to create valid request") + + job_id = self.harmony_client.submit(request) + + return job_id + + def check_order_status(self, job_id: str): + status = self.harmony_client.status(job_id) + return status diff --git a/icepyx/core/query.py b/icepyx/core/query.py index 573ca8b1b..89592e707 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -1,7 +1,7 @@ import datetime as dt from functools import cached_property import pprint -from typing import Optional, Union, cast +from typing import Optional, Union import geopandas as gpd import holoviews as hv @@ -17,11 +17,8 @@ import icepyx.core.is2ref as is2ref import icepyx.core.spatial as spat import icepyx.core.temporal as tp -from icepyx.core.types import ( +from icepyx.core.types.api import ( CMRParams, - EGIParamsSubset, - EGIRequiredParams, - EGIRequiredParamsDownload, ) import icepyx.core.validate_inputs as val from icepyx.core.variables import Variables as Variables @@ -597,7 +594,7 @@ def CMRparams(self) -> CMRParams: return self._CMRparams.fmted_keys @property - def reqparams(self) -> EGIRequiredParams: + def reqparams(self): # -> EGIRequiredParams: """ Display the required key:value pairs that will be submitted. It generates the dictionary if it does not already exist. @@ -613,8 +610,6 @@ def reqparams(self) -> EGIRequiredParams: >>> reg_a.reqparams # doctest: +SKIP {'short_name': 'ATL06', 'version': '006', 'page_size': 2000, 'page_num': 1, 'request_mode': 'async', 'include_meta': 'Y', 'client_string': 'icepyx'} """ - raise RefactoringException - if not hasattr(self, "_reqparams"): self._reqparams = apifmt.Parameters("required", reqtype="search") self._reqparams.build_params(product=self.product, version=self._version) @@ -624,7 +619,7 @@ def reqparams(self) -> EGIRequiredParams: # @property # DevQuestion: if I make this a property, I get a "dict" object is not callable # when I try to give input kwargs... what approach should I be taking? - def subsetparams(self, **kwargs) -> Union[EGIParamsSubset, dict[Never, Never]]: + def subsetparams(self, **kwargs): # -> Union[EGIParamsSubset, dict[Never, Never]]: """ Display the subsetting key:value pairs that will be submitted. It generates the dictionary if it does not already exist @@ -650,7 +645,7 @@ def subsetparams(self, **kwargs) -> Union[EGIParamsSubset, dict[Never, Never]]: {'time': '2019-02-20T00:00:00,2019-02-28T23:59:59', 'bbox': '-55.0,68.0,-48.0,71.0'} """ - raise RefactoringException + # raise RefactoringException if not hasattr(self, "_subsetparams"): self._subsetparams = apifmt.Parameters("subset") @@ -1024,12 +1019,13 @@ def order_granules( . Retry request status is: complete """ - breakpoint() - raise RefactoringException - - if not hasattr(self, "reqparams"): - self.reqparams + # breakpoint() + # raise RefactoringException + # TODO: this probably shouldn't be mutated based on which method is being called... + # It is also very confusing to have both `self.reqparams` and + # `self._reqparams`, each of which does something different! + self.reqparams if self._reqparams._reqtype == "search": self._reqparams._reqtype = "download" @@ -1065,7 +1061,7 @@ def order_granules( tempCMRparams["readable_granule_name[]"] = gran self.granules.place_order( tempCMRparams, - cast(EGIRequiredParamsDownload, self.reqparams), + self.reqparams, self.subsetparams(**kwargs), verbose, subset, @@ -1075,7 +1071,7 @@ def order_granules( else: self.granules.place_order( self.CMRparams, - cast(EGIRequiredParamsDownload, self.reqparams), + self.reqparams, self.subsetparams(**kwargs), verbose, subset, diff --git a/icepyx/core/spatial.py b/icepyx/core/spatial.py index fef61846f..234722327 100644 --- a/icepyx/core/spatial.py +++ b/icepyx/core/spatial.py @@ -821,3 +821,44 @@ def fmt_for_EGI(self) -> str: else: raise icepyx.core.exceptions.ExhaustiveTypeGuardException + + def fmt_for_harmony(self) -> ...: + """ + Format the spatial extent input into format expected by `harmony-py`. + + `harmony-py` can take two different spatial parameters: + + * `spatial`: "Bounding box spatial constraints on the data or Well Known + Text (WKT) string describing the spatial constraints." The "Bounding + box" is expected to be a `harmony.BBox`. + * `shape`: "a file path to an ESRI Shapefile zip, GeoJSON file, or KML + file to use for spatial subsetting. Note: not all collections support + shapefile subsetting" + + Question: is `spatial` the same as `shape`, in terms of performance? If + so, we could be consistent and always turn the input into geojson and + pass that along to harmony. Otherwise we should choose `spatial` if the + extent_type is bounding, otherwise `shape`. + Answer: No! They're not the same. They map to different harmony + parameters and each is a different service. E.g., some collections may + have bounding box subsetting while others have shape subsetting (or + both). + TODO: think more about how we verify if certain inputs are valid for + harmony. E.g., do we need to check the capabilities of each and + cross-check that with user inputs to determine which action to take? + Also: Does `icepyx` always perform subsetting based on user input? If + not, how do we determine which parameters are for finding granules vs + performing subetting? + + Question: is there any way to pass in a geojson string directly, so that + we do not have to mock out a file just for harmony? Answer: no, not + direcly. `harmony-py` wants a path to a file on disk. We may want to + have the function that submits the request to harmony with `harmony-py` + accept something that's easily-serializable to a geojson file so that it + can manage the lifespan of the file. It would be best (I think) to avoid + writing tmp files to disk in this function, because it doesn't know when + the request gets made/when to cleanup the file. That means that we may + leave stray files on the user's computer. Ideally, we would be able to + pass `harmony-py` a bytes object (or a shapely Polygon!) + """ + # TODO! diff --git a/icepyx/core/types/api.py b/icepyx/core/types/api.py index b29ba8fb4..9af9bb9d6 100644 --- a/icepyx/core/types/api.py +++ b/icepyx/core/types/api.py @@ -1,11 +1,14 @@ -from typing import Literal, TypedDict, Union +from typing import TypedDict, Union -from typing_extensions import NotRequired from pydantic import BaseModel +from typing_extensions import NotRequired CMRParamsBase = TypedDict( "CMRParamsBase", { + "short_name": str, + "version": str, + "page_size": int, "temporal": NotRequired[str], "options[readable_granule_name][pattern]": NotRequired[str], "options[spatial][or]": NotRequired[str], @@ -25,4 +28,4 @@ class CMRParamsWithPolygon(CMRParamsBase): CMRParams = Union[CMRParamsWithBbox, CMRParamsWithPolygon] -class HarmonyCoverageAPIParamsBase(BaseModel): +class HarmonyCoverageAPIParamsBase(BaseModel): ...