Skip to content

Commit

Permalink
WIP: begin implementing use of harmony-py
Browse files Browse the repository at this point in the history
Comment out some items that might be nice to reference (e.g,. where EGI-specific
parameters were assumed) and refactoring exceptions that are in the way.
  • Loading branch information
trey-stafford committed Nov 11, 2024
1 parent e7ff043 commit fb1eefd
Show file tree
Hide file tree
Showing 6 changed files with 85 additions and 41 deletions.
1 change: 0 additions & 1 deletion icepyx/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
from _icepyx_version import version as __version__

from icepyx.core.query import GenQuery, Query
from icepyx.core.read import Read
from icepyx.core.variables import Variables
Expand Down
14 changes: 7 additions & 7 deletions icepyx/core/APIformatting.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
"""Generate and format information for submitting to API (CMR and NSIDC)."""

import datetime as dt
from typing import Any, Generic, Literal, Optional, TypeVar, Union, overload
from typing import Any, Generic, Literal, Optional, TypeVar, overload

from icepyx.core.exceptions import ExhaustiveTypeGuardException, TypeGuardException
from icepyx.core.types import (
from icepyx.core.types.api import (
CMRParams,
EGIParamsSubset,
EGIRequiredParams,
)

# ----------------------------------------------------------------------
Expand Down Expand Up @@ -212,20 +210,22 @@ def __get__(
self,
instance: 'Parameters[Literal["required"]]',
owner: Any,
) -> EGIRequiredParams: ...
): # -> EGIRequiredParams: ...
...

@overload
def __get__(
self,
instance: 'Parameters[Literal["subset"]]',
owner: Any,
) -> EGIParamsSubset: ...
): # -> EGIParamsSubset: ...
...

def __get__(
self,
instance: "Parameters",
owner: Any,
) -> Union[CMRParams, EGIRequiredParams, EGIParamsSubset]:
) -> CMRParams:
"""
Returns the dictionary of formatted keys associated with the
parameter object.
Expand Down
46 changes: 33 additions & 13 deletions icepyx/core/granules.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,17 @@

import icepyx.core.APIformatting as apifmt
from icepyx.core.auth import EarthdataAuthMixin
from icepyx.core.cmr import CMR_PROVIDER
from icepyx.core.cmr import CMR_PROVIDER, get_concept_id
import icepyx.core.exceptions
from icepyx.core.types import (
from icepyx.core.harmony import HarmonyApi
from icepyx.core.types.api import (
CMRParams,
EGIRequiredParamsDownload,
EGIRequiredParamsSearch,
)
from icepyx.core.urls import DOWNLOAD_BASE_URL, GRANULE_SEARCH_BASE_URL, ORDER_BASE_URL
from icepyx.uat import EDL_ACCESS_TOKEN

# TODO: mix this into existing classes rather than declaring as a global
# variable.
HARMONY_API = HarmonyApi()


def info(grans: list[dict]) -> dict[str, Union[int, float]]:
Expand Down Expand Up @@ -191,7 +193,6 @@ def __init__(
def get_avail(
self,
CMRparams: CMRParams,
reqparams: EGIRequiredParamsSearch,
cloud: bool = False,
):
"""
Expand Down Expand Up @@ -222,24 +223,20 @@ def get_avail(
query.Query.avail_granules
"""

assert (
CMRparams is not None and reqparams is not None
), "Missing required input parameter dictionaries"
assert CMRparams is not None, "Missing required input parameter dictionary"

# if not hasattr(self, 'avail'):
self.avail = []

headers = {
"Accept": "application/json",
"Client-Id": "icepyx",
"Authorization": f"Bearer {EDL_ACCESS_TOKEN}",
}
# note we should also check for errors whenever we ping NSIDC-API -
# make a function to check for errors

params = apifmt.combine_params(
CMRparams,
{k: reqparams[k] for k in ["short_name", "version", "page_size"]},
{"provider": CMR_PROVIDER},
)

Expand Down Expand Up @@ -292,7 +289,7 @@ def get_avail(
def place_order(
self,
CMRparams: CMRParams,
reqparams: EGIRequiredParamsDownload,
reqparams, # : EGIRequiredParamsDownload,
subsetparams,
verbose,
subset=True,
Expand Down Expand Up @@ -337,7 +334,7 @@ def place_order(
--------
query.Query.order_granules
"""
raise icepyx.core.exceptions.RefactoringException
# raise icepyx.core.exceptions.RefactoringException

self.get_avail(CMRparams, reqparams)

Expand All @@ -348,6 +345,29 @@ def place_order(
else:
request_params = apifmt.combine_params(CMRparams, reqparams, subsetparams)

concept_id = get_concept_id(
product=request_params["short_name"], version=request_params["version"]
)

# TODO: At this point, the request parameters have been formatted into
# strings. `harmony-py` expects python objects (e.g., `dt.datetime` for
# temporal values)

# Place the order.
# TODO: there are probably other options we want to more generically
# expose here. E.g., instead of just accepting a `bounding_box` of a
# particular flavor, we want to be able to pass in a polygon?
HARMONY_API.place_order(
concept_id=concept_id,
bounding_box=request_params["bounding_box"],
temporal=request_params["temporal"],
)

########################################################################
# Most of what exists after this point will go away. `harmony-py` deals
# with submitting orders and tracking status via a single job ID.
########################################################################

order_fn = ".order_restart"

total_pages = int(np.ceil(len(self.avail) / reqparams["page_size"]))
Expand Down
26 changes: 26 additions & 0 deletions icepyx/core/harmony.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,29 @@ def get_capabilities(self, concept_id: str) -> dict[str, Any]:
response = self.harmony_client.submit(capabilities_request)

return response

def place_order(self, concept_id: str, bounding_box, temporal) -> str:
"""Places a Harmony order with the given parameters.
Return a string representing a job ID.
"""
collection = harmony.Collection(id=concept_id)
request = harmony.Request(
collection=collection,
# TODO: add spatial bounding box is a `harmony.BBox`.
spatial=bounding_box,
# TODO: temporal should be a dict {"start": dt.datetime, "end": dt.datetime}
temporal=temporal,
)

if not request.is_valid():
# TODO: consider more specific error class & message
raise RuntimeError("Failed to create valid request")

job_id = self.harmony_client.submit(request)

return job_id

def check_order_status(self, job_id: str):
status = self.harmony_client.status(job_id)
return status
30 changes: 13 additions & 17 deletions icepyx/core/query.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import datetime as dt
from functools import cached_property
import pprint
from typing import Optional, Union, cast
from typing import Optional, Union

import geopandas as gpd
import holoviews as hv
Expand All @@ -17,11 +17,8 @@
import icepyx.core.is2ref as is2ref
import icepyx.core.spatial as spat
import icepyx.core.temporal as tp
from icepyx.core.types import (
from icepyx.core.types.api import (
CMRParams,
EGIParamsSubset,
EGIRequiredParams,
EGIRequiredParamsDownload,
)
import icepyx.core.validate_inputs as val
from icepyx.core.variables import Variables as Variables
Expand Down Expand Up @@ -597,7 +594,7 @@ def CMRparams(self) -> CMRParams:
return self._CMRparams.fmted_keys

@property
def reqparams(self) -> EGIRequiredParams:
def reqparams(self): # -> EGIRequiredParams:
"""
Display the required key:value pairs that will be submitted.
It generates the dictionary if it does not already exist.
Expand All @@ -613,8 +610,6 @@ def reqparams(self) -> EGIRequiredParams:
>>> reg_a.reqparams # doctest: +SKIP
{'short_name': 'ATL06', 'version': '006', 'page_size': 2000, 'page_num': 1, 'request_mode': 'async', 'include_meta': 'Y', 'client_string': 'icepyx'}
"""
raise RefactoringException

if not hasattr(self, "_reqparams"):
self._reqparams = apifmt.Parameters("required", reqtype="search")
self._reqparams.build_params(product=self.product, version=self._version)
Expand All @@ -624,7 +619,7 @@ def reqparams(self) -> EGIRequiredParams:
# @property
# DevQuestion: if I make this a property, I get a "dict" object is not callable
# when I try to give input kwargs... what approach should I be taking?
def subsetparams(self, **kwargs) -> Union[EGIParamsSubset, dict[Never, Never]]:
def subsetparams(self, **kwargs): # -> Union[EGIParamsSubset, dict[Never, Never]]:
"""
Display the subsetting key:value pairs that will be submitted.
It generates the dictionary if it does not already exist
Expand All @@ -650,7 +645,7 @@ def subsetparams(self, **kwargs) -> Union[EGIParamsSubset, dict[Never, Never]]:
{'time': '2019-02-20T00:00:00,2019-02-28T23:59:59',
'bbox': '-55.0,68.0,-48.0,71.0'}
"""
raise RefactoringException
# raise RefactoringException

if not hasattr(self, "_subsetparams"):
self._subsetparams = apifmt.Parameters("subset")
Expand Down Expand Up @@ -1024,12 +1019,13 @@ def order_granules(
.
Retry request status is: complete
"""
breakpoint()
raise RefactoringException

if not hasattr(self, "reqparams"):
self.reqparams
# breakpoint()
# raise RefactoringException

# TODO: this probably shouldn't be mutated based on which method is being called...
# It is also very confusing to have both `self.reqparams` and
# `self._reqparams`, each of which does something different!
self.reqparams
if self._reqparams._reqtype == "search":
self._reqparams._reqtype = "download"

Expand Down Expand Up @@ -1065,7 +1061,7 @@ def order_granules(
tempCMRparams["readable_granule_name[]"] = gran
self.granules.place_order(
tempCMRparams,
cast(EGIRequiredParamsDownload, self.reqparams),
self.reqparams,
self.subsetparams(**kwargs),
verbose,
subset,
Expand All @@ -1075,7 +1071,7 @@ def order_granules(
else:
self.granules.place_order(
self.CMRparams,
cast(EGIRequiredParamsDownload, self.reqparams),
self.reqparams,
self.subsetparams(**kwargs),
verbose,
subset,
Expand Down
9 changes: 6 additions & 3 deletions icepyx/core/types/api.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
from typing import Literal, TypedDict, Union
from typing import TypedDict, Union

from typing_extensions import NotRequired
from pydantic import BaseModel
from typing_extensions import NotRequired

CMRParamsBase = TypedDict(
"CMRParamsBase",
{
"short_name": str,
"version": str,
"page_size": int,
"temporal": NotRequired[str],
"options[readable_granule_name][pattern]": NotRequired[str],
"options[spatial][or]": NotRequired[str],
Expand All @@ -25,4 +28,4 @@ class CMRParamsWithPolygon(CMRParamsBase):
CMRParams = Union[CMRParamsWithBbox, CMRParamsWithPolygon]


class HarmonyCoverageAPIParamsBase(BaseModel):
class HarmonyCoverageAPIParamsBase(BaseModel): ...

0 comments on commit fb1eefd

Please sign in to comment.