From 4a9e16df9ae1789ac3e099d0be5e1750df4a60c9 Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Fri, 5 Jan 2024 14:36:23 -0500 Subject: [PATCH] Revert "update QUEST and GenQuery classes for argo integration (#441)" This reverts commit 938262f89ddae37e2a7859909f45da407ef0a777. --- .../contributing/quest-available-datasets.rst | 25 -- icepyx/core/query.py | 345 +++++++++--------- icepyx/quest/__init__.py | 0 icepyx/quest/dataset_scripts/dataset.py | 90 ++--- icepyx/quest/quest.py | 104 +----- icepyx/tests/test_query.py | 12 - icepyx/tests/test_quest.py | 80 ---- 7 files changed, 232 insertions(+), 424 deletions(-) delete mode 100644 doc/source/contributing/quest-available-datasets.rst create mode 100644 icepyx/quest/__init__.py delete mode 100644 icepyx/tests/test_quest.py diff --git a/doc/source/contributing/quest-available-datasets.rst b/doc/source/contributing/quest-available-datasets.rst deleted file mode 100644 index 91a6283a0..000000000 --- a/doc/source/contributing/quest-available-datasets.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. _quest_supported_label: - -QUEST Supported Datasets -======================== - -On this page, we outline the datasets that are supported by the QUEST module. Click on the links for each dataset to view information about the API and sensor/data platform used. - - -List of Datasets ----------------- - -* `Argo `_ - * The Argo mission involves a series of floats that are designed to capture vertical ocean profiles of temperature, salinity, and pressure down to ~2000 m. Some floats are in support of BGC-Argo, which also includes data relevant for biogeochemical applications: oxygen, nitrate, chlorophyll, backscatter, and solar irradiance. - * (Link Kelsey's paper here) - * (Link to example workbook here) - - -Adding a Dataset to QUEST -------------------------- - -Want to add a new dataset to QUEST? No problem! QUEST includes a template script (``dataset.py``) that may be used to create your own querying module for a dataset of interest. - -Guidelines on how to construct your dataset module may be found here: (link to be added) - -Once you have developed a script with the template, you may request for the module to be added to QUEST via Github. Please see the How to Contribute page :ref:`dev_guide_label` for instructions on how to contribute to icepyx. \ No newline at end of file diff --git a/icepyx/core/query.py b/icepyx/core/query.py index 3459fd132..e8f1d8e7c 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -12,9 +12,11 @@ import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin import icepyx.core.granules as granules -# QUESTION: why doesn't from granules import Granules work, since granules=icepyx.core.granules? -from icepyx.core.granules import Granules +from icepyx.core.granules import Granules as Granules import icepyx.core.is2ref as is2ref + +# QUESTION: why doesn't from granules import Granules as Granules work, since granules=icepyx.core.granules? +# from icepyx.core.granules import Granules import icepyx.core.spatial as spat import icepyx.core.temporal as tp import icepyx.core.validate_inputs as val @@ -146,177 +148,6 @@ def __str__(self): ) return str - # ---------------------------------------------------------------------- - # Properties - - @property - def temporal(self): - """ - Return the Temporal object containing date/time range information for the query object. - - See Also - -------- - temporal.Temporal.start - temporal.Temporal.end - temporal.Temporal - - Examples - -------- - >>> reg_a = GenQuery([-55, 68, -48, 71],['2019-02-20','2019-02-28']) - >>> print(reg_a.temporal) - Start date and time: 2019-02-20 00:00:00 - End date and time: 2019-02-28 23:59:59 - - >>> reg_a = GenQuery([-55, 68, -48, 71],cycles=['03','04','05','06','07'], tracks=['0849','0902']) - >>> print(reg_a.temporal) - ['No temporal parameters set'] - """ - - if hasattr(self, "_temporal"): - return self._temporal - else: - return ["No temporal parameters set"] - - @property - def spatial(self): - """ - Return the spatial object, which provides the underlying functionality for validating - and formatting geospatial objects. The spatial object has several properties to enable - user access to the stored spatial extent in multiple formats. - - See Also - -------- - spatial.Spatial.spatial_extent - spatial.Spatial.extent_type - spatial.Spatial.extent_file - spatial.Spatial - - Examples - -------- - >>> reg_a = ipx.GenQuery([-55, 68, -48, 71],['2019-02-20','2019-02-28']) - >>> reg_a.spatial # doctest: +SKIP - - - >>> print(reg_a.spatial) - Extent type: bounding_box - Coordinates: [-55.0, 68.0, -48.0, 71.0] - - """ - return self._spatial - - @property - def spatial_extent(self): - """ - Return an array showing the spatial extent of the query object. - Spatial extent is returned as an input type (which depends on how - you initially entered your spatial data) followed by the geometry data. - Bounding box data is [lower-left-longitude, lower-left-latitute, upper-right-longitude, upper-right-latitude]. - Polygon data is [longitude1, latitude1, longitude2, latitude2, - ... longitude_n,latitude_n, longitude1,latitude1]. - - Returns - ------- - tuple of length 2 - First tuple element is the spatial type ("bounding box" or "polygon"). - Second tuple element is the spatial extent as a list of coordinates. - - Examples - -------- - - # Note: coordinates returned as float, not int - >>> reg_a = GenQuery([-55, 68, -48, 71],['2019-02-20','2019-02-28']) - >>> reg_a.spatial_extent - ('bounding_box', [-55.0, 68.0, -48.0, 71.0]) - - >>> reg_a = GenQuery([(-55, 68), (-55, 71), (-48, 71), (-48, 68), (-55, 68)],['2019-02-20','2019-02-28']) - >>> reg_a.spatial_extent - ('polygon', [-55.0, 68.0, -55.0, 71.0, -48.0, 71.0, -48.0, 68.0, -55.0, 68.0]) - - # NOTE Is this where we wanted to put the file-based test/example? - # The test file path is: examples/supporting_files/simple_test_poly.gpkg - - See Also - -------- - Spatial.extent - Spatial.extent_type - Spatial.extent_as_gdf - - """ - - return (self._spatial._ext_type, self._spatial._spatial_ext) - - @property - def dates(self): - """ - Return an array showing the date range of the query object. - Dates are returned as an array containing the start and end datetime objects, inclusive, in that order. - - Examples - -------- - >>> reg_a = ipx.GenQuery([-55, 68, -48, 71],['2019-02-20','2019-02-28']) - >>> reg_a.dates - ['2019-02-20', '2019-02-28'] - - >>> reg_a = GenQuery([-55, 68, -48, 71]) - >>> reg_a.dates - ['No temporal parameters set'] - """ - if not hasattr(self, "_temporal"): - return ["No temporal parameters set"] - else: - return [ - self._temporal._start.strftime("%Y-%m-%d"), - self._temporal._end.strftime("%Y-%m-%d"), - ] # could also use self._start.date() - - @property - def start_time(self): - """ - Return the start time specified for the start date. - - Examples - -------- - >>> reg_a = ipx.GenQuery([-55, 68, -48, 71],['2019-02-20','2019-02-28']) - >>> reg_a.start_time - '00:00:00' - - >>> reg_a = ipx.GenQuery([-55, 68, -48, 71],['2019-02-20','2019-02-28'], start_time='12:30:30') - >>> reg_a.start_time - '12:30:30' - - >>> reg_a = GenQuery([-55, 68, -48, 71]) - >>> reg_a.start_time - ['No temporal parameters set'] - """ - if not hasattr(self, "_temporal"): - return ["No temporal parameters set"] - else: - return self._temporal._start.strftime("%H:%M:%S") - - @property - def end_time(self): - """ - Return the end time specified for the end date. - - Examples - -------- - >>> reg_a = ipx.GenQuery([-55, 68, -48, 71],['2019-02-20','2019-02-28']) - >>> reg_a.end_time - '23:59:59' - - >>> reg_a = ipx.GenQuery([-55, 68, -48, 71],['2019-02-20','2019-02-28'], end_time='10:20:20') - >>> reg_a.end_time - '10:20:20' - - >>> reg_a = GenQuery([-55, 68, -48, 71]) - >>> reg_a.end_time - ['No temporal parameters set'] - """ - if not hasattr(self, "_temporal"): - return ["No temporal parameters set"] - else: - return self._temporal._end.strftime("%H:%M:%S") - # DevGoal: update docs throughout to allow for polygon spatial extent # Note: add files to docstring once implemented @@ -502,6 +333,174 @@ def product_version(self): """ return self._version + @property + def temporal(self): + """ + Return the Temporal object containing date/time range information for the query object. + + See Also + -------- + temporal.Temporal.start + temporal.Temporal.end + temporal.Temporal + + Examples + -------- + >>> reg_a = Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28']) + >>> print(reg_a.temporal) + Start date and time: 2019-02-20 00:00:00 + End date and time: 2019-02-28 23:59:59 + + >>> reg_a = Query('ATL06',[-55, 68, -48, 71],cycles=['03','04','05','06','07'], tracks=['0849','0902']) + >>> print(reg_a.temporal) + ['No temporal parameters set'] + """ + + if hasattr(self, "_temporal"): + return self._temporal + else: + return ["No temporal parameters set"] + + @property + def spatial(self): + """ + Return the spatial object, which provides the underlying functionality for validating + and formatting geospatial objects. The spatial object has several properties to enable + user access to the stored spatial extent in multiple formats. + + See Also + -------- + spatial.Spatial.spatial_extent + spatial.Spatial.extent_type + spatial.Spatial.extent_file + spatial.Spatial + + Examples + -------- + >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28']) + >>> reg_a.spatial # doctest: +SKIP + + + >>> print(reg_a.spatial) + Extent type: bounding_box + Coordinates: [-55.0, 68.0, -48.0, 71.0] + + """ + return self._spatial + + @property + def spatial_extent(self): + """ + Return an array showing the spatial extent of the query object. + Spatial extent is returned as an input type (which depends on how + you initially entered your spatial data) followed by the geometry data. + Bounding box data is [lower-left-longitude, lower-left-latitute, upper-right-longitude, upper-right-latitude]. + Polygon data is [longitude1, latitude1, longitude2, latitude2, + ... longitude_n,latitude_n, longitude1,latitude1]. + + Returns + ------- + tuple of length 2 + First tuple element is the spatial type ("bounding box" or "polygon"). + Second tuple element is the spatial extent as a list of coordinates. + + Examples + -------- + + # Note: coordinates returned as float, not int + >>> reg_a = Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28']) + >>> reg_a.spatial_extent + ('bounding_box', [-55.0, 68.0, -48.0, 71.0]) + + >>> reg_a = Query('ATL06',[(-55, 68), (-55, 71), (-48, 71), (-48, 68), (-55, 68)],['2019-02-20','2019-02-28']) + >>> reg_a.spatial_extent + ('polygon', [-55.0, 68.0, -55.0, 71.0, -48.0, 71.0, -48.0, 68.0, -55.0, 68.0]) + + # NOTE Is this where we wanted to put the file-based test/example? + # The test file path is: examples/supporting_files/simple_test_poly.gpkg + + See Also + -------- + Spatial.extent + Spatial.extent_type + Spatial.extent_as_gdf + + """ + + return (self._spatial._ext_type, self._spatial._spatial_ext) + + @property + def dates(self): + """ + Return an array showing the date range of the query object. + Dates are returned as an array containing the start and end datetime objects, inclusive, in that order. + + Examples + -------- + >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28']) + >>> reg_a.dates + ['2019-02-20', '2019-02-28'] + + >>> reg_a = Query('ATL06',[-55, 68, -48, 71],cycles=['03','04','05','06','07'], tracks=['0849','0902']) + >>> reg_a.dates + ['No temporal parameters set'] + """ + if not hasattr(self, "_temporal"): + return ["No temporal parameters set"] + else: + return [ + self._temporal._start.strftime("%Y-%m-%d"), + self._temporal._end.strftime("%Y-%m-%d"), + ] # could also use self._start.date() + + @property + def start_time(self): + """ + Return the start time specified for the start date. + + Examples + -------- + >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28']) + >>> reg_a.start_time + '00:00:00' + + >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28'], start_time='12:30:30') + >>> reg_a.start_time + '12:30:30' + + >>> reg_a = Query('ATL06',[-55, 68, -48, 71],cycles=['03','04','05','06','07'], tracks=['0849','0902']) + >>> reg_a.start_time + ['No temporal parameters set'] + """ + if not hasattr(self, "_temporal"): + return ["No temporal parameters set"] + else: + return self._temporal._start.strftime("%H:%M:%S") + + @property + def end_time(self): + """ + Return the end time specified for the end date. + + Examples + -------- + >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28']) + >>> reg_a.end_time + '23:59:59' + + >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28'], end_time='10:20:20') + >>> reg_a.end_time + '10:20:20' + + >>> reg_a = Query('ATL06',[-55, 68, -48, 71],cycles=['03','04','05','06','07'], tracks=['0849','0902']) + >>> reg_a.end_time + ['No temporal parameters set'] + """ + if not hasattr(self, "_temporal"): + return ["No temporal parameters set"] + else: + return self._temporal._end.strftime("%H:%M:%S") + @property def cycles(self): """ diff --git a/icepyx/quest/__init__.py b/icepyx/quest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/icepyx/quest/dataset_scripts/dataset.py b/icepyx/quest/dataset_scripts/dataset.py index e76081e08..13e926229 100644 --- a/icepyx/quest/dataset_scripts/dataset.py +++ b/icepyx/quest/dataset_scripts/dataset.py @@ -1,5 +1,4 @@ import warnings -from icepyx.core.query import GenQuery warnings.filterwarnings("ignore") @@ -7,75 +6,78 @@ class DataSet: """ - Template parent class for all QUEST supported datasets (i.e. ICESat-2, Argo BGC, Argo, MODIS, etc.). - All sub-classes must support the following methods for use via the QUEST class. + Parent Class for all supported datasets (i.e. ATL03, ATL07, MODIS, etc.) + all sub classes must support the following methods for use in + colocated data class """ - def __init__( - self, spatial_extent=None, date_range=None, start_time=None, end_time=None - ): + def __init__(self, boundingbox, timeframe): """ - Complete any dataset specific initializations (i.e. beyond space and time) required here. - For instance, ICESat-2 requires a product, and Argo requires parameters. - One can also check that the "default" space and time supplied by QUEST are the right format - (e.g. if the spatial extent must be a bounding box). + * use existing Icepyx functionality to initialise this + :param timeframe: datetime """ - raise NotImplementedError - - # ---------------------------------------------------------------------- - # Formatting API Inputs + self.bounding_box = boundingbox + self.time_frame = timeframe def _fmt_coordinates(self): - """ - Convert spatial extent into format needed by DataSet API, - if different than the formats available directly from SuperQuery. - """ + # use icepyx geospatial module (icepyx core) raise NotImplementedError def _fmt_timerange(self): """ - Convert temporal information into format needed by DataSet API, - if different than the formats available directly from SuperQuery. + will return list of datetime objects [start_time, end_time] """ raise NotImplementedError - # ---------------------------------------------------------------------- - # Validation - - def _validate_inputs(self): + # todo: merge with Icepyx SuperQuery + def _validate_input(self): """ - Create any additional validation functions for verifying inputs. - This function is not explicitly called by QUEST, - but is frequently needed for preparing API requests. - - See Also - -------- - quest.dataset_scripts.argo.Argo._validate_parameters + This may already be done in icepyx. + Not sure if we need this here """ raise NotImplementedError - # ---------------------------------------------------------------------- - # Querying and Getting Data - - def search_data(self): + def search_data(self, delta_t): """ - Query the dataset (i.e. search for available data) - given the spatiotemporal criteria and other parameters specific to the dataset. + query dataset given the spatio temporal criteria + and other params specic to the dataset """ raise NotImplementedError - def download(self): + def download(self, out_path): """ - Download the data to your local machine. + once data is querried, the user may choose to dowload the + data locally """ raise NotImplementedError - # ---------------------------------------------------------------------- - # Working with Data - def visualize(self): """ - Tells QUEST how to plot data (for instance, which parameters to plot) on a basemap. - For ICESat-2, it might show a photon track, and for Argo it might show a profile location. + (once data is downloaded)?, makes a quick plot showing where + data are located + e.g. Plots location of Argo profile or highlights ATL03 photon track """ raise NotImplementedError + + def _add2colocated_plot(self): + """ + Takes visualise() functionality and adds the plot to central + plot with other coincident data. This will be called by + show_area_overlap() in Colocateddata class + """ + raise NotImplementedError + + """ + The following are low priority functions + Not sure these are even worth keeping. Doesn't make sense for + all datasets. + """ + + # def get_meltpond_fraction(self): + # raise NotImplementedError + # + # def get_sea_ice_fraction(self): + # raise NotImplementedError + # + # def get_roughness(self): + # raise NotImplementedError diff --git a/icepyx/quest/quest.py b/icepyx/quest/quest.py index c54e49b73..2855a879c 100644 --- a/icepyx/quest/quest.py +++ b/icepyx/quest/quest.py @@ -1,26 +1,25 @@ import matplotlib.pyplot as plt -from icepyx.core.query import GenQuery, Query - -# from icepyx.quest.dataset_scripts.argo import Argo +from icepyx.core.query import GenQuery # todo: implement the subclass inheritance class Quest(GenQuery): """ QUEST - Query Unify Explore SpatioTemporal - object to query, obtain, and perform basic - operations on datasets (i.e. Argo, BGC Argo, MODIS, etc) for combined analysis with ICESat-2 - data products. A new dataset can be added using the `dataset.py` template. - QUEST expands the icepyx GenQuery superclass. + operations on datasets for combined analysis with ICESat-2 data products. + A new dataset can be added using the `dataset.py` template. + A list of already supported datasets is available at: + Expands the icepyx GenQuery superclass. See the doc page for GenQuery for details on temporal and spatial input parameters. Parameters ---------- - proj : proj4 string - Geospatial projection. - Not yet implemented + projection : proj4 string + Not yet implemented + Ex text: a string name of projection to be used for plotting (e.g. 'Mercator', 'NorthPolarStereographic') Returns ------- @@ -39,6 +38,7 @@ class Quest(GenQuery): Date range: (2019-02-20 00:00:00, 2019-02-28 23:59:59) Data sets: None + # todo: make this work with real datasets Add datasets to the quest object. >>> reg_a.datasets = {'ATL07':None, 'Argo':None} @@ -61,11 +61,13 @@ def __init__( end_time=None, proj="Default", ): - """ - Tells QUEST to initialize data given the user input spatiotemporal data. - """ super().__init__(spatial_extent, date_range, start_time, end_time) self.datasets = {} + self.projection = self._determine_proj(proj) + + # todo: maybe move this to icepyx superquery class + def _determine_proj(self, proj): + return None def __str__(self): str = super(Quest, self).__str__() @@ -81,82 +83,4 @@ def __str__(self): return str - # ---------------------------------------------------------------------- - # Datasets - - def add_icesat2( - self, - product=None, - start_time=None, - end_time=None, - version=None, - cycles=None, - tracks=None, - files=None, - **kwargs, - ): - """ - Adds ICESat-2 datasets to QUEST structure. - """ - - query = Query( - product, - self._spatial.extent, - [self._temporal.start, self._temporal.end], - start_time, - end_time, - version, - cycles, - tracks, - files, - **kwargs, - ) - - self.datasets["icesat2"] = query - - # def add_argo(self, params=["temperature"], presRange=None): - - # argo = Argo(self._spatial, self._temporal, params, presRange) - # self.datasets["argo"] = argo - - # ---------------------------------------------------------------------- - # Methods (on all datasets) - - # error handling? what happens when one of i fails... - def search_all(self): - """ - Searches for requred dataset within platform (i.e. ICESat-2, Argo) of interest. - """ - print("\nSearching all datasets...") - - for i in self.datasets.values(): - print() - try: - # querying ICESat-2 data - if isinstance(i, Query): - print("---ICESat-2---") - msg = i.avail_granules() - print(msg) - else: # querying all other data sets - print(i) - i.search_data() - except: - dataset_name = type(i).__name__ - print("Error querying data from {0}".format(dataset_name)) - - # error handling? what happens when one of i fails... - def download_all(self, path=""): - ' ' 'Downloads requested dataset(s).' ' ' - print("\nDownloading all datasets...") - - for i in self.datasets.values(): - print() - if isinstance(i, Query): - print("---ICESat-2---") - msg = i.download_granules(path) - print(msg) - else: - i.download() - print(i) - # DEVNOTE: see colocated data branch and phyto team files for code that expands quest functionality diff --git a/icepyx/tests/test_query.py b/icepyx/tests/test_query.py index 7738c424a..55b25ef4a 100644 --- a/icepyx/tests/test_query.py +++ b/icepyx/tests/test_query.py @@ -41,18 +41,6 @@ def test_icepyx_boundingbox_query(): assert obs_tuple == exp_tuple -def test_temporal_properties_cycles_tracks(): - reg_a = ipx.Query( - "ATL06", - [-55, 68, -48, 71], - cycles=["03", "04", "05", "06", "07"], - tracks=["0849", "0902"], - ) - exp = ["No temporal parameters set"] - - assert [obs == exp for obs in (reg_a.dates, reg_a.start_time, reg_a.end_time)] - - # Tests need to add (given can't do them within docstrings/they're behind NSIDC login) # reqparams post-order # product_all_info diff --git a/icepyx/tests/test_quest.py b/icepyx/tests/test_quest.py deleted file mode 100644 index 043ee159e..000000000 --- a/icepyx/tests/test_quest.py +++ /dev/null @@ -1,80 +0,0 @@ -import pytest -import re - -import icepyx as ipx -from icepyx.quest.quest import Quest - - -@pytest.fixture -def quest_instance(scope="module", autouse=True): - bounding_box = [-150, 30, -120, 60] - date_range = ["2022-06-07", "2022-06-14"] - my_quest = Quest(spatial_extent=bounding_box, date_range=date_range) - return my_quest - - -########## PER-DATASET ADDITION TESTS ########## - -# Paramaterize these add_dataset tests once more datasets are added -def test_add_is2(quest_instance): - # Add ATL06 as a test to QUEST - - prod = "ATL06" - quest_instance.add_icesat2(product=prod) - exp_key = "icesat2" - exp_type = ipx.Query - - obs = quest_instance.datasets - - assert type(obs) == dict - assert exp_key in obs.keys() - assert type(obs[exp_key]) == exp_type - assert quest_instance.datasets[exp_key].product == prod - - -# def test_add_argo(quest_instance): -# params = ["down_irradiance412", "temperature"] -# quest_instance.add_argo(params=params) -# exp_key = "argo" -# exp_type = ipx.quest.dataset_scripts.argo.Argo - -# obs = quest_instance.datasets - -# assert type(obs) == dict -# assert exp_key in obs.keys() -# assert type(obs[exp_key]) == exp_type -# assert quest_instance.datasets[exp_key].params == params - -# def test_add_multiple_datasets(): -# bounding_box = [-150, 30, -120, 60] -# date_range = ["2022-06-07", "2022-06-14"] -# my_quest = Quest(spatial_extent=bounding_box, date_range=date_range) -# -# # print(my_quest.spatial) -# # print(my_quest.temporal) -# -# # my_quest.add_argo(params=["down_irradiance412", "temperature"]) -# # print(my_quest.datasets["argo"].params) -# -# my_quest.add_icesat2(product="ATL06") -# # print(my_quest.datasets["icesat2"].product) -# -# print(my_quest) -# -# # my_quest.search_all() -# # -# # # this one still needs work for IS2 because of auth... -# # my_quest.download_all() - -########## ALL DATASET METHODS TESTS ########## - -# is successful execution enough here? -# each of the query functions should be tested in their respective modules -def test_search_all(quest_instance): - # Search and test all datasets - quest_instance.search_all() - - -def test_download_all(): - # this will require auth in some cases... - pass