diff --git a/.coveragerc b/.coveragerc index 444b172a2..2f02d0030 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,5 +3,4 @@ branch = True source = icepyx omit = setup.py - test/* doc/* diff --git a/doc/source/user_guide/documentation/classes_dev_uml.svg b/doc/source/user_guide/documentation/classes_dev_uml.svg index 8079985cc..d2ca62b29 100644 --- a/doc/source/user_guide/documentation/classes_dev_uml.svg +++ b/doc/source/user_guide/documentation/classes_dev_uml.svg @@ -4,11 +4,11 @@ - + classes_dev_uml - + icepyx.quest.dataset_scripts.argo.Argo @@ -61,7 +61,7 @@ () - + icepyx.quest.dataset_scripts.argo.Argo->icepyx.quest.dataset_scripts.dataset.DataSet @@ -69,403 +69,408 @@ icepyx.core.auth.AuthenticationError - -AuthenticationError - - - + +AuthenticationError + + + icepyx.core.exceptions.DeprecationError - -DeprecationError - - - + +DeprecationError + + + icepyx.core.auth.EarthdataAuthMixin - -EarthdataAuthMixin - -_auth : NoneType -_s3_initial_ts : NoneType, datetime -_s3login_credentials : NoneType -_session : NoneType -auth -s3login_credentials -session - -__init__(auth) -__str__() -earthdata_login(uid, email, s3token): None + +EarthdataAuthMixin + +_auth : NoneType +_s3_initial_ts : NoneType, datetime +_s3login_credentials : NoneType +_session : NoneType +auth +s3login_credentials +session + +__init__(auth) +__str__() +earthdata_login(uid, email, s3token): None icepyx.core.query.GenQuery - -GenQuery - -_spatial -_temporal -dates -end_time -spatial -spatial_extent -start_time -temporal - -__init__(spatial_extent, date_range, start_time, end_time) -__str__() + +GenQuery + +_spatial +_temporal +dates +end_time +spatial +spatial_extent +start_time +temporal + +__init__(spatial_extent, date_range, start_time, end_time) +__str__() icepyx.core.granules.Granules - -Granules - -avail : list -orderIDs : list - -__init__ -() -download(verbose, path, session, restart) -get_avail(CMRparams, reqparams, cloud) -place_order(CMRparams, reqparams, subsetparams, verbose, subset, session, geom_filepath) + +Granules + +avail : list +orderIDs : list + +__init__() +download(verbose, path, restart) +get_avail(CMRparams, reqparams, cloud) +place_order(CMRparams, reqparams, subsetparams, verbose, subset, geom_filepath) + + + +icepyx.core.granules.Granules->icepyx.core.auth.EarthdataAuthMixin + + icepyx.core.query.Query - -Query - -CMRparams -_CMRparams -_about_product -_cust_options : dict -_cycles : list -_granules -_order_vars -_prod : NoneType, str -_readable_granule_name : list -_reqparams -_subsetparams : NoneType -_tracks : list -_version -cycles -dataset -granules -order_vars -product -product_version -reqparams -tracks - -__init__(product, spatial_extent, date_range, start_time, end_time, version, cycles, tracks, auth) -__str__() -avail_granules(ids, cycles, tracks, cloud) -download_granules(path, verbose, subset, restart) -latest_version() -order_granules(verbose, subset, email) -product_all_info() -product_summary_info() -show_custom_options(dictview) -subsetparams() -visualize_elevation() -visualize_spatial_extent() + +Query + +CMRparams +_CMRparams +_about_product +_cust_options : dict +_cycles : list +_granules +_order_vars +_prod : NoneType, str +_readable_granule_name : list +_reqparams +_subsetparams : NoneType +_tracks : list +_version +cycles +dataset +granules +order_vars +product +product_version +reqparams +tracks + +__init__(product, spatial_extent, date_range, start_time, end_time, version, cycles, tracks, auth) +__str__() +avail_granules(ids, cycles, tracks, cloud) +download_granules(path, verbose, subset, restart) +latest_version() +order_granules(verbose, subset, email) +product_all_info() +product_summary_info() +show_custom_options(dictview) +subsetparams() +visualize_elevation() +visualize_spatial_extent() - + icepyx.core.granules.Granules->icepyx.core.query.Query - - -_granules + + +_granules - + icepyx.core.granules.Granules->icepyx.core.query.Query - - -_granules + + +_granules icepyx.core.icesat2data.Icesat2Data - -Icesat2Data - - -__init__() + +Icesat2Data + + +__init__() icepyx.core.exceptions.NsidcQueryError - -NsidcQueryError - -errmsg -msgtxt : str - -__init__(errmsg, msgtxt) -__str__() + +NsidcQueryError + +errmsg +msgtxt : str + +__init__(errmsg, msgtxt) +__str__() icepyx.core.exceptions.QueryError - -QueryError - - - + +QueryError + + + icepyx.core.exceptions.NsidcQueryError->icepyx.core.exceptions.QueryError - - + + icepyx.core.APIformatting.Parameters - -Parameters - -_fmted_keys : NoneType, dict -_poss_keys : dict -_reqtype : NoneType, str -fmted_keys -partype -poss_keys - -__init__(partype, values, reqtype) -_check_valid_keys() -_get_possible_keys() -build_params() -check_req_values() -check_values() - - - -icepyx.core.APIformatting.Parameters->icepyx.core.query.Query - - -_CMRparams + +Parameters + +_fmted_keys : NoneType, dict +_poss_keys : dict +_reqtype : NoneType, str +fmted_keys +partype +poss_keys + +__init__(partype, values, reqtype) +_check_valid_keys() +_get_possible_keys() +build_params() +check_req_values() +check_values() icepyx.core.APIformatting.Parameters->icepyx.core.query.Query - - -_reqparams + + +_CMRparams icepyx.core.APIformatting.Parameters->icepyx.core.query.Query - - -_subsetparams + + +_reqparams icepyx.core.APIformatting.Parameters->icepyx.core.query.Query - - -_subsetparams + + +_subsetparams + + + +icepyx.core.APIformatting.Parameters->icepyx.core.query.Query + + +_subsetparams - + icepyx.core.query.Query->icepyx.core.auth.EarthdataAuthMixin - - + + - + icepyx.core.query.Query->icepyx.core.query.GenQuery - - + + icepyx.quest.quest.Quest - -Quest - -datasets : dict - -__init__(spatial_extent, date_range, start_time, end_time, proj) -__str__() -add_argo(params, presRange): None -add_icesat2(product, start_time, end_time, version, cycles, tracks, files): None -download_all(path) -save_all(path) -search_all() + +Quest + +datasets : dict + +__init__(spatial_extent, date_range, start_time, end_time, proj) +__str__() +add_argo(params, presRange): None +add_icesat2(product, start_time, end_time, version, cycles, tracks, files): None +download_all(path) +save_all(path) +search_all() - + icepyx.quest.quest.Quest->icepyx.core.query.GenQuery - - + + icepyx.core.read.Read - -Read - -_filelist -_out_obj : Dataset -_product -_read_vars -filelist -is_s3 -product -vars - -__init__(data_source, glob_kwargs, out_obj_type, product, filename_pattern, catalog) -_add_vars_to_ds(is2ds, ds, grp_path, wanted_groups_tiered, wanted_dict) -_build_dataset_template(file) -_build_single_file_dataset(file, groups_list) -_combine_nested_vars(is2ds, ds, grp_path, wanted_dict) -_read_single_grp(file, grp_path) -load() + +Read + +_filelist +_out_obj : Dataset +_product +_read_vars +filelist +is_s3 +product +vars + +__init__(data_source, glob_kwargs, out_obj_type, product, filename_pattern, catalog) +_add_vars_to_ds(is2ds, ds, grp_path, wanted_groups_tiered, wanted_dict) +_build_dataset_template(file) +_build_single_file_dataset(file, groups_list) +_combine_nested_vars(is2ds, ds, grp_path, wanted_dict) +_read_single_grp(file, grp_path) +load() - + icepyx.core.read.Read->icepyx.core.auth.EarthdataAuthMixin - - + + icepyx.core.spatial.Spatial - -Spatial - -_ext_type : str -_gdf_spat : GeoDataFrame -_geom_file : NoneType -_spatial_ext -_xdateln -extent -extent_as_gdf -extent_file -extent_type - -__init__(spatial_extent) -__str__() -fmt_for_CMR() -fmt_for_EGI() + +Spatial + +_ext_type : str +_gdf_spat : GeoDataFrame +_geom_file : NoneType +_spatial_ext +_xdateln +extent +extent_as_gdf +extent_file +extent_type + +__init__(spatial_extent) +__str__() +fmt_for_CMR() +fmt_for_EGI() - + icepyx.core.spatial.Spatial->icepyx.core.query.GenQuery - - -_spatial + + +_spatial - + icepyx.core.spatial.Spatial->icepyx.core.query.GenQuery - - -_spatial + + +_spatial icepyx.core.temporal.Temporal - -Temporal - -_end : datetime -_start : datetime -end -start - -__init__(date_range, start_time, end_time) -__str__() + +Temporal + +_end : datetime +_start : datetime +end +start + +__init__(date_range, start_time, end_time) +__str__() - + icepyx.core.temporal.Temporal->icepyx.core.query.GenQuery - - -_temporal + + +_temporal icepyx.core.variables.Variables - -Variables - -_avail : NoneType, list -_path : NoneType -_product : NoneType, str -_version -path -product -version -wanted : NoneType, dict - -__init__(vartype, path, product, version, avail, wanted, auth) -_check_valid_lists(vgrp, allpaths, var_list, beam_list, keyword_list) -_get_combined_list(beam_list, keyword_list) -_get_sum_varlist(var_list, all_vars, defaults) -_iter_paths(sum_varlist, req_vars, vgrp, beam_list, keyword_list) -_iter_vars(sum_varlist, req_vars, vgrp) -append(defaults, var_list, beam_list, keyword_list) -avail(options, internal) -parse_var_list(varlist, tiered, tiered_vars) -remove(all, var_list, beam_list, keyword_list) + +Variables + +_avail : NoneType, list +_path : NoneType +_product : NoneType, str +_version +path +product +version +wanted : NoneType, dict + +__init__(vartype, path, product, version, avail, wanted, auth) +_check_valid_lists(vgrp, allpaths, var_list, beam_list, keyword_list) +_get_combined_list(beam_list, keyword_list) +_get_sum_varlist(var_list, all_vars, defaults) +_iter_paths(sum_varlist, req_vars, vgrp, beam_list, keyword_list) +_iter_vars(sum_varlist, req_vars, vgrp) +append(defaults, var_list, beam_list, keyword_list) +avail(options, internal) +parse_var_list(varlist, tiered, tiered_vars) +remove(all, var_list, beam_list, keyword_list) - + icepyx.core.variables.Variables->icepyx.core.auth.EarthdataAuthMixin - - + + - + icepyx.core.variables.Variables->icepyx.core.query.Query - - -_order_vars + + +_order_vars - + icepyx.core.variables.Variables->icepyx.core.query.Query - - -_order_vars + + +_order_vars - + icepyx.core.variables.Variables->icepyx.core.read.Read - - -_read_vars + + +_read_vars icepyx.core.visualization.Visualize - -Visualize - -bbox : list -cycles : NoneType -date_range : NoneType -product : NoneType, str -tracks : NoneType - -__init__(query_obj, product, spatial_extent, date_range, cycles, tracks) -generate_OA_parameters(): list -grid_bbox(binsize): list -make_request(base_url, payload) -parallel_request_OA(): da.array -query_icesat2_filelist(): tuple -request_OA_data(paras): da.array -viz_elevation(): (hv.DynamicMap, hv.Layout) + +Visualize + +bbox : list +cycles : NoneType +date_range : NoneType +product : NoneType, str +tracks : NoneType + +__init__(query_obj, product, spatial_extent, date_range, cycles, tracks) +generate_OA_parameters(): list +grid_bbox(binsize): list +make_request(base_url, payload) +parallel_request_OA(): da.array +query_icesat2_filelist(): tuple +request_OA_data(paras): da.array +viz_elevation(): (hv.DynamicMap, hv.Layout) diff --git a/doc/source/user_guide/documentation/classes_user_uml.svg b/doc/source/user_guide/documentation/classes_user_uml.svg index 256cc1794..6ecef9681 100644 --- a/doc/source/user_guide/documentation/classes_user_uml.svg +++ b/doc/source/user_guide/documentation/classes_user_uml.svg @@ -4,11 +4,11 @@ - + classes_user_uml - + icepyx.core.auth.AuthenticationError @@ -30,300 +30,306 @@ icepyx.core.auth.EarthdataAuthMixin - -EarthdataAuthMixin - -auth -s3login_credentials -session - -earthdata_login(uid, email, s3token): None + +EarthdataAuthMixin + +auth +s3login_credentials +session + +earthdata_login(uid, email, s3token): None icepyx.core.query.GenQuery - -GenQuery - -dates -end_time -spatial -spatial_extent -start_time -temporal - - + +GenQuery + +dates +end_time +spatial +spatial_extent +start_time +temporal + + icepyx.core.granules.Granules - -Granules - -avail : list -orderIDs : list - -download(verbose, path, session, restart) -get_avail(CMRparams, reqparams, cloud) -place_order(CMRparams, reqparams, subsetparams, verbose, subset, session, geom_filepath) + +Granules + +avail : list +orderIDs : list + +download(verbose, path, restart) +get_avail(CMRparams, reqparams, cloud) +place_order(CMRparams, reqparams, subsetparams, verbose, subset, geom_filepath) + + + +icepyx.core.granules.Granules->icepyx.core.auth.EarthdataAuthMixin + + icepyx.core.query.Query - -Query - -CMRparams -cycles -dataset -granules -order_vars -product -product_version -reqparams -tracks - -avail_granules(ids, cycles, tracks, cloud) -download_granules(path, verbose, subset, restart) -latest_version() -order_granules(verbose, subset, email) -product_all_info() -product_summary_info() -show_custom_options(dictview) -subsetparams() -visualize_elevation() -visualize_spatial_extent() + +Query + +CMRparams +cycles +dataset +granules +order_vars +product +product_version +reqparams +tracks + +avail_granules(ids, cycles, tracks, cloud) +download_granules(path, verbose, subset, restart) +latest_version() +order_granules(verbose, subset, email) +product_all_info() +product_summary_info() +show_custom_options(dictview) +subsetparams() +visualize_elevation() +visualize_spatial_extent() - + icepyx.core.granules.Granules->icepyx.core.query.Query - - -_granules + + +_granules - + icepyx.core.granules.Granules->icepyx.core.query.Query - - -_granules + + +_granules icepyx.core.icesat2data.Icesat2Data - -Icesat2Data - - - + +Icesat2Data + + + icepyx.core.exceptions.NsidcQueryError - -NsidcQueryError - -errmsg -msgtxt : str - - + +NsidcQueryError + +errmsg +msgtxt : str + + icepyx.core.exceptions.QueryError - -QueryError - - - + +QueryError + + + icepyx.core.exceptions.NsidcQueryError->icepyx.core.exceptions.QueryError - - + + icepyx.core.APIformatting.Parameters - -Parameters - -fmted_keys -partype -poss_keys - -build_params() -check_req_values() -check_values() - - - -icepyx.core.APIformatting.Parameters->icepyx.core.query.Query - - -_CMRparams + +Parameters + +fmted_keys +partype +poss_keys + +build_params() +check_req_values() +check_values() icepyx.core.APIformatting.Parameters->icepyx.core.query.Query - - -_reqparams + + +_CMRparams icepyx.core.APIformatting.Parameters->icepyx.core.query.Query - - -_subsetparams + + +_reqparams icepyx.core.APIformatting.Parameters->icepyx.core.query.Query - - -_subsetparams + + +_subsetparams + + + +icepyx.core.APIformatting.Parameters->icepyx.core.query.Query + + +_subsetparams - + icepyx.core.query.Query->icepyx.core.auth.EarthdataAuthMixin - - + + - + icepyx.core.query.Query->icepyx.core.query.GenQuery - - + + icepyx.core.read.Read - -Read - -filelist -is_s3 -product -vars - -load() + +Read + +filelist +is_s3 +product +vars + +load() - + icepyx.core.read.Read->icepyx.core.auth.EarthdataAuthMixin - - + + icepyx.core.spatial.Spatial - -Spatial - -extent -extent_as_gdf -extent_file -extent_type - -fmt_for_CMR() -fmt_for_EGI() + +Spatial + +extent +extent_as_gdf +extent_file +extent_type + +fmt_for_CMR() +fmt_for_EGI() - + icepyx.core.spatial.Spatial->icepyx.core.query.GenQuery - - -_spatial + + +_spatial - + icepyx.core.spatial.Spatial->icepyx.core.query.GenQuery - - -_spatial + + +_spatial icepyx.core.temporal.Temporal - -Temporal - -end -start - - + +Temporal + +end +start + + - + icepyx.core.temporal.Temporal->icepyx.core.query.GenQuery - - -_temporal + + +_temporal icepyx.core.variables.Variables - -Variables - -path -product -version -wanted : NoneType, dict - -append(defaults, var_list, beam_list, keyword_list) -avail(options, internal) -parse_var_list(varlist, tiered, tiered_vars) -remove(all, var_list, beam_list, keyword_list) + +Variables + +path +product +version +wanted : NoneType, dict + +append(defaults, var_list, beam_list, keyword_list) +avail(options, internal) +parse_var_list(varlist, tiered, tiered_vars) +remove(all, var_list, beam_list, keyword_list) - + icepyx.core.variables.Variables->icepyx.core.auth.EarthdataAuthMixin - - + + - + icepyx.core.variables.Variables->icepyx.core.query.Query - - -_order_vars + + +_order_vars - + icepyx.core.variables.Variables->icepyx.core.query.Query - - -_order_vars + + +_order_vars - + icepyx.core.variables.Variables->icepyx.core.read.Read - - -_read_vars + + +_read_vars icepyx.core.visualization.Visualize - -Visualize - -bbox : list -cycles : NoneType -date_range : NoneType -product : NoneType, str -tracks : NoneType - -generate_OA_parameters(): list -grid_bbox(binsize): list -make_request(base_url, payload) -parallel_request_OA(): da.array -query_icesat2_filelist(): tuple -request_OA_data(paras): da.array -viz_elevation(): (hv.DynamicMap, hv.Layout) + +Visualize + +bbox : list +cycles : NoneType +date_range : NoneType +product : NoneType, str +tracks : NoneType + +generate_OA_parameters(): list +grid_bbox(binsize): list +make_request(base_url, payload) +parallel_request_OA(): da.array +query_icesat2_filelist(): tuple +request_OA_data(paras): da.array +viz_elevation(): (hv.DynamicMap, hv.Layout) diff --git a/doc/source/user_guide/documentation/packages_user_uml.svg b/doc/source/user_guide/documentation/packages_user_uml.svg index 5c45fc92b..029a13015 100644 --- a/doc/source/user_guide/documentation/packages_user_uml.svg +++ b/doc/source/user_guide/documentation/packages_user_uml.svg @@ -4,11 +4,11 @@ - + packages_user_uml - + icepyx.core @@ -24,146 +24,152 @@ icepyx.core.auth - -icepyx.core.auth + +icepyx.core.auth icepyx.core.exceptions - -icepyx.core.exceptions + +icepyx.core.exceptions icepyx.core.auth->icepyx.core.exceptions - - + + icepyx.core.granules - -icepyx.core.granules + +icepyx.core.granules + + + +icepyx.core.granules->icepyx.core.auth + + icepyx.core.icesat2data - -icepyx.core.icesat2data + +icepyx.core.icesat2data - + icepyx.core.icesat2data->icepyx.core.exceptions - - + + icepyx.core.is2ref - -icepyx.core.is2ref + +icepyx.core.is2ref icepyx.core.query - -icepyx.core.query + +icepyx.core.query - + icepyx.core.query->icepyx.core.auth - - + + - + icepyx.core.query->icepyx.core.exceptions - - + + - + icepyx.core.query->icepyx.core.granules - - + + icepyx.core.variables - -icepyx.core.variables + +icepyx.core.variables - + icepyx.core.query->icepyx.core.variables - - + + icepyx.core.visualization - -icepyx.core.visualization + +icepyx.core.visualization - + icepyx.core.query->icepyx.core.visualization - - + + icepyx.core.read - -icepyx.core.read + +icepyx.core.read - + icepyx.core.read->icepyx.core.auth - - + + - + icepyx.core.read->icepyx.core.exceptions - - + + - + icepyx.core.read->icepyx.core.variables - - + + icepyx.core.spatial - -icepyx.core.spatial + +icepyx.core.spatial icepyx.core.temporal - -icepyx.core.temporal + +icepyx.core.temporal icepyx.core.validate_inputs - -icepyx.core.validate_inputs + +icepyx.core.validate_inputs - + icepyx.core.variables->icepyx.core.auth - - + + - + icepyx.core.variables->icepyx.core.exceptions - - + + diff --git a/icepyx/core/APIformatting.py b/icepyx/core/APIformatting.py index b5d31bdfa..eb77c37e9 100644 --- a/icepyx/core/APIformatting.py +++ b/icepyx/core/APIformatting.py @@ -1,7 +1,6 @@ # Generate and format information for submitting to API (CMR and NSIDC) import datetime as dt -import pprint # ---------------------------------------------------------------------- @@ -134,13 +133,13 @@ def combine_params(*param_dicts): Examples -------- - >>> CMRparams = {'short_name': 'ATL06', 'version': '002', 'temporal': '2019-02-20T00:00:00Z,2019-02-28T23:59:59Z', 'bounding_box': '-55,68,-48,71'} - >>> reqparams = {'page_size': 2000, 'page_num': 1} + >>> CMRparams = {'temporal': '2019-02-20T00:00:00Z,2019-02-28T23:59:59Z', 'bounding_box': '-55,68,-48,71'} + >>> reqparams = {'short_name': 'ATL06', 'version': '002', 'page_size': 2000, 'page_num': 1} >>> ipx.core.APIformatting.combine_params(CMRparams, reqparams) - {'short_name': 'ATL06', - 'version': '002', - 'temporal': '2019-02-20T00:00:00Z,2019-02-28T23:59:59Z', + {'temporal': '2019-02-20T00:00:00Z,2019-02-28T23:59:59Z', 'bounding_box': '-55,68,-48,71', + 'short_name': 'ATL06', + 'version': '002', 'page_size': 2000, 'page_num': 1} """ @@ -164,17 +163,18 @@ def to_string(params): Examples -------- - >>> CMRparams = {'short_name': 'ATL06', 'version': '002', 'temporal': '2019-02-20T00:00:00Z,2019-02-28T23:59:59Z', 'bounding_box': '-55,68,-48,71'} - >>> reqparams = {'page_size': 2000, 'page_num': 1} + >>> CMRparams = {'temporal': '2019-02-20T00:00:00Z,2019-02-28T23:59:59Z', + ... 'bounding_box': '-55,68,-48,71'} + >>> reqparams = {'short_name': 'ATL06', 'version': '002', 'page_size': 2000, 'page_num': 1} >>> params = ipx.core.APIformatting.combine_params(CMRparams, reqparams) >>> ipx.core.APIformatting.to_string(params) - 'short_name=ATL06&version=002&temporal=2019-02-20T00:00:00Z,2019-02-28T23:59:59Z&bounding_box=-55,68,-48,71&page_size=2000&page_num=1' + 'temporal=2019-02-20T00:00:00Z,2019-02-28T23:59:59Z&bounding_box=-55,68,-48,71&short_name=ATL06&version=002&page_size=2000&page_num=1' """ param_list = [] for k, v in params.items(): if isinstance(v, list): - for l in v: - param_list.append(k + "=" + l) + for i in v: + param_list.append(k + "=" + i) else: param_list.append(k + "=" + str(v)) # return the parameter string @@ -255,7 +255,6 @@ def _get_possible_keys(self): if self.partype == "CMR": self._poss_keys = { - "default": ["short_name", "version"], "spatial": ["bounding_box", "polygon"], "optional": [ "temporal", @@ -266,8 +265,10 @@ def _get_possible_keys(self): } elif self.partype == "required": self._poss_keys = { - "search": ["page_size"], + "search": ["short_name", "version", "page_size"], "download": [ + "short_name", + "version", "page_size", "page_num", "request_mode", @@ -279,7 +280,6 @@ def _get_possible_keys(self): } elif self.partype == "subset": self._poss_keys = { - "default": [], "spatial": ["bbox", "Boundingshape"], "optional": [ "time", @@ -305,6 +305,7 @@ def _check_valid_keys(self): "An invalid key (" + key + ") was passed. Please remove it using `del`" ) + # DevNote: can check_req_values and check_values be combined? def check_req_values(self): """ Check that all of the required keys have values, if the key was passed in with @@ -333,22 +334,14 @@ def check_values(self): self.partype != "required" ), "You cannot call this function for your parameter type" - default_keys = self.poss_keys["default"] spatial_keys = self.poss_keys["spatial"] - if all(keys in self._fmted_keys.keys() for keys in default_keys): - assert all( - self.fmted_keys.get(key, -9999) != -9999 for key in default_keys + # not the most robust check, but better than nothing... + if any(keys in self._fmted_keys.keys() for keys in spatial_keys): + assert any( + self.fmted_keys.get(key, -9999) != -9999 for key in spatial_keys ), "One of your formated parameters is missing a value" - - # not the most robust check, but better than nothing... - if any(keys in self._fmted_keys.keys() for keys in spatial_keys): - assert any( - self.fmted_keys.get(key, -9999) != -9999 for key in default_keys - ), "One of your formated parameters is missing a value" - return True - else: - return False + return True else: return False @@ -360,14 +353,19 @@ def build_params(self, **kwargs): Parameters ---------- **kwargs - Keyword inputs containing the needed information to build the parameter list, depending on - parameter type, if the already formatted key:value is not submitted as a kwarg. - May include optional keyword arguments to be passed to the subsetter. Valid keywords - are time, bbox OR Boundingshape, format, projection, projection_parameters, and Coverage. - - Keyword argument inputs for 'CMR' may include: dataset (data product), version, start, end, extent_type, spatial_extent - Keyword argument inputs for 'required' may include: page_size, page_num, request_mode, include_meta, client_string - Keyword argument inputs for 'subset' may include: geom_filepath, start, end, extent_type, spatial_extent + Keyword inputs containing the needed information to build the parameter list, depending + on parameter type, if the already formatted key:value is not submitted as a kwarg. + May include optional keyword arguments to be passed to the subsetter. + Valid keywords are time, bbox OR Boundingshape, format, projection, + projection_parameters, and Coverage. + + Keyword argument inputs for 'CMR' may include: + start, end, extent_type, spatial_extent + Keyword argument inputs for 'required' may include: + product or short_name, version, page_size, page_num, + request_mode, include_meta, client_string + Keyword argument inputs for 'subset' may include: + geom_filepath, start, end, extent_type, spatial_extent """ @@ -388,8 +386,16 @@ def build_params(self, **kwargs): "include_meta": "Y", "client_string": "icepyx", } + for key in reqkeys: - if key in kwargs: + if key == "short_name": + try: + self._fmted_keys.update({key: kwargs[key]}) + except KeyError: + self._fmted_keys.update({key: kwargs["product"]}) + elif key == "version": + self._fmted_keys.update({key: kwargs["version"]}) + elif key in kwargs: self._fmted_keys.update({key: kwargs[key]}) elif key in defaults: self._fmted_keys.update({key: defaults[key]}) @@ -397,22 +403,12 @@ def build_params(self, **kwargs): pass else: - if self.check_values == True and kwargs == None: + if self.check_values is True and kwargs is None: pass else: - default_keys = self.poss_keys["default"] spatial_keys = self.poss_keys["spatial"] opt_keys = self.poss_keys["optional"] - for key in default_keys: - if key in self._fmted_keys.values(): - assert self._fmted_keys[key] - else: - if key == "short_name": - self._fmted_keys.update({key: kwargs["product"]}) - elif key == "version": - self._fmted_keys.update({key: kwargs["version"]}) - for key in opt_keys: if key == "Coverage" and key in kwargs.keys(): # DevGoal: make there be an option along the lines of Coverage=default, which will get the default variables for that product without the user having to input is2obj.build_wanted_wanted_var_list as their input value for using the Coverage kwarg diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index 4ff30d198..5c298c625 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -11,6 +11,7 @@ import zipfile import icepyx.core.APIformatting as apifmt +from icepyx.core.auth import EarthdataAuthMixin import icepyx.core.exceptions @@ -139,7 +140,7 @@ def gran_IDs(grans, ids=False, cycles=False, tracks=False, dates=False, cloud=Fa # DevGoal: this will be a great way/place to manage data from the local file system # where the user already has downloaded data! # DevNote: currently this class is not tested -class Granules: +class Granules(EarthdataAuthMixin): """ Interact with ICESat-2 data granules. This includes finding, ordering, and downloading them as well as (not yet implemented) getting already @@ -157,7 +158,9 @@ def __init__( # files=[], # session=None ): - pass + # initialize authentication properties + EarthdataAuthMixin.__init__(self) + # self.avail = avail # self.orderIDs = orderIDs # self.files = files @@ -207,7 +210,7 @@ def get_avail(self, CMRparams, reqparams, cloud=False): params = apifmt.combine_params( CMRparams, - {k: reqparams[k] for k in ["page_size"]}, + {k: reqparams[k] for k in ["short_name", "version", "page_size"]}, {"provider": "NSIDC_CPRD"}, ) @@ -264,7 +267,6 @@ def place_order( subsetparams, verbose, subset=True, - session=None, geom_filepath=None, ): # , **kwargs): """ @@ -294,11 +296,6 @@ def place_order( Spatial subsetting returns all data that are within the area of interest (but not complete granules. This eliminates false-positive granules returned by the metadata-level search) - session : requests.session object - A session object authenticating the user to order data using their - Earthdata login information. - The session object will automatically be passed from the query object if you - have successfully logged in there. geom_filepath : string, default None String of the full filename and path when the spatial input is a file. @@ -312,11 +309,6 @@ def place_order( query.Query.order_granules """ - if session is None: - raise ValueError( - "Don't forget to log in to Earthdata using query.earthdata_login()" - ) - base_url = "https://n5eil02u.ecs.nsidc.org/egi/request" self.get_avail(CMRparams, reqparams) @@ -352,15 +344,12 @@ def place_order( total_pages, " is submitting to NSIDC", ) - request_params = apifmt.combine_params(CMRparams, reqparams, subsetparams) request_params.update({"page_num": page_num}) - # DevNote: earlier versions of the code used a file upload+post rather than putting the geometries - # into the parameter dictionaries. However, this wasn't working with shapefiles, but this more general - # solution does, so the geospatial parameters are included in the parameter dictionaries. - request = session.get(base_url, params=request_params) + request = self.session.get(base_url, params=request_params) - # DevGoal: use the request response/number to do some error handling/give the user better messaging for failures + # DevGoal: use the request response/number to do some error handling/ + # give the user better messaging for failures # print(request.content) root = ET.fromstring(request.content) # print([subset_agent.attrib for subset_agent in root.iter('SubsetAgent')]) @@ -394,7 +383,7 @@ def place_order( print("status URL: ", statusURL) # Find order status - request_response = session.get(statusURL) + request_response = self.session.get(statusURL) if verbose is True: print( "HTTP response from order response URL: ", @@ -419,7 +408,7 @@ def place_order( ) # print('Status is not complete. Trying again') time.sleep(10) - loop_response = session.get(statusURL) + loop_response = self.session.get(statusURL) # Raise bad request: Loop will stop for bad response code. loop_response.raise_for_status() @@ -473,7 +462,7 @@ def place_order( return self.orderIDs - def download(self, verbose, path, session=None, restart=False): + def download(self, verbose, path, restart=False): """ Downloads the data for the object's orderIDs, which are generated by ordering data from the NSIDC. @@ -485,10 +474,6 @@ def download(self, verbose, path, session=None, restart=False): Progress information is automatically printed regardless of the value of verbose. path : string String with complete path to desired download directory and location. - session : requests.session object - A session object authenticating the user to download data using their Earthdata login information. - The session object will automatically be passed from the query object if you - have successfully logged in there. restart : boolean, default False Restart your download if it has been interrupted. If the kernel has been restarted, but you successfully @@ -509,13 +494,6 @@ def download(self, verbose, path, session=None, restart=False): Unzip the downloaded granules. """ - # Note: need to test these checks still - if session is None: - raise ValueError( - "Don't forget to log in to Earthdata using query.earthdata_login()" - ) - # DevGoal: make this a more robust check for an active session - # DevNote: this will replace any existing orderIDs with the saved list # (could create confusion depending on whether download was interrupted or kernel restarted) order_fn = ".order_restart" @@ -529,7 +507,8 @@ def download(self, verbose, path, session=None, restart=False): "Please confirm that you have submitted a valid order and it has successfully completed." ) - # DevNote: Temporary. Hard code the orderID info files here. order_fn should be consistent with place_order. + # DevNote: Temporary. Hard code the orderID info files here. + # order_fn should be consistent with place_order. downid_fn = ".download_ID" @@ -552,7 +531,7 @@ def download(self, verbose, path, session=None, restart=False): print("Beginning download of zipped output...") try: - zip_response = session.get(downloadURL) + zip_response = self.session.get(downloadURL) # Raise bad request: Loop will stop for bad response code. zip_response.raise_for_status() print( @@ -566,7 +545,8 @@ def download(self, verbose, path, session=None, restart=False): print( "Unable to download ", order, ". Check granule order for messages." ) - # DevGoal: move this option back out to the is2class level and implement it in an alternate way? + # DevGoal: move this option back out to the is2class level + # and implement it in an alternate way? # #Note: extract the data to save it locally else: with zipfile.ZipFile(io.BytesIO(zip_response.content)) as z: diff --git a/icepyx/core/query.py b/icepyx/core/query.py index 25f13d5b6..46a306dd2 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -532,15 +532,14 @@ def tracks(self): @property def CMRparams(self): """ - Display the CMR key:value pairs that will be submitted. It generates the dictionary if it does not already exist. + Display the CMR key:value pairs that will be submitted. + It generates the dictionary if it does not already exist. Examples -------- >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28']) >>> reg_a.CMRparams - {'short_name': 'ATL06', - 'version': '006', - 'temporal': '2019-02-20T00:00:00Z,2019-02-28T23:59:59Z', + {'temporal': '2019-02-20T00:00:00Z,2019-02-28T23:59:59Z', 'bounding_box': '-55.0,68.0,-48.0,71.0'} """ @@ -552,7 +551,7 @@ def CMRparams(self): # dictionary of optional CMR parameters kwargs = {} # temporal CMR parameters - if hasattr(self, "_temporal"): + if hasattr(self, "_temporal") and self.product != "ATL11": kwargs["start"] = self._temporal._start kwargs["end"] = self._temporal._end # granule name CMR parameters (orbital or file name) @@ -564,8 +563,6 @@ def CMRparams(self): if self._CMRparams.fmted_keys == {}: self._CMRparams.build_params( - product=self.product, - version=self._version, extent_type=self._spatial._ext_type, spatial_extent=self._spatial.fmt_for_CMR(), **kwargs, @@ -583,22 +580,23 @@ def reqparams(self): -------- >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28']) >>> reg_a.reqparams - {'page_size': 2000} + {'short_name': 'ATL06', 'version': '006', 'page_size': 2000} >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28']) # doctest: +SKIP >>> reg_a.order_granules() # doctest: +SKIP >>> reg_a.reqparams # doctest: +SKIP - {'page_size': 2000, 'page_num': 1, 'request_mode': 'async', 'include_meta': 'Y', 'client_string': 'icepyx'} + {'short_name': 'ATL06', 'version': '006', 'page_size': 2000, 'page_num': 1, 'request_mode': 'async', 'include_meta': 'Y', 'client_string': 'icepyx'} """ if not hasattr(self, "_reqparams"): self._reqparams = apifmt.Parameters("required", reqtype="search") - self._reqparams.build_params() + self._reqparams.build_params(product=self.product, version=self._version) return self._reqparams.fmted_keys # @property - # DevQuestion: if I make this a property, I get a "dict" object is not callable when I try to give input kwargs... what approach should I be taking? + # DevQuestion: if I make this a property, I get a "dict" object is not callable + # when I try to give input kwargs... what approach should I be taking? def subsetparams(self, **kwargs): """ Display the subsetting key:value pairs that will be submitted. @@ -629,7 +627,7 @@ def subsetparams(self, **kwargs): self._subsetparams = apifmt.Parameters("subset") # temporal subsetting parameters - if hasattr(self, "temporal"): + if hasattr(self, "_temporal") and self.product != "ATL11": kwargs["start"] = self._temporal._start kwargs["end"] = self._temporal._end @@ -1031,14 +1029,13 @@ def order_granules(self, verbose=False, subset=True, email=False, **kwargs): "NSIDC only allows ordering of one granule by name at a time; your orders will be placed accordingly." ) for gran in gran_name_list: - tempCMRparams.update({"readable_granule_name[]": gran}) + tempCMRparams["readable_granule_name[]"] = gran self._granules.place_order( tempCMRparams, self.reqparams, self.subsetparams(**kwargs), verbose, subset, - session=self.session, geom_filepath=self._spatial._geom_file, ) @@ -1049,7 +1046,6 @@ def order_granules(self, verbose=False, subset=True, email=False, **kwargs): self.subsetparams(**kwargs), verbose, subset, - session=self.session, geom_filepath=self._spatial._geom_file, ) @@ -1114,7 +1110,7 @@ def download_granules( ): self.order_granules(verbose=verbose, subset=subset, **kwargs) - self._granules.download(verbose, path, session=self.session, restart=restart) + self._granules.download(verbose, path, restart=restart) # DevGoal: add testing? What do we test, and how, given this is a visualization. # DevGoal(long term): modify this to accept additional inputs, etc. diff --git a/icepyx/tests/test_APIformatting.py b/icepyx/tests/test_APIformatting.py index d934a97dd..6dfe98353 100644 --- a/icepyx/tests/test_APIformatting.py +++ b/icepyx/tests/test_APIformatting.py @@ -1,4 +1,3 @@ -import pytest import datetime as dt import icepyx.core.APIformatting as apifmt @@ -103,18 +102,52 @@ def test_combine_params(): ############ to_string ############# def test_to_string(): CMRparams = { + "temporal": "2019-02-20T00:00:00Z,2019-02-28T23:59:59Z", + "bounding_box": "-55,68,-48,71", + } + reqparams = { "short_name": "ATL06", "version": "002", - "temporal": "2019-02-20T00:00:00Z,2019-02-28T23:59:59Z", + "page_size": 2000, + "page_num": 1, + } + params = apifmt.combine_params(CMRparams, reqparams) + obs = apifmt.to_string(params) + expected = ( + "temporal=2019-02-20T00:00:00Z,2019-02-28T23:59:59Z" + "&bounding_box=-55,68,-48,71" + "&short_name=ATL06&version=002" + "&page_size=2000&page_num=1" + ) + assert obs == expected + + +def test_to_string_with_list(): + CMRparams = { + "options[readable_granule_name][pattern]": "true", + "options[spatial][or]": "true", + "readable_granule_name[]": [ + "ATL06_??????????????_084903??_*", + "ATL06_??????????????_090203??_*", + ], "bounding_box": "-55,68,-48,71", } - reqparams = {"page_size": 2000, "page_num": 1} + reqparams = { + "short_name": "ATL06", + "version": "002", + "page_size": 2000, + "page_num": 1, + } params = apifmt.combine_params(CMRparams, reqparams) obs = apifmt.to_string(params) expected = ( - "short_name=ATL06&version=002" - "&temporal=2019-02-20T00:00:00Z,2019-02-28T23:59:59Z" - "&bounding_box=-55,68,-48,71&page_size=2000&page_num=1" + "options[readable_granule_name][pattern]=true" + "&options[spatial][or]=true" + "&readable_granule_name[]=ATL06_??????????????_084903??_*" + "&readable_granule_name[]=ATL06_??????????????_090203??_*" + "&bounding_box=-55,68,-48,71" + "&short_name=ATL06&version=002" + "&page_size=2000&page_num=1" ) assert obs == expected @@ -131,7 +164,6 @@ def test_CMRparams_no_other_inputs(): CMRparams = apifmt.Parameters("CMR") # TestQuestion: the next statement essentially tests _get_possible_keys as well, so how would I test them independently? assert CMRparams.poss_keys == { - "default": ["short_name", "version"], "spatial": ["bounding_box", "polygon"], "optional": [ "temporal", @@ -143,14 +175,9 @@ def test_CMRparams_no_other_inputs(): assert CMRparams.fmted_keys == {} assert CMRparams._check_valid_keys # Note: this test must be done before the next one - if CMRparams.partype == "required": - assert CMRparams.check_req_values() == False - else: - assert CMRparams.check_values() == False + assert CMRparams.check_values() is False CMRparams.build_params( - product="ATL06", - version="006", start=dt.datetime(2019, 2, 20, 0, 0), end=dt.datetime(2019, 2, 24, 23, 59, 59), extent_type="bounding_box", @@ -158,8 +185,6 @@ def test_CMRparams_no_other_inputs(): ) obs_fmted_params = CMRparams.fmted_keys exp_fmted_params = { - "short_name": "ATL06", - "version": "006", "temporal": "2019-02-20T00:00:00Z,2019-02-24T23:59:59Z", "bounding_box": "-55.0,68.0,-48.0,71.0", } diff --git a/icepyx/tests/test_granules.py b/icepyx/tests/test_granules.py index a53d546a4..4e12c0eb3 100644 --- a/icepyx/tests/test_granules.py +++ b/icepyx/tests/test_granules.py @@ -639,6 +639,6 @@ def test_avail_granule_CMR_error(): ermsg = "An error was returned from NSIDC in regards to your query: temporal start datetime is invalid: [badinput] is not a valid datetime." with pytest.raises(NsidcQueryError, match=re.escape(ermsg)): - CMRparams = {"version": "003", "temporal": "badinput", "short_name": "ATL08"} - reqparams = {"page_size": 1} + CMRparams = {"temporal": "badinput"} + reqparams = {"version": "003", "short_name": "ATL08", "page_size": 1} Granules().get_avail(CMRparams=CMRparams, reqparams=reqparams)