diff --git a/doc/source/example_notebooks/IS2_DEM_comparison_WIP.ipynb b/doc/source/example_notebooks/IS2_DEM_comparison_WIP.ipynb index bf1bc8af4..0d897c393 100644 --- a/doc/source/example_notebooks/IS2_DEM_comparison_WIP.ipynb +++ b/doc/source/example_notebooks/IS2_DEM_comparison_WIP.ipynb @@ -176,7 +176,9 @@ }, "source": [ "```{admonition} Important Authentication Update\n", - "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is deprecated and will result in an error, as icepyx will call the login function as needed. The user will still need to provide their credentials.\n", + "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is no longer required, as icepyx will call the login function as needed. The user will still need to provide their credentials using one of the three methods decribed in the [ICESat-2 Data Access Notebook](https://icepyx.readthedocs.io/en/latest/example_notebooks/IS2_data_access.html) example. The `.earthdata_login()` function is still available for backwards compatibility.\n", + "\n", + "If you are unable to remove `earthdata_login()` calls from your workflow, note that certain inputs, such as `earthdata_uid` and `email`, are no longer required. e.g. `region_a.earthdata_login(earthdata_uid, email)` becomes `region_a.earthdata_login()`\n", "```" ] }, diff --git a/doc/source/example_notebooks/IS2_cloud_data_access.ipynb b/doc/source/example_notebooks/IS2_cloud_data_access.ipynb index 194e25210..5bf5d1a98 100644 --- a/doc/source/example_notebooks/IS2_cloud_data_access.ipynb +++ b/doc/source/example_notebooks/IS2_cloud_data_access.ipynb @@ -229,7 +229,9 @@ }, "source": [ "```{admonition} Important Authentication Update\n", - "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is deprecated and will result in an error, as icepyx will call the login function as needed. The user will still need to provide their credentials.\n", + "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is no longer required, as icepyx will call the login function as needed. The user will still need to provide their credentials using one of the three methods decribed in the [ICESat-2 Data Access Notebook](https://icepyx.readthedocs.io/en/latest/example_notebooks/IS2_data_access.html) example. The `.earthdata_login()` function is still available for backwards compatibility.\n", + "\n", + "If you are unable to remove `earthdata_login()` calls from your workflow, note that certain inputs, such as `earthdata_uid` and `email`, are no longer required. e.g. `region_a.earthdata_login(earthdata_uid, email)` becomes `region_a.earthdata_login()`\n", "```" ] }, @@ -384,7 +386,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.10" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/doc/source/example_notebooks/IS2_data_access.ipynb b/doc/source/example_notebooks/IS2_data_access.ipynb index 704abe10c..0b4a12244 100644 --- a/doc/source/example_notebooks/IS2_data_access.ipynb +++ b/doc/source/example_notebooks/IS2_data_access.ipynb @@ -485,7 +485,9 @@ }, "source": [ "```{admonition} Important Authentication Update\n", - "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is deprecated and will result in an error, as icepyx will call the login function as needed. The user will still need to provide their credentials using one of the three methods decribed above.\n", + "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is no longer required, as icepyx will call the login function as needed. The user will still need to provide their credentials using one of the three methods decribed above. The `.earthdata_login()` function is still available for backwards compatibility.\n", + "\n", + "If you are unable to remove `earthdata_login()` calls from your workflow, note that certain inputs, such as `earthdata_uid` and `email`, are no longer required. e.g. `region_a.earthdata_login(earthdata_uid, email)` becomes `region_a.earthdata_login()`\n", "```" ] }, @@ -624,9 +626,9 @@ ], "metadata": { "kernelspec": { - "display_name": "icepyx", + "display_name": "icepyx-dev", "language": "python", - "name": "python3" + "name": "icepyx-dev" }, "language_info": { "codemirror_mode": { @@ -638,7 +640,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.10" + "version": "3.11.4" } }, "nbformat": 4, diff --git a/doc/source/example_notebooks/IS2_data_access2-subsetting.ipynb b/doc/source/example_notebooks/IS2_data_access2-subsetting.ipynb index 1bb178d8a..3803b9fd6 100644 --- a/doc/source/example_notebooks/IS2_data_access2-subsetting.ipynb +++ b/doc/source/example_notebooks/IS2_data_access2-subsetting.ipynb @@ -77,7 +77,9 @@ }, "source": [ "```{admonition} Important Authentication Update\n", - "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is deprecated and will result in an error, as icepyx will call the login function as needed. The user will still need to provide their credentials.\n", + "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is no longer required, as icepyx will call the login function as needed. The user will still need to provide their credentials using one of the three methods decribed in the [ICESat-2 Data Access Notebook](https://icepyx.readthedocs.io/en/latest/example_notebooks/IS2_data_access.html) example. The `.earthdata_login()` function is still available for backwards compatibility.\n", + "\n", + "If you are unable to remove `earthdata_login()` calls from your workflow, note that certain inputs, such as `earthdata_uid` and `email`, are no longer required. e.g. `region_a.earthdata_login(earthdata_uid, email)` becomes `region_a.earthdata_login()`\n", "```" ] }, diff --git a/doc/source/example_notebooks/IS2_data_read-in.ipynb b/doc/source/example_notebooks/IS2_data_read-in.ipynb index c20cfbc95..9bbac368b 100644 --- a/doc/source/example_notebooks/IS2_data_read-in.ipynb +++ b/doc/source/example_notebooks/IS2_data_read-in.ipynb @@ -157,7 +157,9 @@ }, "source": [ "```{admonition} Important Authentication Update\n", - "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is deprecated and will result in an error, as icepyx will call the login function as needed. The user will still need to provide their credentials.\n", + "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is no longer required, as icepyx will call the login function as needed. The user will still need to provide their credentials using one of the three methods decribed in the [ICESat-2 Data Access Notebook](https://icepyx.readthedocs.io/en/latest/example_notebooks/IS2_data_access.html) example. The `.earthdata_login()` function is still available for backwards compatibility.\n", + "\n", + "If you are unable to remove `earthdata_login()` calls from your workflow, note that certain inputs, such as `earthdata_uid` and `email`, are no longer required. e.g. `region_a.earthdata_login(earthdata_uid, email)` becomes `region_a.earthdata_login()`\n", "```" ] }, @@ -328,7 +330,7 @@ "```{admonition} Read Module Update\n", "Previously, icepyx required two additional conditions: 1) a `product` argument and 2) that your files either matched the default `filename_pattern` or that the user provided their own `filename_pattern`. These two requirements have been removed. `product` is now read directly from the file metadata (the root group's `short_name` attribute). Flexibility to specify multiple files via the `filename_pattern` has been replaced with the [glob string](https://docs.python.org/3/library/glob.html) feature, and by allowing a list of filepaths as an argument.\n", "\n", - "The `product` and `filename_pattern` arguments are now deprecated and were removed in icepyx version 1.0.0.\n", + "The `product` and `filename_pattern` arguments have been maintained for backwards compatibility, but will be fully removed in icepyx version 1.0.0.\n", "```" ] }, diff --git a/doc/source/example_notebooks/IS2_data_variables.ipynb b/doc/source/example_notebooks/IS2_data_variables.ipynb index 90fa8500c..c66445731 100644 --- a/doc/source/example_notebooks/IS2_data_variables.ipynb +++ b/doc/source/example_notebooks/IS2_data_variables.ipynb @@ -347,7 +347,9 @@ }, "source": [ "```{admonition} Important Authentication Update\n", - "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is deprecated and will result in an error, as icepyx will call the login function as needed. The user will still need to provide their credentials.\n", + "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is no longer required, as icepyx will call the login function as needed. The user will still need to provide their credentials using one of the three methods decribed in the [ICESat-2 Data Access Notebook](https://icepyx.readthedocs.io/en/latest/example_notebooks/IS2_data_access.html) example. The `.earthdata_login()` function is still available for backwards compatibility.\n", + "\n", + "If you are unable to remove `earthdata_login()` calls from your workflow, note that certain inputs, such as `earthdata_uid` and `email`, are no longer required. e.g. `region_a.earthdata_login(earthdata_uid, email)` becomes `region_a.earthdata_login()`\n", "```" ] }, diff --git a/doc/source/example_notebooks/IS2_data_visualization.ipynb b/doc/source/example_notebooks/IS2_data_visualization.ipynb index 0f2819862..ec87829c2 100644 --- a/doc/source/example_notebooks/IS2_data_visualization.ipynb +++ b/doc/source/example_notebooks/IS2_data_visualization.ipynb @@ -210,7 +210,9 @@ }, "source": [ "```{admonition} Important Authentication Update\n", - "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is deprecated and will result in an error, as icepyx will call the login function as needed. The user will still need to provide their credentials.\n", + "Previously, icepyx required you to explicitly use the `.earthdata_login()` function to login. Running this function is no longer required, as icepyx will call the login function as needed. The user will still need to provide their credentials using one of the three methods decribed in the [ICESat-2 Data Access Notebook](https://icepyx.readthedocs.io/en/latest/example_notebooks/IS2_data_access.html) example. The `.earthdata_login()` function is still available for backwards compatibility.\n", + "\n", + "If you are unable to remove `earthdata_login()` calls from your workflow, note that certain inputs, such as `earthdata_uid` and `email`, are no longer required. e.g. `region_a.earthdata_login(earthdata_uid, email)` becomes `region_a.earthdata_login()`\n", "```" ] }, diff --git a/doc/source/user_guide/documentation/classes_dev_uml.svg b/doc/source/user_guide/documentation/classes_dev_uml.svg index 09c112f5c..c5cd801aa 100644 --- a/doc/source/user_guide/documentation/classes_dev_uml.svg +++ b/doc/source/user_guide/documentation/classes_dev_uml.svg @@ -232,31 +232,32 @@ icepyx.core.read.Read - -Read - -_filelist : list -_out_obj : Dataset -_product -_read_vars -filelist -is_s3 -product -vars - -__init__(data_source, glob_kwargs, out_obj_type, product, filename_pattern, catalog) -_add_vars_to_ds(is2ds, ds, grp_path, wanted_groups_tiered, wanted_dict) -_build_dataset_template(file) -_build_single_file_dataset(file, groups_list) -_combine_nested_vars(is2ds, ds, grp_path, wanted_dict) -_read_single_grp(file, grp_path) -load() + +Read + +_filelist : NoneType, list +_out_obj : Dataset +_product : NoneType, str +_read_vars +filelist +is_s3 +product +vars + +__init__(data_source, product, filename_pattern, catalog, glob_kwargs, out_obj_type) +_add_vars_to_ds(is2ds, ds, grp_path, wanted_groups_tiered, wanted_dict) +_build_dataset_template(file) +_build_single_file_dataset(file, groups_list) +_check_source_for_pattern(source, filename_pattern) +_combine_nested_vars(is2ds, ds, grp_path, wanted_dict) +_read_single_grp(file, grp_path) +load() icepyx.core.read.Read->icepyx.core.auth.EarthdataAuthMixin - - + + @@ -363,8 +364,8 @@ icepyx.core.variables.Variables->icepyx.core.read.Read - - + + _read_vars diff --git a/doc/source/user_guide/documentation/packages_user_uml.svg b/doc/source/user_guide/documentation/packages_user_uml.svg index 5c45fc92b..a86000de8 100644 --- a/doc/source/user_guide/documentation/packages_user_uml.svg +++ b/doc/source/user_guide/documentation/packages_user_uml.svg @@ -4,11 +4,11 @@ - - + + packages_user_uml - + icepyx.core @@ -24,68 +24,50 @@ icepyx.core.auth - -icepyx.core.auth + +icepyx.core.auth icepyx.core.exceptions - -icepyx.core.exceptions - - - -icepyx.core.auth->icepyx.core.exceptions - - + +icepyx.core.exceptions icepyx.core.granules - -icepyx.core.granules + +icepyx.core.granules icepyx.core.icesat2data - -icepyx.core.icesat2data - - - -icepyx.core.icesat2data->icepyx.core.exceptions - - + +icepyx.core.icesat2data icepyx.core.is2ref - -icepyx.core.is2ref + +icepyx.core.is2ref icepyx.core.query - -icepyx.core.query + +icepyx.core.query - + icepyx.core.query->icepyx.core.auth - - - - - -icepyx.core.query->icepyx.core.exceptions - - + + - + icepyx.core.query->icepyx.core.granules - - + + @@ -94,22 +76,22 @@ icepyx.core.variables - + icepyx.core.query->icepyx.core.variables - - + + icepyx.core.visualization - -icepyx.core.visualization + +icepyx.core.visualization - + icepyx.core.query->icepyx.core.visualization - - + + @@ -118,19 +100,19 @@ icepyx.core.read - + icepyx.core.read->icepyx.core.auth - - + + - + icepyx.core.read->icepyx.core.exceptions - - + + - + icepyx.core.read->icepyx.core.variables @@ -138,32 +120,32 @@ icepyx.core.spatial - -icepyx.core.spatial + +icepyx.core.spatial icepyx.core.temporal - -icepyx.core.temporal + +icepyx.core.temporal icepyx.core.validate_inputs - -icepyx.core.validate_inputs + +icepyx.core.validate_inputs - + icepyx.core.variables->icepyx.core.auth - - + + - + icepyx.core.variables->icepyx.core.exceptions - - + + diff --git a/icepyx/core/auth.py b/icepyx/core/auth.py index ba07ac398..cf771f420 100644 --- a/icepyx/core/auth.py +++ b/icepyx/core/auth.py @@ -3,7 +3,6 @@ import warnings import earthaccess -from icepyx.core.exceptions import DeprecationError class AuthenticationError(Exception): @@ -16,8 +15,7 @@ class AuthenticationError(Exception): class EarthdataAuthMixin: """ - This mixin class generates the needed authentication sessions and tokens, - including for NASA Earthdata cloud access. + This mixin class generates the needed authentication sessions and tokens, including for NASA Earthdata cloud access. Authentication is completed using the [earthaccess library](https://nsidc.github.io/earthaccess/). Methods for authenticating are: 1. Storing credentials as environment variables ($EARTHDATA_LOGIN and $EARTHDATA_PASSWORD) @@ -25,14 +23,12 @@ class EarthdataAuthMixin: 3. Storing credentials in a .netrc file (not recommended for security reasons) More details on using these methods is available in the [earthaccess documentation](https://nsidc.github.io/earthaccess/tutorials/restricted-datasets/#auth). - This class can be inherited by any other class that requires authentication. - For example, the `Query` class inherits this one, and so a Query object has the - `.session` property. - The method `earthdata_login()` is included for backwards compatibility. + This class can be inherited by any other class that requires authentication. For + example, the `Query` class inherits this one, and so a Query object has the + `.session` property. The method `earthdata_login()` is included for backwards compatibility. The class can be created without any initialization parameters, and the properties will - be populated when they are called. - It can alternately be initialized with an + be populated when they are called. It can alternately be initialized with an earthaccess.auth.Auth object, which will then be used to create a session or s3login_credentials as they are called. @@ -94,8 +90,8 @@ def session(self): @property def s3login_credentials(self): """ - A dictionary which stores login credentials for AWS s3 access. - This property is accessed if using AWS cloud data. + A dictionary which stores login credentials for AWS s3 access. This property is accessed + if using AWS cloud data. Because s3 tokens are only good for one hour, this function will automatically check if an hour has elapsed since the last token use and generate a new token if necessary. @@ -119,12 +115,9 @@ def set_s3_creds(): def earthdata_login(self, uid=None, email=None, s3token=None, **kwargs) -> None: """ Authenticate with NASA Earthdata to enable data ordering and download. - Credential storage details are described in the - EathdataAuthMixin class section. + Credential storage details are described in the EathdataAuthMixin class section. - **Note:** This method is deprecated and will be removed in a future release. - It is no longer required to explicitly run `.earthdata_login()`. - Authentication will be performed by the module as needed. + **Note:** This method is maintained for backward compatibility. It is no longer required to explicitly run `.earthdata_login()`. Authentication will be performed by the module as needed when `.session` or `.s3login_credentials` are accessed. Parameters ---------- @@ -133,8 +126,7 @@ def earthdata_login(self, uid=None, email=None, s3token=None, **kwargs) -> None: email : string, default None Deprecated keyword for backwards compatibility. s3token : boolean, default None - Deprecated keyword to generate AWS s3 ICESat-2 - data access credentials + Deprecated keyword to generate AWS s3 ICESat-2 data access credentials kwargs : key:value pairs Keyword arguments to be passed into earthaccess.login(). @@ -148,7 +140,15 @@ def earthdata_login(self, uid=None, email=None, s3token=None, **kwargs) -> None: No .netrc found in /Users/username """ - raise DeprecationError( - "It is no longer required to explicitly run the `.earthdata_login()` method." - "Authentication will be performed by the module as needed.", + warnings.warn( + "It is no longer required to explicitly run the `.earthdata_login()` method. Authentication will be performed by the module as needed.", + DeprecationWarning, + stacklevel=2, ) + + if uid != None or email != None or s3token != None: + warnings.warn( + "The user id (uid) and/or email keyword arguments are no longer required.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/icepyx/core/icesat2data.py b/icepyx/core/icesat2data.py index e57305124..aa35fd433 100644 --- a/icepyx/core/icesat2data.py +++ b/icepyx/core/icesat2data.py @@ -1,10 +1,12 @@ -from icepyx.core.exceptions import DeprecationError +import warnings class Icesat2Data: def __init__( self, ): - DeprecationError( + warnings.filterwarnings("always") + warnings.warn( "DEPRECATED. Please use icepyx.Query to create a download data object (all other functionality is the same)", + DeprecationWarning, ) diff --git a/icepyx/core/query.py b/icepyx/core/query.py index 25f13d5b6..32357eb8a 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -2,10 +2,10 @@ import matplotlib.pyplot as plt from pathlib import Path # used in docstring tests import pprint +import warnings import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin -from icepyx.core.exceptions import DeprecationError import icepyx.core.granules as granules from icepyx.core.granules import Granules import icepyx.core.is2ref as is2ref @@ -455,8 +455,10 @@ def dataset(self): -------- product """ - DeprecationError( + warnings.filterwarnings("always") + warnings.warn( "In line with most common usage, 'dataset' has been replaced by 'product'.", + DeprecationWarning, ) @property diff --git a/icepyx/core/read.py b/icepyx/core/read.py index e11015935..f2dae6495 100644 --- a/icepyx/core/read.py +++ b/icepyx/core/read.py @@ -290,30 +290,31 @@ class Read(EarthdataAuthMixin): 3) a [glob string](https://docs.python.org/3/library/glob.html). The List must be a list of strings, each of which is the path of a single file. - glob_kwargs : dict, default {} - Additional arguments to be passed into the [glob.glob()](https://docs.python.org/3/library/glob.html#glob.glob)function - - out_obj_type : object, default xarray.Dataset - The desired format for the data to be read in. - Currently, only xarray.Dataset objects (default) are available. - Please ask us how to help enable usage of other data objects! - product : string ICESat-2 data product ID, also known as "short name" (e.g. ATL03). Available data products can be found at: https://nsidc.org/data/icesat-2/data-sets - **Deprecation warning:** This argument is no longer required and has been deprecated. + **Deprecation warning:** This argument is no longer required and will be deprecated in version 1.0.0. The dataset product is read from the file metadata. filename_pattern : string, default None String that shows the filename pattern as previously required for Intake's path_as_pattern argument. The default describes files downloaded directly from NSIDC (subsetted and non-subsetted) for most products (e.g. ATL06). The ATL11 filename pattern from NSIDC is: 'ATL{product:2}_{rgt:4}{orbitsegment:2}_{cycles:4}_{version:3}_{revision:2}.h5'. - **Deprecation warning:** This argument is no longer required and has been deprecated. + **Deprecation warning:** This argument is no longer required and will be deprecated in version 1.0.0. + catalog : string, default None Full path to an Intake catalog for reading in data. If you still need to create a catalog, leave as default. **Deprecation warning:** This argument has been deprecated. Please use the data_source argument to pass in valid data. + glob_kwargs : dict, default {} + Additional arguments to be passed into the [glob.glob()](https://docs.python.org/3/library/glob.html#glob.glob)function + + out_obj_type : object, default xarray.Dataset + The desired format for the data to be read in. + Currently, only xarray.Dataset objects (default) are available. + Please ask us how to help enable usage of other data objects! + Returns ------- read object @@ -343,34 +344,54 @@ class Read(EarthdataAuthMixin): def __init__( self, - data_source, - glob_kwargs={}, - out_obj_type=None, # xr.Dataset, - # deprecated arguments + data_source=None, # DevNote: Make this a required arg when catalog is removed product=None, filename_pattern=None, catalog=None, + glob_kwargs={}, + out_obj_type=None, # xr.Dataset, ): + # Raise error for deprecated argument + if catalog: + raise DeprecationError( + "The `catalog` argument has been deprecated and intake is no longer supported. " + "Please use the `data_source` argument to specify your dataset instead." + ) + + if data_source is None: + raise ValueError("data_source is a required arguemnt") + # initialize authentication properties EarthdataAuthMixin.__init__(self) - # Raise errors for deprecated arguments + # Raise warnings for deprecated arguments if filename_pattern: - raise DeprecationError( + warnings.warn( "The `filename_pattern` argument is deprecated. Instead please provide a " - "string, list, or glob string to the `data_source` argument." + "string, list, or glob string to the `data_source` argument.", + stacklevel=2, ) if product: - raise DeprecationError("The `product` argument is no longer required.") - - if catalog: - raise DeprecationError( - "The `catalog` argument has been deprecated and intake is no longer supported. " - "Please use the `data_source` argument to specify your dataset instead." + product = is2ref._validate_product(product) + warnings.warn( + "The `product` argument is no longer required. If the `data_source` argument given " + "contains files with multiple products the `product` argument will be used " + "to filter that list. In all other cases the product argument is ignored. " + "The recommended approach is to not include a `product` argument and instead " + "provide a `data_source` with files of only a single product type`.", + stacklevel=2, ) - if isinstance(data_source, list): + # Create the filelist from the `data_source` argument + if filename_pattern: + # maintained for backward compatibility + pattern_ck, filelist = Read._check_source_for_pattern( + data_source, filename_pattern + ) + assert pattern_ck + self._filelist = filelist + elif isinstance(data_source, list): # if data_source is a list pass that directly to _filelist self._filelist = data_source elif os.path.isdir(data_source): @@ -389,7 +410,6 @@ def __init__( "data_source should be a list of files, a directory, the path to a file, " "or a glob string." ) - # Remove any directories from the list (these get generated during recursive # glob search) self._filelist = [f for f in self._filelist if not os.path.isdir(f)] @@ -426,11 +446,31 @@ def __init__( # Raise warnings or errors for multiple products or products not matching the user-specified product all_products = list(set(product_dict.values())) if len(all_products) > 1: - raise TypeError( - f"Multiple product types were found in the file list: {product_dict}." - "Please provide a valid `data_source` parameter indicating files of a single " - "product" - ) + if product: + warnings.warn( + f"Multiple products found in list of files: {product_dict}. Files that " + "do not match the user specified product will be removed from processing.\n" + "Filtering files using a `product` argument is deprecated. Please use the " + "`data_source` argument to specify a list of files with the same product.", + stacklevel=2, + ) + self._filelist = [] + for key, value in product_dict.items(): + if value == product: + self._filelist.append(key) + if len(self._filelist) == 0: + raise TypeError( + "No files found in the file list matching the user-specified " + "product type" + ) + # Use the cleaned filelist to assign a product + self._product = product + else: + raise TypeError( + f"Multiple product types were found in the file list: {product_dict}." + "Please provide a valid `data_source` parameter indicating files of a single " + "product" + ) elif len(all_products) == 0: raise TypeError( "No files found matching the specified `data_source`. Check your glob " @@ -439,6 +479,13 @@ def __init__( else: # Assign the identified product to the property self._product = all_products[0] + # Raise a warning if the metadata-located product differs from the user-specified product + if product and self._product != product: + warnings.warn( + f"User specified product {product} does not match the product from the file" + " metadata {self._product}", + stacklevel=2, + ) if out_obj_type is not None: print( @@ -450,6 +497,8 @@ def __init__( # ---------------------------------------------------------------------- # Properties + # I cut and pasted this directly out of the Query class - going to need to reconcile the _source/file stuff there + @property def vars(self): """ @@ -487,6 +536,35 @@ def product(self): # ---------------------------------------------------------------------- # Methods + @staticmethod + def _check_source_for_pattern(source, filename_pattern): + """ + Check that the entered data source contains files that match the input filename_pattern + """ + glob_pattern = _pattern_to_glob(filename_pattern) + + if os.path.isdir(source): + _, filelist = _run_fast_scandir(source, glob_pattern) + assert ( + len(filelist) > 0 + ), "None of your filenames match the specified pattern." + print( + f"You have {len(filelist)} files matching the filename pattern to be read in." + ) + return True, filelist + elif os.path.isfile(source): + assert fnmatch.fnmatch( + os.path.basename(source), glob_pattern + ), "Your input filename does not match the filename pattern." + return True, [source] + elif isinstance(source, str): + if source.startswith("s3://"): + return True, [source] + elif isinstance(source, list): + if all(source.startswith("s3://")): + return True, source + + return False, None @staticmethod def _add_vars_to_ds(is2ds, ds, grp_path, wanted_groups_tiered, wanted_dict): @@ -695,13 +773,9 @@ def load(self): # add a check that wanted variables exists, and create them with defaults if possible (and let the user know) # write tests for the functions! - # Notes: intake wants an entire group, not an individual variable - # (which makes sense if we're using its smarts to set up lat, lon, etc) - # so to get a combined dataset, we need to keep track of spots under the hood, - # open each group, and then combine them into one xarray where the spots are IDed somehow - # (or only the strong ones are returned) - # this means we need to get/track from each dataset we open some of the metadata, - # which we include as mandatory variables when constructing the wanted list + # Notes: intake wants an entire group, not an individual variable (which makes sense if we're using its smarts to set up lat, lon, etc) + # so to get a combined dataset, we need to keep track of spots under the hood, open each group, and then combine them into one xarray where the spots are IDed somehow (or only the strong ones are returned) + # this means we need to get/track from each dataset we open some of the metadata, which we include as mandatory variables when constructing the wanted list if not self.vars.wanted: raise AttributeError( @@ -742,7 +816,11 @@ def load(self): pass all_dss = [] - + # DevNote: I'd originally hoped to rely on intake-xarray in order to not have to iterate through the files myself, + # by providing a generalized url/source in building the catalog. + # However, this led to errors when I tried to combine two identical datasets because the single dimension was equal. + # In these situations, xarray recommends manually controlling the merge/concat process yourself. + # While unlikely to be a broad issue, I've heard of multiple matching timestamps causing issues for combining multiple IS2 datasets. for file in self.filelist: if file.startswith("s3"): # If path is an s3 path create an s3fs filesystem to reference the file diff --git a/icepyx/core/variables.py b/icepyx/core/variables.py index 15d5268e5..b611b861d 100644 --- a/icepyx/core/variables.py +++ b/icepyx/core/variables.py @@ -1,6 +1,7 @@ import numpy as np import os import pprint +import warnings from icepyx.core.auth import EarthdataAuthMixin import icepyx.core.is2ref as is2ref diff --git a/icepyx/tests/test_auth.py b/icepyx/tests/test_auth.py index c8f8e8f5d..8507b1e40 100644 --- a/icepyx/tests/test_auth.py +++ b/icepyx/tests/test_auth.py @@ -4,7 +4,6 @@ import earthaccess from icepyx.core.auth import EarthdataAuthMixin -from icepyx.core.exceptions import DeprecationError @pytest.fixture() @@ -32,5 +31,12 @@ def test_get_s3login_credentials(auth_instance): # Test that earthdata_login generates an auth object def test_login_function(auth_instance): + auth_instance.earthdata_login() assert isinstance(auth_instance.auth, earthaccess.auth.Auth) assert auth_instance.auth.authenticated + + +# Test that earthdata_login raises a warning if email is provided +def test_depreciation_warning(auth_instance): + with pytest.warns(DeprecationWarning): + auth_instance.earthdata_login(email="me@gmail.com")