From f27d53dd76b91913aacff9059b03e26445d0fe4a Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Thu, 1 Aug 2024 18:46:28 -0400 Subject: [PATCH] Validate CMR XML using XML schema --- .dockerignore | 1 + hls_vi/__init__.py | 0 hls_vi/generate_indices.py | 34 +- hls_vi/generate_metadata.py | 26 +- hls_vi/generate_stac_items.py | 37 +- hls_vi/py.typed | 0 hls_vi/schema/Granule.xsd | 1997 ++++++++++++++++++++++++++++++ hls_vi/schema/MetadataCommon.xsd | 858 +++++++++++++ mypy.ini | 11 + setup.py | 19 +- tests/test_vi.py | 45 +- tox.ini | 1 + 12 files changed, 2951 insertions(+), 78 deletions(-) create mode 100644 hls_vi/__init__.py create mode 100644 hls_vi/py.typed create mode 100644 hls_vi/schema/Granule.xsd create mode 100644 hls_vi/schema/MetadataCommon.xsd create mode 100644 mypy.ini diff --git a/.dockerignore b/.dockerignore index 6cdb0c4..cff214f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,5 +2,6 @@ !hls_vi/ !tests/ tests/fixtures/ +!mypy.ini !setup.py !tox.ini diff --git a/hls_vi/__init__.py b/hls_vi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hls_vi/generate_indices.py b/hls_vi/generate_indices.py index f2bb343..5650138 100644 --- a/hls_vi/generate_indices.py +++ b/hls_vi/generate_indices.py @@ -1,4 +1,6 @@ -# pyright: reportAttributeAccessIssue=false, reportOperatorIssue=false +# pyright: reportAttributeAccessIssue=false +# pyright: reportOperatorIssue=false +# pyright: reportOptionalOperand=false import getopt import os @@ -8,7 +10,7 @@ from datetime import datetime, timezone from enum import Enum, unique from pathlib import Path -from typing import Callable, Mapping, Optional, SupportsFloat, Tuple, Type +from typing import Callable, List, Mapping, Optional, SupportsFloat, Tuple, Type from typing_extensions import TypeAlias import numpy as np @@ -91,7 +93,7 @@ class Instrument(Enum): def __init__( self, band_type: Type[InstrumentBand], parse_satellite: Callable[[Tags], str] ) -> None: - self.bands = list(band_type) + self.bands: List[InstrumentBand] = list(band_type) self.parse_satellite = parse_satellite @classmethod @@ -207,7 +209,7 @@ def select_tags(granule_id: GranuleId, tags: Tags) -> Tags: } -def write_granule_indices(output_dir: Path, granule: Granule): +def write_granule_indices(output_dir: Path, granule: Granule) -> None: os.makedirs(output_dir, exist_ok=True) processing_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") @@ -232,7 +234,7 @@ def write_granule_index( granule: Granule, index: "Index", processing_time: str, -): +) -> None: """Save raster data to a GeoTIFF file using rasterio.""" data = index(granule.data) @@ -264,16 +266,16 @@ def write_granule_index( def evi(data: BandData) -> np.ma.masked_array: b, r, nir = data[Band.B], data[Band.R], data[Band.NIR] - return 2.5 * (nir - r) / (nir + 6 * r - 7.5 * b + 1) # type: ignore + return 2.5 * (nir - r) / (nir + 6 * r - 7.5 * b + 1) def msavi(data: BandData) -> np.ma.masked_array: r, nir = data[Band.R], data[Band.NIR] - sqrt_term = (2 * nir + 1) ** 2 - 8 * (nir - r) # type: ignore + sqrt_term = (2 * nir + 1) ** 2 - 8 * (nir - r) result: np.ma.masked_array = np.ma.where( sqrt_term >= 0, - (2 * nir + 1 - np.sqrt(sqrt_term)) / 2, # type: ignore + (2 * nir + 1 - np.sqrt(sqrt_term)) / 2, np.nan, ) result.fill_value = r.fill_value @@ -283,38 +285,38 @@ def msavi(data: BandData) -> np.ma.masked_array: def nbr(data: BandData) -> np.ma.masked_array: nir, swir2 = data[Band.NIR], data[Band.SWIR2] - return (nir - swir2) / (nir + swir2) # type: ignore + return (nir - swir2) / (nir + swir2) def nbr2(data: BandData) -> np.ma.masked_array: swir1, swir2 = data[Band.SWIR1], data[Band.SWIR2] - return (swir1 - swir2) / (swir1 + swir2) # type: ignore + return (swir1 - swir2) / (swir1 + swir2) def ndmi(data: BandData) -> np.ma.masked_array: nir, swir1 = data[Band.NIR], data[Band.SWIR1] - return (nir - swir1) / (nir + swir1) # type: ignore + return (nir - swir1) / (nir + swir1) def ndvi(data: BandData) -> np.ma.masked_array: r, nir = data[Band.R], data[Band.NIR] - return (nir - r) / (nir + r) # type: ignore + return (nir - r) / (nir + r) def ndwi(data: BandData) -> np.ma.masked_array: g, nir = data[Band.G], data[Band.NIR] - return (g - nir) / (g + nir) # type: ignore + return (g - nir) / (g + nir) def savi(data: BandData) -> np.ma.masked_array: r, nir = data[Band.R], data[Band.NIR] - return 1.5 * (nir - r) / (nir + r + 0.5) # type: ignore + return 1.5 * (nir - r) / (nir + r + 0.5) def tvi(data: BandData) -> np.ma.masked_array: g, r, nir = data[Band.G], data[Band.R], data[Band.NIR] # We do NOT multiply by 10_000 like we do for other indices. - return (120 * (nir - g) - 200 * (r - g)) / 2 # type: ignore + return (120 * (nir - g) - 200 * (r - g)) / 2 # pyright: ignore[reportReturnType] class Index(Enum): @@ -386,7 +388,7 @@ def generate_vi_granule(input_dir: Path, output_dir: Path, id_str: str) -> Granu return granule -def main(): +def main() -> None: input_dir, output_dir, id_str = parse_args() generate_vi_granule(input_dir, output_dir, id_str) diff --git a/hls_vi/generate_metadata.py b/hls_vi/generate_metadata.py index 554de79..95e7b73 100644 --- a/hls_vi/generate_metadata.py +++ b/hls_vi/generate_metadata.py @@ -1,15 +1,18 @@ import getopt +import importlib_resources import os import sys -import xml.etree.ElementTree as ET + from datetime import datetime, timezone from pathlib import Path from typing import Tuple import rasterio +from lxml import etree as ET +from lxml.etree import Element, ElementBase -def generate_metadata(input_dir: Path, output_dir: Path): +def generate_metadata(input_dir: Path, output_dir: Path) -> None: """ Create CMR XML metadata file for an HLS VI granule. @@ -23,7 +26,7 @@ def generate_metadata(input_dir: Path, output_dir: Path): to this directory with the name `HLS-VI.*.cmr.xml`. """ metadata_path = next(input_dir.glob("HLS.*.cmr.xml")) - tree = ET.parse(metadata_path) + tree = ET.parse(metadata_path, None) with rasterio.open(next(output_dir.glob("*.tif"))) as vi_tif: sensing_times = [t.strip() for t in vi_tif.tags()["SENSING_TIME"].split(";")] @@ -61,6 +64,9 @@ def generate_metadata(input_dir: Path, output_dir: Path): tree.find("Temporal/RangeDateTime/BeginningDateTime").text = sensing_time_begin tree.find("Temporal/RangeDateTime/EndingDateTime").text = sensing_time_end + with (importlib_resources.files("hls_vi") / "schema" / "Granule.xsd").open() as xsd: + ET.XMLSchema(file=xsd).assertValid(tree) + tree.write( output_dir / metadata_path.name.replace("HLS", "HLS-VI"), encoding="utf-8", @@ -68,17 +74,17 @@ def generate_metadata(input_dir: Path, output_dir: Path): ) -def set_additional_attribute(attrs: ET.Element, name: str, value: str): - attr = attrs.find(f'./AdditionalAttribute[Name="{name}"]') +def set_additional_attribute(attrs: ElementBase, name: str, value: str) -> None: + attr = attrs.find(f'./AdditionalAttribute[Name="{name}"]', None) if attr is not None: attr.find(".//Value").text = value else: - attr = ET.Element("AdditionalAttribute") - attr_name = ET.Element("Name") + attr = Element("AdditionalAttribute", None, None) + attr_name = Element("Name", None, None) attr_name.text = name - attr_values = ET.Element("Values") - attr_value = ET.Element("Value") + attr_values = Element("Values", None, None) + attr_value = Element("Value", None, None) attr_value.text = value attr_values.append(attr_value) attr.append(attr_name) @@ -115,7 +121,7 @@ def parse_args() -> Tuple[Path, Path]: return Path(input_dir), Path(output_dir) -def main(): +def main() -> None: input_dir, output_dir = parse_args() generate_metadata(input_dir=input_dir, output_dir=output_dir) diff --git a/hls_vi/generate_stac_items.py b/hls_vi/generate_stac_items.py index 4e4d28d..b69d382 100644 --- a/hls_vi/generate_stac_items.py +++ b/hls_vi/generate_stac_items.py @@ -2,6 +2,7 @@ import os import json import argparse +from typing import Any, Mapping import pystac import rasterio import untangle @@ -80,7 +81,7 @@ } -def get_geometry(granule): +def get_geometry(granule: untangle.Element) -> MultiPolygon: """Function returns the geometry of the HLS_VI granule Args: @@ -109,7 +110,7 @@ def get_geometry(granule): return MultiPolygon(multipolygon) -def process_common_metadata(item, granule): +def process_common_metadata(item: pystac.Item, granule: untangle.Element) -> None: """Function fetches and processes the general information from the granule metadata. and updates the information in the STAC item. @@ -133,14 +134,11 @@ def process_common_metadata(item, granule): ) # For L30, the instrument is "OLI", but for S30, it is "Sentinel-2 MSI", we simply # split on spaces and grab the last element, so we get either "OLI" or "MSI". - if " " in instrument: - item_instrument = instrument.split()[1] - else: - item_instrument = instrument + item_instrument = instrument.split()[1] if " " in instrument else instrument item.common_metadata.instruments = [item_instrument] -def process_eo(item, granule): +def process_eo(item: pystac.Item, granule: untangle.Element) -> None: """Function processes the Earth observation information from the STAC item. Args: @@ -153,7 +151,9 @@ def process_eo(item, granule): eo_extension.cloud_cover = float(attribute.Values.Value.cdata) -def add_assets(item, granule, endpoint, version): +def add_assets( + item: pystac.Item, granule: untangle.Element, endpoint: str, version: str +) -> None: """Function adds all the assets to the STAC item Args: @@ -191,7 +191,9 @@ def add_assets(item, granule, endpoint, version): item.set_self_href(f"{public_url}{item_id}_stac.json") -def process_projection(item, granule, index_file): +def process_projection( + item: pystac.Item, granule: untangle.Element, index_file: str +) -> None: """Function fetches the projection information from the HLS_VI band file and compares if the projection is same for the granule as well as the HLS_VI band image. @@ -210,7 +212,7 @@ def process_projection(item, granule, index_file): proj_ext.epsg = int(f"326{mgrs_tile_id[:2]}") -def process_view_geometry(item, granule): +def process_view_geometry(item: pystac.Item, granule: untangle.Element) -> None: """Function checks the geometry within the attributes of the STAC item and the HLS_VI granule @@ -222,11 +224,11 @@ def process_view_geometry(item, granule): for attribute in granule.AdditionalAttributes.AdditionalAttribute: if attribute.Name == "MEAN_SUN_AZIMUTH_ANGLE": view_extension.sun_azimuth = float(attribute.Values.Value.cdata) - if attribute.Name == "MEAN_VIEW_AZIMUTH_ANGLE": + elif attribute.Name == "MEAN_VIEW_AZIMUTH_ANGLE": view_extension.azimuth = float(attribute.Values.Value.cdata) -def process_scientific(item, granule): +def process_scientific(item: pystac.Item, granule: untangle.Element) -> None: """Function checks the attribute value in STAC item and the granule. Args: @@ -239,7 +241,7 @@ def process_scientific(item, granule): scientific_extension.doi = attribute.Values.Value.cdata -def cmr_to_item(hls_vi_metadata, endpoint, version): +def cmr_to_item(hls_vi_metadata: str, endpoint: str, version: str) -> Mapping[str, Any]: """Function creates a pystac item from the CMR XML file provided as an input Args: @@ -280,10 +282,13 @@ def cmr_to_item(hls_vi_metadata, endpoint, version): process_projection(item, granule, index_file) process_view_geometry(item, granule) process_scientific(item, granule) - return item.to_dict() + return item.to_dict() # type: ignore -def create_item(hls_vi_metadata, out_json, endpoint, version): + +def create_item( + hls_vi_metadata: str, out_json: str, endpoint: str, version: str +) -> None: """Function acts as an endpoint to create a STAC item for the HLS_VI granule. Args: @@ -298,7 +303,7 @@ def create_item(hls_vi_metadata, out_json, endpoint, version): json.dump(item, outfile) -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( "--cmr_xml", diff --git a/hls_vi/py.typed b/hls_vi/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/hls_vi/schema/Granule.xsd b/hls_vi/schema/Granule.xsd new file mode 100644 index 0000000..ddb5b36 --- /dev/null +++ b/hls_vi/schema/Granule.xsd @@ -0,0 +1,1997 @@ + + + + + + The main element that holds all granules + being ingested. + + + + + + + + + + + + + + Name of data center where this metadata + file was originated. + + + + + + + + + + + The granules to insert or replace. A + replace will be performed automatically if a granule + already exists. + + + + + Partial metadata adds to be applied to + granules. The metadata will be updated if the referenced + field already exists. + + + + + Partial metadata deletes to be applied + to granules + + + + + Granules that should be immediately + deleted. To set a future delete time, use a partial + metadata update. + + + + + Granules to be verified. Each granule + supplied will be matched with the current instance in + ECHO in order to find any metadata mismatches. The + supplied granules will then be inserted or + replaced. + + + + + Inventories of granules to be checked + for existence in ECHO. Supplying zero inventories + indicates that existence checking should be + skipped. + + + + + + + List of granules + + + + + + + + + Stores the basic descriptive characteristics + associated with a granule. + Adopted from Release B Science Data Processing + Segment (SDPS) for the ECS Project. Refer to 311-CD-008-001 + May 15, 1996. Additional attributes been added for book + keeping information. + + + + + + + The date/time this granule was entered + data provider's database. + + + + + The date/time that data provider last + updated the granule info on data provider's + database. + + + + + The date/time that data provider + deleted the granule from data provider's + database. + + + + + This entity holds the reference + information from a granule to a collection. Granule + referencing the collection either by collection short + name and collection version or by collection dataset + ID. + + + + + The numerical value indicates the type + of the restriction that applies on the granule for data + access. + + + + + The restriction comment applied on the + granule for data access. This data could be defined for + each granule. If there is no restriction comment given + for the granule, the restriction comment of its primary + collection will be used if applicable. + + + + + + + + + + + This entity stores basic descriptive + characteristics associated with a + granule. + + + + + This entity stores basic descriptive + characteristics related to the Program Generation + Executable associated with a granule. + + + + + This entity contains records, which + describe the basis of the time system used for a specific + collection. + + + + + Holds granule spatial domain + information. + + + + + This entity is used to store the + characteristics of the orbit calculated spatial domain to + include the model name, orbit number, start and stop + orbit number, equator crossing date and time, and equator + crossing longitude. + + + + + Names of the geophysical parameter + expressed in the data as well as associated quality flags + and quality status. + + + + + References to the relevant platforms + associated with the acquisition of the + granule. + + + + + References to attributes describing the + scientific endeavor(s) to which the collection is + associated. Scientific endeavors include campaigns, + projects, interdisciplinary science investigations, + missions, field experiments, etc. + + + + + An additional attribute for the + granule. The attribute name and type must exist in the + parent collection for this granule. Additional attributes + are the same as PSAs. + + + + + This entity contains the identification + of the input granules for a specific + granule. + + + + + This entity stores the two dimensional + coordinate system information for the granule. The two + dimensional coordinate system information is an + alternative way to express granule's spatial coverage + based on a certain two dimensional coordinate system + defined by the providers. + + + + + The price of the granule data when + order. + + + + + This entity stores the online URL(s) + for the collections if there is any. Those URLs either + provide the site where the user can obtain collection + data or give the further instruction of obtaining the + collection data. + + + + + This entity holds all types of online + URL associated with the granule such as guide document or + ordering site etc. + + + + + Indication of whether the granule is + orderable. + + + + + The file format of the raw data (such + as HDF) for this granule. + + + + + + + + + + + +

Indication of whether the granule is visible or not. + Visibility is a basic access control mechanism that + bypasses all ACL rules. If a granule is not visible, + only users with the owning provider role will be able + to see the item. All other users will not see the item + no matter what ACL permissions are in place.

+

If group based permissions are needed, use the + restriction flag field instead of visibility. Normally + visibility is used when an item is first ingested. + Making the item not visible allows the provider time to + install any ACL rules or order options required before + users will see the item. Visibility is more commonly + set at the collection level than the granule level. If + a collection is not visible, none of the granules in + the collection will be visible.

+ + + + + + A percentage value indicating how much + of the area of a granule (the ECS data unit) has been + obscured by clouds. It is worth noting that there are + many different measures of cloud cover within the ECS + data holdings and that the cloud cover parameter that is + represented in the DataPool is specific to Data + Set. + + + + + + + + The list of browse images associated + with this granule. + + + + + The list of browse image urls + associated with this granule. + + + + + + + + This entity stores the basic descriptive + characteristics associated with a granule. + + + + + The size in Bytes of the volume of data contained in the granule. + Bytes are defined as eight bits. Please use this element instead of or inclusive + with the SizeMBDataGranule element. The issue with SizeMBDataGranule is that end + users don't know if the size was calculated by using 1000xE2 Bytes (MegaBytes) or + 1024xE2 Bytes (mebibytes) and therefore there is no systematic way to know the + actual size of a granule by using the granule metadata record. + + + + + The size attribute will indicate the volume of data contained in + the granule. Please use DataGranuleSizeInBytes for the size of the granule. The issue + with this element is that end users don't know if the size was calculated by using + 1000xE2 Bytes (MegaBytes) or 1024xE2 Bytes (mebibytes) and therefore there is no + systematic way to know the actual size of a granule by using the granule metadata + record. + + + + + Allows the provider to provide the checksum value for the file. + + + + + Granule level, stating what + reprocessing may be performed on this + granule. + + + + + + + + + + + Granule level, stating what + reprocessing has been performed on this + granule. + + + + + + + + + + + Unique identifier for locally produced + granule that ECS ingests and is required to + capture. + + + + + + + + + + + This attribute is used to identify if a + granule was collected during the day, night (between + sunset and sunrise) or both. + + + + + The date and time a specific granule + was produced by a PGE. + + + + + Granule version identifier for PGE + defined granule defined by the + producer. + + + + + + + + + + + If a granule contains more than 1 data file, + the rest of the files can be described in this element. + + + + + + + Allows the provider to describe other granule files if more than 1 exists. + + + + + This field describes the name of the actual file. + + + + + + + + + + + The size in Bytes of the volume of data contained in the granule. + Bytes are defined as eight bits. + + + + + The file format of the raw data (such + as HDF) for this granule. + + + + + + + + + + + The file mime-type of the raw data (such + as application/x-hdf) for this granule. + + + + + + + + + + + Allows the provider to provide the checksum value for the file. + + + + + + + Allows the provider to provide a checksum value and checksum algorithm + name to allow the user to calculate the checksum. If Checksum is used both the value and the algorithm are required. + + + + + Describes the checksum value for a file. + + + + + + + + + + + The algorithm name by which the checksum was calulated. This allows + the user to re-calculate the checksum to verify the integrity of the downloaded + data. + + + + + + + + + + + + + + + + + + + + + + + + This entity stores the basic descriptive + characteristics associated with a granule. + + + + + Name of product generation + executive. + + + + + + + + + + + Version of the Delivered Algorithm + Package that applied when produce the + granule. + + + + + + + + + + + + + This is a pseudo entity to hold granule + spatial domain information. + + + + + This entity stores information used at + the granule level to describe the labeling of granules + with compounded time/space text values and which are + subsequently used to define more phenomenological-based + granules, thus the locality type and description are + contained. + + + + + This entity contains the domain value + and type for the granule's vertical spatial + domain. + + + + + This is a pseudo entity to hold granule + horizontal spatial domain information. + + + + + + + This entity stores information used at the + granule level to describe the labeling of granules with + compounded time/space text values and which are subsequently + used to define more phenomenological-based granules, thus the + locality type and description are + contained. + + + + + Provides name which spatial/temporal + entity is known. This could change on a granule by + granule basis. This attribute is paralleled by the + AggregationType which applies at the collection level + although locality has a more restricted usage. Several + locality measures could be included in each + granule. + + + + + + + + + + + + + This is a pseudo entity to hold granule + horizontal spatial domain information. + + + + + The appropriate numeric or alpha code + used to identify the various zones in this grid + coordinate system. + + + + + + + + + + + + + This entity stores orbital coverage + information of the granule. This coverage is an + alternative way of express granule spatial coverage. + This information supports orbital backtrack search + apply on granule. + + + + + + + + This entity holds the geometry representing + the spatial coverage information of a + granule. + + + + + The horizontal spatial coverage of a + point. + + + + + This entity holds the horizontal + spatial coverage of a line. ECHO stores horizontal + spatial coverage bounding rectangle type information + using oracle spatial type expression as a four points + polygon. + + + + + A GPolygon represents an area on the + earth represented by a main boundary with optional + boundaries excluded regions from the main + boundary. + + + + + This entity holds the horizontal + spatial coverage of a line. A line area contains at least + two points expressed with (PointLongitude, + PointLatitude). A Line entity forms with at least two + Point entity. ECHO stores horizontal spatial coverage + Line type information using oracle spatial type + expression. + + + + + + + This entity stores orbital coverage + information of the granule. This coverage is an alternative + way of express granule spatial coverage. This information + supports orbital backtrack search apply on + granule. + + + + + Equatorial crossing on the ascending + pass in decimal degrees longitude. The convention we've + been using is it's the first included ascending crossing + if one is included, and the prior ascending crossing if + none is included (e.g. descending half + orbits). + + + + + Granule's starting + latitude. + + + + + Ascending or descending. Valid input: + "A" or "D". + + + + + Granule's ending + latitude. + + + + + Ascending or descending. Valid input: + "A" or "D". + + + + + Granule's horizontal spatial coverage + of a point. + + + + + + + This entity is used to store the + characteristics of the orbit calculated spatial domain to + include the model name, orbit number, start and stop orbit + number, equator crossing date and time, and equator crossing + longitude. + + + + + + + + This entity is used to store the + characteristics of the orbit calculated spatial domain to + include the model name, orbit number, start and stop orbit + number, equator crossing date and time, and equator crossing + longitude. + + + + + The reference to the orbital model to + be used to calculate the geo-location of this data in + order to determine global spatial + extent. + + + + + + + + + + + The orbit number to be used in + calculating the spatial extent of this + data. + + + + + Orbit number at start of data + granule. + + + + + Orbit number at end of data + granule. + + + + + This attribute represents the + terrestrial longitude of the descending equator + crossing. + + + + + This attribute represents the date and + time of the descending equator + crossing. + + + + + + + This entity contains the name of the + geophysical parameter expressed in the data as well as + associated quality flags and quality status. The quality + status contains measures of quality for the granule. The + parameters used to set these measures are not preset and will + be determined by the data producer. Each set of measures can + occur many times either for the granule as a whole or for + individual parameters. The quality flags contain the science, + operational and automatic quality flags which indicate the + overall quality assurance levels of specific parameter values + within a granule. + + + + + + + + This entity contains the name of the + geophysical parameter expressed in the data as well as + associated quality flags and quality status. The quality + status contains measures of quality for the granule. The + parameters used to set these measures are not preset and will + be determined by the data producer. Each set of measures can + occur many times either for the granule as a whole or for + individual parameters. The quality flags contain the science, + operational and automatic quality flags which indicate the + overall quality assurance levels of specific parameter values + within a granule. + + + + + The measured science parameter + expressed in the data granule. + + + + + + + + + + + The name of the geophysical parameter + expressed in the data as well as associated quality flags + and quality status. + + + + + The name of the geophysical parameter + expressed in the data as well as associated quality flags + and quality status. + + + + + + + This entity contains the name of the + geophysical parameter expressed in the data as well as + associated quality flags and quality status. The quality + status contains measures of quality for the granule. The + parameters used to set these measures are not preset and will + be determined by the data producer. Each set of measures can + occur many times either for the granule as a whole or for + individual parameters. The quality flags contain the science, + operational and automatic quality flags which indicate the + overall quality assurance levels of specific parameter values + within a granule. + + + + + Granule level % missing data. This + attribute can be repeated for individual parameters + within a granule. + + + + + Granule level % out of bounds data. + This attribute can be repeated for individual parameters + within a granule. + + + + + Granule level % interpolated data. This + attribute can be repeated for individual parameters + within a granule. + + + + + This attribute is used to characterize + the cloud cover amount of a granule. This attribute may + be repeated for individual parameters within a granule. + (Note - there may be more than one way to define a cloud + or it's effects within a product containing several + parameters; i.e. this attribute may be parameter + specific) + + + + + + + This entity contains the name of the + geophysical parameter expressed in the data as well as + associated quality flags and quality status. The quality + status contains measures of quality for the granule. The + parameters used to set these measures are not preset and will + be determined by the data producer. Each set of measures can + occur many times either for the granule as a whole or for + individual parameters. The quality flags contain the science, + operational and automatic quality flags which indicate the + overall quality assurance levels of specific parameter values + within a granule. + + + + + The granule level flag applying + generally to the granule and specifically to parameters + the granule level. When applied to parameter, the flag + refers to the quality of that parameter for the granule + (as applicable). The parameters determining whether the + flag is set are defined by the developer and documented + in the Quality Flag Explanation. + + + + + + + + + + + A text explanation of the criteria used + to set automatic quality flag; including thresholds or + other criteria. + + + + + + + + + + + The granule level flag applying both + generally to a granule and specifically to parameters at + the granule level. When applied to parameter, the flag + refers to the quality of that parameter for the granule + (as applicable). The parameters determining whether the + flag is set are defined by the developers and documented + in the QualityFlagExplanation. + + + + + + + + + + + A text explanation of the criteria used + to set operational quality flag; including thresholds or + other criteria. + + + + + + + + + + + Granule level flag applying to a + granule, and specifically to parameters. When applied to + parameter, the flag refers to the quality of that + parameter for the granule (as applicable). The parameters + determining whether the flag is set are defined by the + developers and documented in the Quality Flag + Explanation. + + + + + + + + + + + A text explanation of the criteria used + to set science quality flag; including thresholds or + other criteria. + + + + + + + + + + + + + A reference to a platform in the parent + collection that is associated with the acquisition of the + granule. The platform must exist in the parent collection. + For example, Platform types may include (but are not limited + to): ADEOS-II, AEM-2, AM-1, Aqua, Aura, BALLOONS, BUOYS, + C-130, DEM, DMSP-F1, ERS-1, GOES-10, LANDSAT-1, METEOSAT-2, + NIMBUS-2, NOAA-6, TERRA, TRMM, etc. + + + + + The unique platform name. (e.g. + GOES-8). + + + + + + + + + + + References to the devices in the parent + collection that were used to measure or record data, + including direct human observation. + + + + + + + + + + + + This entity is used to reference + characteristics defined in the parent + collection. + + + + + The name of the characteristic + attribute. + + + + + + + + + + + The value of the Characteristic + attribute. + + + + + + + + + + + + + A reference to the device in the parent + collection that was used to measure or record data, including + direct human observation. In cases where instruments have a + single sensor or the instrument and sensor are used + synonymously (e.g. AVHRR) the both Instrument and Sensor + should be recorded. The Sensor information is represented by + child entities. The instrument reference may contain granule + specific characteristics and operation modes. These + characteristics and modes are not checked against the + referenced instrument. + + + + + The unique identifier of an + instrument. + + + + + + + + + + + This entity is used to define item + additional attributes (unprocessed, custom + data). + + + + + References to sensory subcomponents in + the parent collection's instrument used by various + sources in the granule. A sensor reference may contain + characteristics specific to the + granule. + + + + + This entity identifies the instrument's + operational modes for a specific collection associated + with the channel, wavelength, and FOV (e.g., launch, + survival, initialization, safe, diagnostic, standby, + crosstrack, biaxial, solar + calibration). + + + + + + + A reference to a sensory subcomponent in + the parent collection's instrument used by various sources in + this granule. A sensor reference may contain characteristics + specific to the granule. These characteristics are not + checked against the referenced sensor. + + + + + A sensor is a defined sensory + sub-component of an instrument. (e.g. + InstrumentShortName=ASTER, NumberofSensors= 3, + SensorShortName= SWIR, SensorShortName= TIR, + SensorShortName= VNIR) In cases where the Instrument has + a single Sensor or the Instrument and Sensor are + synonymous then both attributes should be populated. + (e.g. AVHRR). Sensors cannot exist without + Instruments. + + + + + + + + + + + + + + This entity contains attributes describing + the scientific endeavor(s) to which the collection is + associated. Scientific endeavors include campaigns, projects, + interdisciplinary science investigations, missions, field + experiments, etc. Seems to be a reference to one that exists + at collection level. + + + + + The unique identifier by which a + campaign/project/experiment is known. The campain/project + is the scientific endeavor associated with the + acquisition of the collection. Collections may be + associated with multiple campaigns. + + + + + + + + + + + + + This entity stores the Product Specific + Attributes with value a granule + associates. + + + + + + + + A reference to an additional attribute in + the parent collection. The attribute reference may contain a + granule specific value that will override the value in the + parent collection for this granule. An attribute with the + same name must exist in the parent + collection. + + + + + Additional attribute + name. + + + + + + + + + + + The ordered list of values of the + additioanl attribute for this granule. The values will be + kept in the order which they appear. + + + + + + + This entity contains the identification of + the input granule(s) for a specific + granule. + + + + + + + + + + + + + + + + This entity stores the two dimensional coordinate + system information for the granule. The two dimensional + coordinate system information is an alternative way to + express granule's spatial coverage based on a certain two + dimensional coordinate system defined by the + providers. + The following discussion on Landsat's PATH/ROW is + an example of grid information. The Path is the + longitudinal center line of a Landsat scene corresponding + to the center of an orbital track that represented by grid + Y lines. + The Row is the Latitudinal center line of a Landsat + scene that corresponding to grid X lines. The indication of + the two dimensional coordinate system type is "WRS-2". The + two dimensional coordinate system information can be used + to designate a geographic search for a nominal scene + center. + + + + + + The horizontal starting line of the + nominal scene center based on the coordinate system. This + value could be either entered directly as part of + metadata or extracted from PSA value based on a + pre-defined PSA name. + + + + + The horizontal ending line of the + nominal scene center based on the coordinate system. This + value could be either entered directly as part of + metadata or extracted from PSA value based on a + pre-defined PSA name. + + + + + The vertical starting line of the + nominal scene center based on the coordinate system. This + value could be either entered directly as part of + metadata or extracted from PSA value based on a + pre-defined PSA name. + + + + + The vertical ending line of the nominal + scene center based on the coordinate system. This value + could be either entered directly as part of metadata or + extracted from PSA value based on a pre-defined PSA + name. + + + + + The type of the two dimensional + coordinate system defined by the providers such as + "WRS-2" for Landsat scene. + + + + + + + + + + + + + + + + + + Granule that should be immediately + deleted. + + + + + + + + + + + + + + + + + + + + + + + + + + + + Orbit start and end direction. A for + ascending orbit and D for descending. + + + + + + + + + A list of fields that can be updated on a + granule. + + + + + A single field to be + updated. + + + + + + + A field to be added to a granule. Not all + fields of a granule support partial updates. If an update is + required for a non-supported field, perform a complete + granule replacement rather than a partial metadata + update. + + + + + + + Updates a link to a browse image. + The browse image's ProviderBrowseId must be provided + to identify it. + + + + + Updates a link to a browse image. + The browse image's ProviderBrowseUrl must be provided + to identify it. + + + + + + Updates an online access URL. The URL + will be used as the key to identify the online access + URL to be added or replaced. + + + + + Updates an online access URL. The URL + will be used as the key to identify the online resource + to be added or replaced. + + + + + Updates a measured parameter. The + parameter name will be used as the key to identify the + measured parameter to be added or replaced.The entire + measured parameter will be added or + replaced. + + + + + Updates an additional attribute + reference. The attribute name will be used as the key + to identify the additional attribute reference to be + added or replaced.The entire additional attribute + reference will be added or replaced. + + + + + Updates the temporal information for + the granule. The entire temporal information will be + replaced during an add. Refer to the Granule + documentation for more information. + + + + + The day/night flag to update. Refer + to the Granule documentation for more + information. + + + + + Updates the delete time of the + granule. Refer to the Granule documentation for more + information. + + + + + Updates the cloud cover of the + granule. Refer to the Granule documentation for more + information. + + + + + Updates the restriction flag of the + granule. Refer to the Granule documentation for more + information. + + + + + Updates the visible flag of the + granule. Refer to the granule documentation for more + information. + + + + + Updates the spatial coverage for the + granule. The entire spatial coverage will be replaced + during an add. Refer to the Granule documentation for + more information. + + + + + + The fields that should be targeted by + this update. Only the fields in this list will be updated + and the other fields in the added element will be + ignored. This element is only for backward compatibility + and should not be used for new development. This field + will be removed in the future. + + + + + Indicates the previous value held by an + updated field. This will only be set when updating + primary key values explicitly using the + UpdateTargetFields. This element is only for backward + compatibility and should not be used for new development. + This field will be removed in the + future. + + + + + + + + + + + + This entity contains records, which + describe the basis of the time system used for a specific + collection. + + + + + This entity stores the start and end + date/time of a collection. + + + + + This entity stores the time of day and + calendar date for an ECS collection. This information + provides a means of encoding a single date and time for a + granule occurring at that time or during the period + covered by the time (e.g. one-day for a single date + excluding the time within the day). + + + + + + + A list of fields to be deleted from a + granule. + + + + + + + + The field that should be deleted. Some + fields require a value to indicate the key of the field to + the correct instance can be identified. + + + + + Deletes a link to the browse image with + the specified ProviderBrowseId. + + + + + Deletes an online access URL with the + given value. The entire URL element will be deleted. The + value of this element should be the + URL. + + + + + + + + + + + Deletes an online resource URL with the + given value. The entire URL element will be deleted. The + value of this element should be the + URL. + + + + + + + + + + + Deletes the measured parameter. The + value of this element should be the measured parameter + name. The entire measured parameter will be + deleted. + + + + + + + + + + + Deletes an additional attribute + reference from the granule. The value of this element + should be the additional attribute + name. + + + + + + + + + + + Deletes the temporal information from + the granule. + + + + + Deletes the day/night flag from the + granule. + + + + + Deletes the remove time from the + granule. The granule will no longer be automatically + removed. + + + + + Deletes the cloud cover information + from the granule. + + + + + Deletes the restriction flag from the + granule. + + + + + Indicates that all online access URLs + should be deleted from the granules. + + + + + Indicates that all online resources + should be deleted from the granules. + + + + + Indicates that all additional + attributes should be deleted from the + granules. + + + + + Indicates that all links to browse + images should be deleted from the + granules. + + + + + Deletes the spatial coverage from the + granule. + + + + + + + A target field for an update that should + only affect a single field of the metadata. This type is only + for backward compatibility and should not be used for new + development. + + + + + + + + + + + + + + + + + + + + + + A list of single fields that are the target + of the update. + + + + + A single field that is the target of + the update. + + + + + + + A partial add to be applied to a granule. + The partial add may contain one or more fields. If the field + being updated contains a unique key and the field already + exists, it will be updated or replaced, otherwise it will be + added. Refer to the individual field documentation for + details. + + + + + The list of granules to + update. + + + + + The fields that should be added to the + referenced granule. If the field already exists in the + collection, it will be updated or + replaced. + + + + + + + A partial delete to be applied to a + granule. The partial delete may contain one or more fields. + If a delete is attempted the field will be set to a default + value. Refer to the individual field documentation for + details. + + + + + The list of granules to + update. + + + + + The fields that should be deleted from + the granule. + + + + + + + A list of granule partial metadata + adds. + + + + + A single granule partial metadata + add. + + + + + + + A list of granule partial metadata + deletes. + + + + + A single granule partial metadata + delete. + + + + + + + + + + A reference to a granule to be + updated + + + + + The date this change occured in the + provider's database. If this is provided we will validate + that this date is greater than or equal to the date in + the granule. The granule's last update data will be + updated with this value. + + + + + The granule UR of the granule to which + the update will be applied. + + + + + + + A list of granule update + targets. + + + + + A single granule update + target + + + + + + + Possible values for the day night flag. If + no day night flag information is available, the unspecified + value should be used. + + + + + + + + + + + + + + + + + + + + + + + List of granule + inventories. + + + + + + + + An inventory of granules for a given + collection. An empty GranuleURs element represents an + inventory with no granules. + + + + + The parent collection for this granule + inventory. + + + + + The references to granules that are + expected to exist for the given parent collection in + ECHO. + + + + + diff --git a/hls_vi/schema/MetadataCommon.xsd b/hls_vi/schema/MetadataCommon.xsd new file mode 100644 index 0000000..4ef8338 --- /dev/null +++ b/hls_vi/schema/MetadataCommon.xsd @@ -0,0 +1,858 @@ + + + + + + + + Indicates a cartesian coordinate + system. + + + + + Indicates a geodetic coordinate + system. + + + + + + + This entity contains the indication of + spatial coordinate system for all the granules in the + collection. + + + + + + + + + + + + + + + + + + + + + + + + + + + This entity holds the horizontal spatial + coverage of a line. A line area contains at lease two points + expressed with (PointLongitude, PointLatitude). A Line entity + forms with at least two Point entity. ECHO stores horizontal + spatial coverage Line type information using oracle spatial + type expression. + + + + + + + + + This entity holds the horizontal spatial + coverage of a line. ECHO stores horizontal spatial coverage + bounding rectangle type information using oracle spatial type + expression as a four points polygon. TODO possible merge with + collection but collection does not have + CenterPoint + + + + + Western-most coordinate of the limit of + coverage expressed in longitude + (degree). + + + + + Northern-most coordinate of the limit + of coverage expressed in latitude + (degree). + + + + + Eastern-most coordinate of the limit of + coverage expressed in longitude + (degree). + + + + + Southern-most coordinate of the limit + of coverage expressed in latitude + (degree). + + + + + The center point of the bounding + rectangle. This value is ignored for + collections. + + + + + + + A GPolygon represents an area on the earth + represented by a main boundary with optional boundaries + excluded regions from the main boundary. + + + + + The boundary representing the outer + ring of the GPolygon. + + + + + Contains the excluded boundaries from + the GPolygon. + + + + + The center point of the polygon. This + value is ignored for collections. + + + + + + + This entity holds the horizontal spatial + coverage of a point. + + + + + The longitude value of a spatially + referenced pointer in degree. + + + + + The latitude value of a spatially + referenced pointer in degree. + + + + + + + Contains a list of boundaries to exclude + from some area. + + + + + + + + + + + + A boundary is set of points connected by + straight lines representing a polygon on the earth. It takes + a minimum of three points to make a boundary. Points must be + specified in clockwise order. + + + + + + + + This entity contains the domain value and + type for the granule's vertical spatial + domain. + + + + + + + + This entity contains the domain value and + type for the granule's vertical spatial domain. The reference + frame or system from which altitude or depths are measured. + The term 'altitude' is used instead of the common term + 'elevation' to conform to the terminology in Federal + Information Processing Standards 70-1 and 173. The + information contains the datum name, distance units and + encoding method, which provide the definition for the + system. + + + + + This attribute describes the type of + the area of vertical space covered by the + locality. + + + + + + + + + + + This attribute describes the extent of + the area of vertical space covered by the granule. Must + be accompanied by an Altitude Encoding Method + description. The datatype for this attribute is the value + of the attribute VerticalSpatialDomainType. The unit for + this attribute is the value of either DepthDistanceUnits + or AltitudeDistanceUnits. + + + + + + + + + + + + + + This entity stores the start and end + date/time of a collection. + + + + + The time when the temporal coverage + period being described began. + + + + + The time of the temporal coverage + period being described ended + + + + + + + + This entity stores the online URL(s) for + the granule if there is any. Those URL either provides the + site that user can obtain granule data or give the further + instruction of obtaining the granule data. + + + + + + + + This entity stores the online URL(s) for + the granule if there is any. Those URL either provides the + site that user can obtain granule data or give the further + instruction of obtaining the granule data. + + + + + If the granule data is available + online, then the URL will be provided and recorded + here. + + + + + + + + + + + Description about the URL or any + associated information. + + + + + + + + + + + + + + + + + + + + + + + + This entity records the documentation + information of the collection including documentation type + and documentation URL where apply etc. + + + + + The URL of the + resource. + + + + + + + + + + + Comment about the online + resource. + + + + + + + + + + The type of the resource such as + 'Collection Guide' or 'Campaign Guide' + etc. + + + + + + + + + + + The mime type of the online + resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A reference to a collection using one of + the unique keys in the collection. A collection can be + referenced using a combination of the short name and version + id or the dataset id. Setting only one of short name and + version id without the other is not a valid collection + reference. + + + + + + The short name of the collection. The + short name can be used as a key for the collection when + combined with the version id. + + + + + The version id of the collection. The + version id can be used as a unique key for the + collection when combined with the short + name. + + + + + + + + + + + + The data set id of the + collection. + + + + + + + + + + + + + + + + + + The element should contain no children. In + most cases, this is used when the presence of the element has + meaning, however the element itself does not required + data. + + + + + + + The unique identifier for a browse image + from the provider. + + + + + + + + + The unique id for a browse image from the + provider. + + + + + + + + + The unique url for a browse image from the + provider. + + + + + The URL of the + browse. + + + + + + + + + + + The size of the browse + file. + + + + + The description of the browse + url. + + + + + + + + + + + + The mime type of the browse record. + + + + + + + + + + + + + + The Universal Reference ID of the granule + referred by the data provider. This ID is unique per data + provider. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A list of references to + collections. + + + + + A single reference to a + collection. + + + + + + + + A list of references to + granules. + + + + + A single reference to a + granule. + + + + + + + + A list of references to browse + images. + + + + + A single reference to a browse + image. + + + + + + + A list of urls to browse + images. + + + + + A single url to a browse + image. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This element stores the DOI (Digital Object Identifier) that identifies the collection. Note: The values should start with the directory indicator which in ESDIS' case is 10. If the DOI was registered through ESDIS, the beginning of the string should be 10.5067. The DOI URL is not stored here; it should be stored as an OnlineResource. The DOI organization that is responsible for creating the DOI is described in the Authority element. For ESDIS records the value of https://doi.org/ should be used. While this element is not required, NASA metadata providers are strongly encouraged to include DOI and DOI Authority for their collections. For those that want to specify that a DOI is not applicable for their record use the second option. + + + + + + + This element stores the DOI (Digital Object Identifier) that identifies the collection. Note: The values should start with the directory indicator which in ESDIS' case is 10. If the DOI was registered through ESDIS, the beginning of the string should be 10.5067. The DOI URL is not stored here; it should be stored as an OnlineResource. + + + + + + + + + + + The DOI organization that is responsible for creating the DOI is described in the Authority element. For ESDIS records the value of https://doi.org/ should be used. + + + + + + + + + + + + + + This element stores the fact that the DOI (Digital Object Identifier) is not applicable. + + + + + + + + + + + This element describes the reason the DOI is not applicable. + + + + + + + + + + + + + + + + The DOI Previous Version provides a record of every time that a + DOI was updated and links to the DOI landing page for the + previous version. + + + + + + + + + + + + + + + + + + + + This element stores DOIs that are associated with the collection + such as from campaigns and other related sources. Note: The values + should start with the directory indicator which in ESDIS' case is 10. + If the DOI was registered through ESDIS, the beginning of the string + should be 10.5067. The DOI URL is not stored here; it should be + stored as a RelatedURL. The DOI organization that is responsible + for creating the DOI is described in the Authority element. For + ESDIS records the value of https://doi.org/ should be used. + + + + + + This element stores the DOI (Digital Object Identifier) that + identifies an associated collection. Note: The values should + start with the directory indicator which in ESDIS' case is 10. + If the DOI was registered through ESDIS, the beginning of the + string should be 10.5067. The DOI URL is not stored here; it + should be stored as an OnlineResource. + + + + + + + + + + + + The title of the DOI landing page. The title describes the + DOI object to a user, so they don't have to look it up themselves to + understand the association. + + + + + + + + + + + + The DOI organization that is responsible for creating the + associated DOI is described in the Authority element. + For ESDIS records the value of https://doi.org/ should be used. + + + + + + + + + + + + + The Associated DOI Type element is used to inform users + about what kind or type of DOI association is related to + the element value. The "DescriptionOfOtherType" describes + what is the Other value populated in the type field of + the Associated DOI. + + + + + + + + + + + + + + + + + The "DescriptionOfOtherType" describes what is the Other + value populated in the type field of the Associated DOI. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..979d995 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,11 @@ +[mypy] +strict = True +pretty = True +show_error_codes = True +show_error_context = True +show_column_numbers = True +; mypy has trouble with some of the untangle.Element attribute references +disable_error_code = attr-defined + +[mypy-geojson,lxml.*,numpy,pystac.*,rasterio.*,shapely.*] +ignore_missing_imports = True diff --git a/setup.py b/setup.py index cbd1d94..44f0475 100644 --- a/setup.py +++ b/setup.py @@ -6,15 +6,26 @@ packages=["hls_vi"], install_requires=[ "dataclasses", + "geojson", + "importlib_resources", + "lxml", "numpy~=1.19.0", + "pystac[validation]==1.0.0rc2", "rasterio", + "shapely", "typing-extensions", - "geojson", - "pystac[validation]==1.0.0rc2", "untangle", - "shapely", ], - extras_require={"test": ["black[jupyter]==21.12b0", "flake8", "pytest"]}, + extras_require={ + "test": [ + "black[jupyter]==21.12b0", + "flake8", + "mypy", + "pytest", + "types-dataclasses", + "types-untangle", + ] + }, entry_points={ "console_scripts": [ "vi_generate_indices=hls_vi.generate_indices:main", diff --git a/tests/test_vi.py b/tests/test_vi.py index 175dca2..c821a35 100644 --- a/tests/test_vi.py +++ b/tests/test_vi.py @@ -5,7 +5,6 @@ import contextlib import io import json -import re import pytest import rasterio @@ -87,24 +86,19 @@ def remove_element(root: ET.Element, path: str) -> ET.Element: return child -def remove_datetime_elements( - tree: ET.ElementTree, -) -> Tuple[ET.ElementTree, Tuple[ET.Element, ...]]: +def remove_datetime_elements(tree: ET.ElementTree) -> ET.ElementTree: root = tree.getroot() - return ( - tree, - tuple( - remove_element(root, path) - for path in ( - "./InsertTime", - "./LastUpdate", - "./DataGranule/ProductionDateTime", - "./Temporal/RangeDateTime/BeginningDateTime", - "./Temporal/RangeDateTime/EndingDateTime", - ) - ), - ) + for path in ( + "./InsertTime", + "./LastUpdate", + "./DataGranule/ProductionDateTime", + "./Temporal/RangeDateTime/BeginningDateTime", + "./Temporal/RangeDateTime/EndingDateTime", + ): + remove_element(root, path) + + return tree def assert_indices_equal(granule: Granule, actual_dir: Path, expected_dir: Path): @@ -119,16 +113,6 @@ def assert_indices_equal(granule: Granule, actual_dir: Path, expected_dir: Path) assert_tifs_equal(granule, actual_tif_path, expected_tif_path) -def is_valid_datetime(e: ET.Element) -> bool: - # The CMR accepts ISO 8601 datetime values, optionally with fractional seconds - # with 1 to 9 decimal digits. We are using a regex match because Python's - # strptime function supports only exactly 6 decimal digits, but some of the tif - # tag values include more than 6 decimal places. - return bool( - re.match(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([.]\d{1,9})?Z", str(e.text)) - ) - - @pytest.mark.parametrize( argnames="input_dir,id_str", argvalues=[ @@ -175,11 +159,8 @@ def test_generate_cmr_metadata(input_dir, output_dir): try: generate_metadata(input_dir=input_path, output_dir=output_path) - actual_metadata_tree, dt_elements = remove_datetime_elements( - ET.parse(actual_metadata_path) - ) - assert all(map(is_valid_datetime, dt_elements)) - expected_metadata_tree, _ = remove_datetime_elements( + actual_metadata_tree = remove_datetime_elements(ET.parse(actual_metadata_path)) + expected_metadata_tree = remove_datetime_elements( ET.parse(expected_metadata_path) ) diff --git a/tox.ini b/tox.ini index 75de963..6d0bffd 100644 --- a/tox.ini +++ b/tox.ini @@ -20,4 +20,5 @@ extras = test commands = flake8 + mypy hls_vi pytest -vv