From 38176313355f5f5c73d59e70767ca5ed899c44d3 Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 20 Dec 2024 12:43:01 -0500 Subject: [PATCH 1/8] Include Input_HLS_GranuleUR additional attribute --- hls_vi/generate_metadata.py | 9 +++++++++ hls_vi/schema/Granule.xsd | 1 + .../HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml | 6 ++++++ .../HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml | 6 ++++++ 4 files changed, 22 insertions(+) diff --git a/hls_vi/generate_metadata.py b/hls_vi/generate_metadata.py index 1bd6681..cf9f25c 100644 --- a/hls_vi/generate_metadata.py +++ b/hls_vi/generate_metadata.py @@ -92,7 +92,13 @@ def generate_metadata(input_dir: Path, output_dir: Path) -> None: processing_time = tags["HLS_VI_PROCESSING_TIME"] granule_ur = tree.find("GranuleUR") + input_granule_ur = granule_ur.text granule_ur.text = granule_ur.text.replace("HLS", "HLS-VI") + set_additional_attribute( + tree.find("AdditionalAttributes"), + "Input_HLS_GranuleUR", + input_granule_ur, + ) time_format = "%Y-%m-%dT%H:%M:%S.%fZ" formatted_date = datetime.now(timezone.utc).strftime(time_format) @@ -125,6 +131,9 @@ def generate_metadata(input_dir: Path, output_dir: Path) -> None: tree.find("DataFormat").text = "COG" + # ensure any added attributes are indented + ET.indent(tree) + with ( importlib_resources.files("hls_vi") / "schema" diff --git a/hls_vi/schema/Granule.xsd b/hls_vi/schema/Granule.xsd index ddb5b36..14e430f 100644 --- a/hls_vi/schema/Granule.xsd +++ b/hls_vi/schema/Granule.xsd @@ -14,6 +14,7 @@ xmlns:xs="http://www.w3.org/2001/XMLSchema"> + diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml index 01ba39b..28d6269 100644 --- a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml @@ -240,6 +240,12 @@ https://doi.org + + Input_HLS_GranuleUR + + HLS.L30.T06WVS.2024120T211159.v2.0 + + diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml index 9211d51..1ec103c 100644 --- a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml @@ -302,6 +302,12 @@ https://doi.org + + Input_HLS_GranuleUR + + HLS.S30.T13RCN.2024128T173909.v2.0 + + From a2f0ff4785620a9829b9b9cf319169b3295ef8ba Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 20 Dec 2024 12:47:53 -0500 Subject: [PATCH 2/8] Fix some typos found by typos_lsp --- hls_vi/generate_metadata.py | 2 +- hls_vi/schema/Granule.xsd | 2 +- tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml | 2 +- tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hls_vi/generate_metadata.py b/hls_vi/generate_metadata.py index cf9f25c..d75cf11 100644 --- a/hls_vi/generate_metadata.py +++ b/hls_vi/generate_metadata.py @@ -151,7 +151,7 @@ def generate_metadata(input_dir: Path, output_dir: Path) -> None: def normalize_additional_attributes(container: ElementBase) -> None: """Normalize additional attribute values. - On rare occassions, granule data is split and recombined upstream. When this + On rare occasions, granule data is split and recombined upstream. When this occurs, the associated metadata is also split and recombined, resulting in values for additional attributes that are created by joining the separate parts with the string `" + "`. diff --git a/hls_vi/schema/Granule.xsd b/hls_vi/schema/Granule.xsd index 14e430f..52e46e0 100644 --- a/hls_vi/schema/Granule.xsd +++ b/hls_vi/schema/Granule.xsd @@ -1299,7 +1299,7 @@ xmlns:xs="http://www.w3.org/2001/XMLSchema"> type="ListOfAdditionalAttributeValues"> The ordered list of values of the - additioanl attribute for this granule. The values will be + additional attribute for this granule. The values will be kept in the order which they appear. diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml index 28d6269..c4a2b7e 100644 --- a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml @@ -9,7 +9,7 @@ HLS-VI.L30.T06WVS.2024120T211159 DAY - UPDATE HLS Prodution DATETIME + UPDATE HLS Production DATETIME 2.0 diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml index 1ec103c..80e3062 100644 --- a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml @@ -9,7 +9,7 @@ HLS-VI.S30.T13RCN.2024128T173909 DAY - UPDATE HLS Prodution DATETIME + UPDATE HLS Production DATETIME 2.0 From d627f973e2e656bbd3913d16ab4ca0f10aecb334 Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 20 Dec 2024 17:13:30 -0500 Subject: [PATCH 3/8] Include links to Fmask from HLS granule in OnlineAccessURLs --- hls_vi/generate_metadata.py | 37 +++++++++++++++++++ ...-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml | 8 ++++ ...-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml | 8 ++++ 3 files changed, 53 insertions(+) diff --git a/hls_vi/generate_metadata.py b/hls_vi/generate_metadata.py index d75cf11..e5f9048 100644 --- a/hls_vi/generate_metadata.py +++ b/hls_vi/generate_metadata.py @@ -131,6 +131,11 @@ def generate_metadata(input_dir: Path, output_dir: Path) -> None: tree.find("DataFormat").text = "COG" + append_fmask_online_access_urls( + tree.find("OnlineAccessURLs"), + input_granule_ur, + ) + # ensure any added attributes are indented ET.indent(tree) @@ -202,6 +207,38 @@ def set_additional_attribute(attrs: ElementBase, name: str, value: str) -> None: attrs.append(attr) +def append_fmask_online_access_urls(access_urls: ElementBase, hls_granule_ur: str) -> None: + """Include links to Fmask layer from HLS granule in metadata + + This is intended to help users find the relevant Fmask band without + having to duplicate it into the HLS-VI product. See, + https://github.com/NASA-IMPACT/hls-vi/issues/47 + """ + prefix = ( + "HLSL30.020" if hls_granule_ur.startswith("HLS.L30") + else "HLSS30.020" + ) + + http_attr = Element("OnlineAccessURL", None, None) + http_attr_url = Element("URL", None, None) + http_attr_url.text = f"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/{prefix}/{hls_granule_ur}/{hls_granule_ur}.Fmask.tif" + http_attr_desc = Element("URLDescription", None, None) + http_attr_desc.text = f"Download Fmask quality layer {hls_granule_ur}.Fmask.tif" + http_attr.append(http_attr_url) + http_attr.append(http_attr_desc) + + s3_attr = Element("OnlineAccessURL", None, None) + s3_attr_url = Element("URL", None, None) + s3_attr_url.text = f"s3://lp-prod-protected//{prefix}/{hls_granule_ur}/{hls_granule_ur}.Fmask.tif" + s3_attr_desc = Element("URLDescription", None, None) + s3_attr_desc.text = f"This link provides direct download access via S3 to the Fmask quality layer {hls_granule_ur}.Fmask.tif" + s3_attr.append(s3_attr_url) + s3_attr.append(s3_attr_desc) + + access_urls.append(http_attr) + access_urls.append(s3_attr) + + def parse_args() -> Tuple[Path, Path]: short_options = "i:o:" long_options = ["instrument=", "inputdir=", "outputdir="] diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml index c4a2b7e..c6911b5 100644 --- a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml @@ -248,6 +248,14 @@ + + https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS.L30.T06WVS.2024120T211159.v2.0/HLS.L30.T06WVS.2024120T211159.v2.0.Fmask.tif + Download Fmask quality layer HLS.L30.T06WVS.2024120T211159.v2.0.Fmask.tif + + + s3://lp-prod-protected//HLSL30.020/HLS.L30.T06WVS.2024120T211159.v2.0/HLS.L30.T06WVS.2024120T211159.v2.0.Fmask.tif + This link provides direct download access via S3 to the Fmask quality layer HLS.L30.T06WVS.2024120T211159.v2.0.Fmask.tif + diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml index 80e3062..00abbd0 100644 --- a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml @@ -310,6 +310,14 @@ + + https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSS30.020/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.Fmask.tif + Download Fmask quality layer HLS.S30.T13RCN.2024128T173909.v2.0.Fmask.tif + + + s3://lp-prod-protected//HLSS30.020/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.Fmask.tif + This link provides direct download access via S3 to the Fmask quality layer HLS.S30.T13RCN.2024128T173909.v2.0.Fmask.tif + From f2ab46b1afb5a67b66bd8a6eb89cec9f61cd1e58 Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 20 Dec 2024 17:24:34 -0500 Subject: [PATCH 4/8] format & lint --- hls_vi/generate_metadata.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/hls_vi/generate_metadata.py b/hls_vi/generate_metadata.py index e5f9048..d9956b9 100644 --- a/hls_vi/generate_metadata.py +++ b/hls_vi/generate_metadata.py @@ -207,21 +207,20 @@ def set_additional_attribute(attrs: ElementBase, name: str, value: str) -> None: attrs.append(attr) -def append_fmask_online_access_urls(access_urls: ElementBase, hls_granule_ur: str) -> None: +def append_fmask_online_access_urls( + access_urls: ElementBase, hls_granule_ur: str +) -> None: """Include links to Fmask layer from HLS granule in metadata This is intended to help users find the relevant Fmask band without having to duplicate it into the HLS-VI product. See, https://github.com/NASA-IMPACT/hls-vi/issues/47 """ - prefix = ( - "HLSL30.020" if hls_granule_ur.startswith("HLS.L30") - else "HLSS30.020" - ) + prefix = "HLSL30.020" if hls_granule_ur.startswith("HLS.L30") else "HLSS30.020" http_attr = Element("OnlineAccessURL", None, None) http_attr_url = Element("URL", None, None) - http_attr_url.text = f"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/{prefix}/{hls_granule_ur}/{hls_granule_ur}.Fmask.tif" + http_attr_url.text = f"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/{prefix}/{hls_granule_ur}/{hls_granule_ur}.Fmask.tif" # noqa: E501 http_attr_desc = Element("URLDescription", None, None) http_attr_desc.text = f"Download Fmask quality layer {hls_granule_ur}.Fmask.tif" http_attr.append(http_attr_url) @@ -229,9 +228,11 @@ def append_fmask_online_access_urls(access_urls: ElementBase, hls_granule_ur: st s3_attr = Element("OnlineAccessURL", None, None) s3_attr_url = Element("URL", None, None) - s3_attr_url.text = f"s3://lp-prod-protected//{prefix}/{hls_granule_ur}/{hls_granule_ur}.Fmask.tif" + s3_attr_url.text = ( + f"s3://lp-prod-protected//{prefix}/{hls_granule_ur}/{hls_granule_ur}.Fmask.tif" + ) s3_attr_desc = Element("URLDescription", None, None) - s3_attr_desc.text = f"This link provides direct download access via S3 to the Fmask quality layer {hls_granule_ur}.Fmask.tif" + s3_attr_desc.text = f"This link provides direct download access via S3 to the Fmask quality layer {hls_granule_ur}.Fmask.tif" # noqa: E501 s3_attr.append(s3_attr_url) s3_attr.append(s3_attr_desc) From 1d73e026f7c59b1a223e54b8ae3a36c34174c36f Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 20 Dec 2024 17:34:19 -0500 Subject: [PATCH 5/8] Bump lxml for indenting and numpy for eaiser Mac ARM installs --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 713b4ed..31f82ef 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,8 @@ "dataclasses", "geojson", "importlib_resources", - "lxml==3.6.0", - "numpy~=1.19.0", + "lxml==5.3.0", + "numpy~=1.19", "pystac[validation]==1.0.0rc2", "rasterio", "shapely", From c1048b31dd317892ae19102ec558b6aa008caf54 Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 20 Dec 2024 17:42:29 -0500 Subject: [PATCH 6/8] Explain pin for numpy --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 31f82ef..c19d61e 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,8 @@ "geojson", "importlib_resources", "lxml==5.3.0", + # 1.19 is latest for our container, but ~1.19 allows + # a more recent 1.x version for our host "numpy~=1.19", "pystac[validation]==1.0.0rc2", "rasterio", From dd6ae5ee4b38b22e357945935c87ab7cf360e44c Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Mon, 23 Dec 2024 10:52:30 -0500 Subject: [PATCH 7/8] self review --- hls_vi/generate_metadata.py | 2 +- tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml | 2 +- tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hls_vi/generate_metadata.py b/hls_vi/generate_metadata.py index d9956b9..225e314 100644 --- a/hls_vi/generate_metadata.py +++ b/hls_vi/generate_metadata.py @@ -229,7 +229,7 @@ def append_fmask_online_access_urls( s3_attr = Element("OnlineAccessURL", None, None) s3_attr_url = Element("URL", None, None) s3_attr_url.text = ( - f"s3://lp-prod-protected//{prefix}/{hls_granule_ur}/{hls_granule_ur}.Fmask.tif" + f"s3://lp-prod-protected/{prefix}/{hls_granule_ur}/{hls_granule_ur}.Fmask.tif" ) s3_attr_desc = Element("URLDescription", None, None) s3_attr_desc.text = f"This link provides direct download access via S3 to the Fmask quality layer {hls_granule_ur}.Fmask.tif" # noqa: E501 diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml index c6911b5..5831deb 100644 --- a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml @@ -253,7 +253,7 @@ Download Fmask quality layer HLS.L30.T06WVS.2024120T211159.v2.0.Fmask.tif - s3://lp-prod-protected//HLSL30.020/HLS.L30.T06WVS.2024120T211159.v2.0/HLS.L30.T06WVS.2024120T211159.v2.0.Fmask.tif + s3://lp-prod-protected/HLSL30.020/HLS.L30.T06WVS.2024120T211159.v2.0/HLS.L30.T06WVS.2024120T211159.v2.0.Fmask.tif This link provides direct download access via S3 to the Fmask quality layer HLS.L30.T06WVS.2024120T211159.v2.0.Fmask.tif diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml index 00abbd0..0f0e6b9 100644 --- a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml @@ -315,7 +315,7 @@ Download Fmask quality layer HLS.S30.T13RCN.2024128T173909.v2.0.Fmask.tif - s3://lp-prod-protected//HLSS30.020/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.Fmask.tif + s3://lp-prod-protected/HLSS30.020/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.Fmask.tif This link provides direct download access via S3 to the Fmask quality layer HLS.S30.T13RCN.2024128T173909.v2.0.Fmask.tif From 16407a68c35cd07a2dffeb31adef85dcd20eb04a Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Mon, 23 Dec 2024 10:55:13 -0500 Subject: [PATCH 8/8] fix typo --- hls_vi/schema/MetadataCommon.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hls_vi/schema/MetadataCommon.xsd b/hls_vi/schema/MetadataCommon.xsd index 4ef8338..12ac2b2 100644 --- a/hls_vi/schema/MetadataCommon.xsd +++ b/hls_vi/schema/MetadataCommon.xsd @@ -446,7 +446,7 @@ - + The element should contain no children. In