diff --git a/.dockerignore b/.dockerignore index 1fbaf8a..6cdb0c4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ * !hls_vi/ !tests/ +tests/fixtures/ !setup.py !tox.ini diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..eb7dbf0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +# See https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..3415e3c --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,14 @@ +name: Test + +on: [push, workflow_dispatch] + +jobs: + tox_in_docker: + runs-on: ubuntu-20.04 + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Run tests in Docker container + run: make test diff --git a/Dockerfile b/Dockerfile index 61b7370..5e64abe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,8 @@ RUN : \ python3-venv \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ - && pip3 install --no-cache --upgrade pip setuptools \ - && pip3 install --no-cache rasterio==1.1.3 tox tox-venv --no-binary rasterio \ + && pip3 install --no-cache-dir --upgrade pip setuptools \ + && pip3 install --no-cache-dir rasterio==1.1.3 tox tox-venv --no-binary rasterio \ && : WORKDIR /hls_vi diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6044966 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +.PHONY = build test + +build: + docker compose build + +test: + docker compose run --build tox diff --git a/README.md b/README.md index d5f8b96..22691bc 100644 --- a/README.md +++ b/README.md @@ -38,5 +38,5 @@ where: You can run tests using Docker: ```bash -docker compose run --build tox +make test ``` diff --git a/compose.yaml b/compose.yaml index 1cf2001..4096d4b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,3 +1,5 @@ services: tox: build: . + volumes: + - ${PWD}/tests/fixtures:/hls_vi/tests/fixtures diff --git a/hls_vi/generate_indices.py b/hls_vi/generate_indices.py index f89cb1a..f2bb343 100644 --- a/hls_vi/generate_indices.py +++ b/hls_vi/generate_indices.py @@ -3,6 +3,7 @@ import getopt import os import re +import shutil import sys from datetime import datetime, timezone from enum import Enum, unique @@ -10,7 +11,6 @@ from typing import Callable, Mapping, Optional, SupportsFloat, Tuple, Type from typing_extensions import TypeAlias -import matplotlib.pyplot as plt import numpy as np import rasterio import rasterio.crs @@ -261,10 +261,6 @@ def write_granule_index( _FillValue=data.fill_value, ) - # Create browse image using NDVI - if index == Index.NDVI: - plt.imsave(str(output_path.with_suffix(".jpeg")), data, dpi=300, cmap="gray") - def evi(data: BandData) -> np.ma.masked_array: b, r, nir = data[Band.B], data[Band.R], data[Band.NIR] @@ -345,7 +341,7 @@ def __init__(self, long_name: str, scale_factor: SupportsFloat = 0.0001) -> None def __call__(self, data: BandData) -> np.ma.masked_array: scaled_index = self.compute_index(data) / self.scale_factor - return np.ma.round(scaled_index).astype(np.int16) + return np.ma.round(scaled_index, decimals=4).astype(np.int16) def parse_args() -> Tuple[Path, Path, str]: @@ -379,9 +375,20 @@ def parse_args() -> Tuple[Path, Path, str]: return Path(input_dir), Path(output_dir), id_str +def generate_vi_granule(input_dir: Path, output_dir: Path, id_str: str) -> Granule: + granule = read_granule_bands(input_dir, id_str) + write_granule_indices(output_dir, granule) + shutil.copy( + input_dir / f"{granule.id_}.jpg", + output_dir / f"{str(granule.id_).replace('HLS', 'HLS-VI')}.jpg", + ) + + return granule + + def main(): input_dir, output_dir, id_str = parse_args() - write_granule_indices(output_dir, read_granule_bands(input_dir, id_str)) + generate_vi_granule(input_dir, output_dir, id_str) if __name__ == "__main__": diff --git a/hls_vi/generate_metadata.py b/hls_vi/generate_metadata.py index 4ac3e86..96440cb 100644 --- a/hls_vi/generate_metadata.py +++ b/hls_vi/generate_metadata.py @@ -38,8 +38,19 @@ def generate_metadata(input_dir: Path, output_dir: Path): tree.find("InsertTime").text = formatted_date tree.find("LastUpdate").text = formatted_date - # TODO: Find out what the Collection ID should be - tree.find("Collection/DataSetId").text = "Update HLS-VI New Collection ID String" + dataset_id = tree.find("Collection/DataSetId") + dataset_id.text = ( + "HLS Operational Land Imager Vegetation Indices Daily Global 30 m V2.0" + if "L30" in metadata_path.name + else "HLS Sentinel-2 Multi-spectral Instrument Vegetation Indices Daily Global 30 m V2.0" # noqa: E501 + ) + set_additional_attribute( + tree.find("AdditionalAttributes"), + "IDENTIFIER_PRODUCT_DOI", + "10.5067/HLS/HLSL30_VI.002" + if "L30" in metadata_path.name + else "10.5067/HLS/HLSS30_VI.002", + ) data_granule = tree.find("DataGranule") data_granule.remove(data_granule.find("DataGranuleSizeInBytes")) @@ -57,6 +68,24 @@ def generate_metadata(input_dir: Path, output_dir: Path): ) +def set_additional_attribute(attrs: ET.Element, name: str, value: str): + attr = attrs.find(f'./AdditionalAttribute[Name="{name}"]') + + if attr is not None: + attr.find(".//Value").text = value + else: + attr = ET.Element("AdditionalAttribute") + attr_name = ET.Element("Name") + attr_name.text = name + attr_values = ET.Element("Values") + attr_value = ET.Element("Value") + attr_value.text = value + attr_values.append(attr_value) + attr.append(attr_name) + attr.append(attr_values) + attrs.append(attr) + + def parse_args() -> Tuple[Path, Path]: short_options = "i:o:" long_options = ["instrument=", "inputdir=", "outputdir="] diff --git a/setup.py b/setup.py index 7d4dd09..08ca76f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,6 @@ packages=["hls_vi"], install_requires=[ "dataclasses", - "matplotlib", "numpy~=1.19.0", "rasterio", "typing-extensions", @@ -16,6 +15,7 @@ "console_scripts": [ "vi_generate_indices=hls_vi.generate_indices:main", "vi_generate_metadata=hls_vi.generate_metadata:main", + "vi_generate_stac_items=hls_vi.generate_stac_items:main" ], }, ) diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml index 84f3595..77fbc45 100644 --- a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml @@ -4,7 +4,7 @@ 2024-05-08T21:25:20.907010Z 2024-05-08T21:25:20.907443Z - Update HLS-VI New Collection ID String + HLS Operational Land Imager Vegetation Indices Daily Global 30 m V2.0 HLS-VI.L30.T06WVS.2024120T211159 @@ -231,7 +231,7 @@ IDENTIFIER_PRODUCT_DOI - 10.5067/HLS/HLSL30.002 + 10.5067/HLS/HLSL30_VI.002 diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.EVI.tif b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.EVI.tif index d989601..8d9525b 100644 Binary files a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.EVI.tif and b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.EVI.tif differ diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.MSAVI.tif b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.MSAVI.tif index aa3b0ea..06ae2d3 100644 Binary files a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.MSAVI.tif and b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.MSAVI.tif differ diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR.tif b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR.tif index cf075bf..bc93e68 100644 Binary files a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR.tif and b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR.tif differ diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR2.tif b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR2.tif index 2e8a2e2..e89d950 100644 Binary files a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR2.tif and b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR2.tif differ diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDMI.tif b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDMI.tif index 7cb7822..d172bb4 100644 Binary files a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDMI.tif and b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDMI.tif differ diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDVI.tif b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDVI.tif index a9db3ca..30a76b4 100644 Binary files a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDVI.tif and b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDVI.tif differ diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDWI.tif b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDWI.tif index 0c2270d..1ae5579 100644 Binary files a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDWI.tif and b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDWI.tif differ diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.SAVI.tif b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.SAVI.tif index 775dead..a4094ef 100644 Binary files a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.SAVI.tif and b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.SAVI.tif differ diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.TVI.tif b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.TVI.tif index 70e5e4b..bd3b4b4 100644 Binary files a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.TVI.tif and b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.TVI.tif differ diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.jpg b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.jpg new file mode 100644 index 0000000..0b6303c Binary files /dev/null and b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.jpg differ diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml index 45d76fa..6350150 100644 --- a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml @@ -4,7 +4,7 @@ 2024-05-09T20:08:04.550553Z 2024-05-09T20:08:04.550572Z - Update HLS-VI New Collection ID String + HLS Sentinel-2 Multi-spectral Instrument Vegetation Indices Daily Global 30 m V2.0 HLS-VI.S30.T13RCN.2024128T173909 @@ -293,7 +293,7 @@ IDENTIFIER_PRODUCT_DOI - 10.5067/HLS/HLSS30.002 + 10.5067/HLS/HLSS30_VI.002 @@ -309,13 +309,5 @@ Cloud Optimized GeoTIFF (COG) - - https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/HLSS30.020/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.jpg - Download HLS.S30.T13RCN.2024128T173909.v2.0.jpg - - - s3://lp-prod-public/HLSS30.020/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.jpg - This link provides direct download access via S3 to the granule - diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.EVI.tif b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.EVI.tif index d897e9b..bef28dd 100644 Binary files a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.EVI.tif and b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.EVI.tif differ diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.MSAVI.tif b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.MSAVI.tif index f5f0600..644777a 100644 Binary files a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.MSAVI.tif and b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.MSAVI.tif differ diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NBR.tif b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NBR.tif index 6ba10a3..447da1a 100644 Binary files a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NBR.tif and b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NBR.tif differ diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NBR2.tif b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NBR2.tif index b284fdd..211bd1c 100644 Binary files a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NBR2.tif and b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NBR2.tif differ diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDMI.tif b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDMI.tif index 4eb1b20..de56cfb 100644 Binary files a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDMI.tif and b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDMI.tif differ diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDVI.tif b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDVI.tif index faf9a3f..7d11781 100644 Binary files a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDVI.tif and b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDVI.tif differ diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDWI.tif b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDWI.tif index c121d3c..689441c 100644 Binary files a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDWI.tif and b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.NDWI.tif differ diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.SAVI.tif b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.SAVI.tif index 7d3a917..4ff849d 100644 Binary files a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.SAVI.tif and b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.SAVI.tif differ diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.TVI.tif b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.TVI.tif index 2731ecc..6d65655 100644 Binary files a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.TVI.tif and b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.TVI.tif differ diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.jpg b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.jpg new file mode 100644 index 0000000..db91e66 Binary files /dev/null and b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0/HLS-VI.S30.T13RCN.2024128T173909.v2.0.jpg differ diff --git a/tests/fixtures/HLS.L30.T06WVS.2024120T211159.v2.0/HLS.L30.T06WVS.2024120T211159.v2.0.jpg b/tests/fixtures/HLS.L30.T06WVS.2024120T211159.v2.0/HLS.L30.T06WVS.2024120T211159.v2.0.jpg new file mode 100644 index 0000000..0b6303c Binary files /dev/null and b/tests/fixtures/HLS.L30.T06WVS.2024120T211159.v2.0/HLS.L30.T06WVS.2024120T211159.v2.0.jpg differ diff --git a/tests/fixtures/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.cmr.xml b/tests/fixtures/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.cmr.xml index 8af139b..b06ff24 100644 --- a/tests/fixtures/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.cmr.xml +++ b/tests/fixtures/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.cmr.xml @@ -310,13 +310,5 @@ Cloud Optimized GeoTIFF (COG) - - https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/HLSS30.020/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.jpg - Download HLS.S30.T13RCN.2024128T173909.v2.0.jpg - - - s3://lp-prod-public/HLSS30.020/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.jpg - This link provides direct download access via S3 to the granule - diff --git a/tests/fixtures/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.jpg b/tests/fixtures/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.jpg new file mode 100644 index 0000000..db91e66 Binary files /dev/null and b/tests/fixtures/HLS.S30.T13RCN.2024128T173909.v2.0/HLS.S30.T13RCN.2024128T173909.v2.0.jpg differ diff --git a/tests/fixtures/test_output.json b/tests/fixtures/test_output.json index 50ebbe9..893238f 100644 --- a/tests/fixtures/test_output.json +++ b/tests/fixtures/test_output.json @@ -15,7 +15,7 @@ "proj:epsg": 32606, "view:sun_azimuth": 168.41547441, "view:azimuth": 179.72857034, - "sci:doi": "10.5067/HLS/HLSL30.002", + "sci:doi": "10.5067/HLS/HLSL30_VI.002", "datetime": "2024-05-08T16:25:12.736230Z" }, "geometry": { @@ -38,7 +38,7 @@ "href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0_stac.json", "type": "application/json" }, - { "rel": "cite-as", "href": "https://doi.org/10.5067/HLS/HLSL30.002" } + { "rel": "cite-as", "href": "https://doi.org/10.5067/HLS/HLSL30_VI.002" } ], "assets": { "NDVI": { diff --git a/tests/fixtures/unit_test_output.json b/tests/fixtures/unit_test_output.json index 436adc9..a735505 100644 --- a/tests/fixtures/unit_test_output.json +++ b/tests/fixtures/unit_test_output.json @@ -1 +1 @@ -{"type": "Feature", "stac_version": "1.0.0", "id": "HLS-VI.L30.T06WVS.2024120T211159.v2.0", "properties": {"start_datetime": "2024-05-08T16:25:12.736230Z", "end_datetime": "2024-05-08T16:25:12.736230Z", "platform": "landsat-8", "instruments": ["oli"], "eo:cloud_cover": 6.0, "proj:transform": [30.0, 0.0, 399960.0, 0.0, -30.0, 7200000.0, 0.0, 0.0, 1.0], "proj:shape": [3660, 3660], "proj:epsg": 32606, "view:sun_azimuth": 168.41547441, "view:azimuth": 179.72857034, "sci:doi": "10.5067/HLS/HLSL30.002", "datetime": "2024-05-08T16:25:12.736230Z"}, "geometry": {"type": "MultiPolygon", "coordinates": [[[[-149.114568, 64.909135], [-146.793616, 64.923996], [-146.800894, 63.938713], [-149.040061, 63.924491], [-149.114568, 64.909135]]]]}, "links": [{"rel": "self", "href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0_stac.json", "type": "application/json"}, {"rel": "cite-as", "href": "https://doi.org/10.5067/HLS/HLSL30.002"}], "assets": {"NDVI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDVI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "NDVI", "common_name": "NDVI"}], "roles": ["data"]}, "EVI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.EVI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "EVI", "common_name": "EVI"}], "roles": ["data"]}, "MSAVI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.MSAVI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "MSAVI", "common_name": "MSAVI"}], "roles": ["data"]}, "NBR": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "NBR", "common_name": "NBR"}], "roles": ["data"]}, "NBR2": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR2.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "NBR2", "common_name": "NBR2"}], "roles": ["data"]}, "NDMI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDMI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "NDMI", "common_name": "NDMI"}], "roles": ["data"]}, "NDWI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDWI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "NDWI", "common_name": "NDWI"}], "roles": ["data"]}, "SAVI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.SAVI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "SAVI", "common_name": "SAVI"}], "roles": ["data"]}, "TVI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.TVI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "TVI", "common_name": "TVI"}], "roles": ["data"]}, "thumbnail": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.jpg", "type": "image/jpeg", "roles": ["thumbnail"]}}, "bbox": [-149.114568, 63.924491, -146.793616, 64.923996], "stac_extensions": ["https://stac-extensions.github.io/eo/v1.1.0/schema.json", "https://stac-extensions.github.io/projection/v1.1.0/schema.json", "https://stac-extensions.github.io/view/v1.0.0/schema.json", "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"]} \ No newline at end of file +{"type": "Feature", "stac_version": "1.0.0", "id": "HLS-VI.L30.T06WVS.2024120T211159.v2.0", "properties": {"start_datetime": "2024-05-08T16:25:12.736230Z", "end_datetime": "2024-05-08T16:25:12.736230Z", "platform": "landsat-8", "instruments": ["oli"], "eo:cloud_cover": 6.0, "proj:transform": [30.0, 0.0, 399960.0, 0.0, -30.0, 7200000.0, 0.0, 0.0, 1.0], "proj:shape": [3660, 3660], "proj:epsg": 32606, "view:sun_azimuth": 168.41547441, "view:azimuth": 179.72857034, "sci:doi": "10.5067/HLS/HLSL30_VI.002", "datetime": "2024-05-08T16:25:12.736230Z"}, "geometry": {"type": "MultiPolygon", "coordinates": [[[[-149.114568, 64.909135], [-146.793616, 64.923996], [-146.800894, 63.938713], [-149.040061, 63.924491], [-149.114568, 64.909135]]]]}, "links": [{"rel": "self", "href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0_stac.json", "type": "application/json"}, {"rel": "cite-as", "href": "https://doi.org/10.5067/HLS/HLSL30_VI.002"}], "assets": {"NDVI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDVI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "NDVI", "common_name": "NDVI"}], "roles": ["data"]}, "EVI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.EVI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "EVI", "common_name": "EVI"}], "roles": ["data"]}, "MSAVI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.MSAVI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "MSAVI", "common_name": "MSAVI"}], "roles": ["data"]}, "NBR": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "NBR", "common_name": "NBR"}], "roles": ["data"]}, "NBR2": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NBR2.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "NBR2", "common_name": "NBR2"}], "roles": ["data"]}, "NDMI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDMI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "NDMI", "common_name": "NDMI"}], "roles": ["data"]}, "NDWI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.NDWI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "NDWI", "common_name": "NDWI"}], "roles": ["data"]}, "SAVI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.SAVI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "SAVI", "common_name": "SAVI"}], "roles": ["data"]}, "TVI": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.TVI.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "eo:bands": [{"name": "TVI", "common_name": "TVI"}], "roles": ["data"]}, "thumbnail": {"href": "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/HLSL30.020/HLS-VI.L30.T06WVS.2024120T211159.v2.0/HLS-VI.L30.T06WVS.2024120T211159.v2.0.jpg", "type": "image/jpeg", "roles": ["thumbnail"]}}, "bbox": [-149.114568, 63.924491, -146.793616, 64.923996], "stac_extensions": ["https://stac-extensions.github.io/eo/v1.1.0/schema.json", "https://stac-extensions.github.io/projection/v1.1.0/schema.json", "https://stac-extensions.github.io/view/v1.0.0/schema.json", "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"]} \ No newline at end of file diff --git a/tests/test_vi.py b/tests/test_vi.py new file mode 100644 index 0000000..bc1f599 --- /dev/null +++ b/tests/test_vi.py @@ -0,0 +1,200 @@ +from datetime import datetime +from pathlib import Path +from typing import Mapping, Optional, Tuple +from xml.etree import ElementTree as ET +import contextlib +import io +import json + +import pytest +import rasterio +from hls_vi.generate_metadata import generate_metadata +from hls_vi.generate_indices import ( + Granule, + Index, + generate_vi_granule, +) +from hls_vi.generate_stac_items import create_item + + +def find_index_by_long_name(long_name: str) -> Index: + for index in Index: + if index.long_name == long_name: + return index + raise ValueError(f"Index with longname '{long_name}' not found") + + +def assert_tifs_equal(granule: Granule, actual: Path, expected: Path): + with rasterio.open(actual) as actual_src: + with rasterio.open(expected) as expected_src: + assert actual_src.count == expected_src.count + assert actual_src.width == expected_src.width + assert actual_src.height == expected_src.height + assert actual_src.bounds == expected_src.bounds + assert actual_src.crs == expected_src.crs + assert actual_src.transform == expected_src.transform + + index = find_index_by_long_name(actual_src.tags()["long_name"]) + index_data = index(granule.data).filled() + actual_data = actual_src.read() + expected_data = expected_src.read() + + assert (actual_data == index_data).all() + assert (actual_data == expected_data).all() + + actual_tags, actual_time_str = remove_item( + actual_src.tags(), "HLS_VI_PROCESSING_TIME" + ) + expected_tags, expected_time_str = remove_item( + expected_src.tags(), "HLS_VI_PROCESSING_TIME" + ) + + assert actual_tags == expected_tags + assert actual_time_str is not None + assert expected_time_str is not None + + actual_time = datetime.strptime(actual_time_str, "%Y-%m-%dT%H:%M:%S.%fZ") + expected_time = datetime.strptime( + expected_time_str, "%Y-%m-%dT%H:%M:%S.%fZ" + ) + + # The actual time should be greater than the expected time because + # the actual time is the time when the VI was generated (now, during + # test execution), which is after the time when the expected VI was + # generated (as a test fixture). + assert actual_time > expected_time + + +def remove_item( + mapping: Mapping[str, str], key: str +) -> Tuple[Mapping[str, str], Optional[str]]: + return {k: v for k, v in mapping.items() if k != key}, mapping.get(key) + + +def remove_element(root: ET.Element, path: str) -> None: + parent_path = "/".join(path.split("/")[:-1]) + parent = root.find(parent_path) + child = root.find(path) + + assert parent is not None + assert child is not None + + parent.remove(child) + + +def remove_datetime_elements(tree: ET.ElementTree) -> ET.ElementTree: + root = tree.getroot() + + remove_element(root, "./InsertTime") + remove_element(root, "./LastUpdate") + remove_element(root, "./DataGranule/ProductionDateTime") + remove_element(root, "./Temporal/RangeDateTime/BeginningDateTime") + remove_element(root, "./Temporal/RangeDateTime/EndingDateTime") + + return tree + + +def assert_indices_equal(granule: Granule, actual_dir: Path, expected_dir: Path): + actual_tif_paths = sorted(actual_dir.glob("*.tif")) + actual_tif_names = [path.name for path in actual_tif_paths] + expected_tif_paths = sorted(expected_dir.glob("*.tif")) + expected_tif_names = [path.name for path in expected_tif_paths] + + assert actual_tif_names == expected_tif_names + + for actual_tif_path, expected_tif_path in zip(actual_tif_paths, expected_tif_paths): + assert_tifs_equal(granule, actual_tif_path, expected_tif_path) + + +@pytest.mark.parametrize( + argnames="input_dir,id_str", + argvalues=[ + ( + "tests/fixtures/HLS.L30.T06WVS.2024120T211159.v2.0", + "HLS.L30.T06WVS.2024120T211159.v2.0", + ), + ( + "tests/fixtures/HLS.S30.T13RCN.2024128T173909.v2.0", + "HLS.S30.T13RCN.2024128T173909.v2.0", + ), + ], +) +def test_generate_indices(input_dir, id_str, tmp_path: Path): + granule = generate_vi_granule(Path(input_dir), tmp_path, id_str) + assert_indices_equal(granule, tmp_path, Path(input_dir.replace("HLS", "HLS-VI"))) + assert (tmp_path / f"{id_str.replace('HLS', 'HLS-VI')}.jpg").exists() + + +@pytest.mark.parametrize( + argnames="input_dir,output_dir", + argvalues=[ + ( + "tests/fixtures/HLS.L30.T06WVS.2024120T211159.v2.0", + "tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0", + ), + ( + "tests/fixtures/HLS.S30.T13RCN.2024128T173909.v2.0", + "tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0", + ), + ], +) +def test_generate_cmr_metadata(input_dir, output_dir): + input_path = Path(input_dir) + output_path = Path(output_dir) + input_cmr_xml_path = next(input_path.glob("HLS.*.cmr.xml")) + output_cmr_xml_basename = input_cmr_xml_path.name.replace("HLS", "HLS-VI") + actual_metadata_path = output_path / output_cmr_xml_basename + + # We keep the expected metadata file outside of the output directory, + # otherwise it would be overwritten by the actual metadata file. + expected_metadata_path = Path("tests/fixtures") / output_cmr_xml_basename + + try: + generate_metadata(input_dir=input_path, output_dir=output_path) + + actual_metadata_tree = remove_datetime_elements(ET.parse(actual_metadata_path)) + expected_metadata_tree = remove_datetime_elements( + ET.parse(expected_metadata_path) + ) + + actual_metadata = io.BytesIO() + actual_metadata_tree.write(actual_metadata, encoding="utf-8") + + expected_metadata = io.BytesIO() + expected_metadata_tree.write(expected_metadata, encoding="utf-8") + + assert ( + actual_metadata.getvalue().decode() == expected_metadata.getvalue().decode() + ) + finally: + with contextlib.suppress(FileNotFoundError): + actual_metadata_path.unlink() + +@pytest.fixture +def cmr_xml(): + return "tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml" + +@pytest.fixture +def json_output(): + return "tests/fixtures/unit_test_output.json" + +@pytest.fixture +def endpoint(): + return "data.lpdaac.earthdatacloud.nasa.gov" + +@pytest.fixture +def version(): + return "020" + +def test_generate_stac_items(cmr_xml, json_output, endpoint, version): + create_item( + cmr_xml, + json_output, + endpoint, + version, + ) + with open('tests/fixtures/test_output.json') as f: + stac_item = json.load(f) + with open(json_output) as f: + unit_test_stac_item = json.load(f) + assert stac_item == unit_test_stac_item \ No newline at end of file diff --git a/tox.ini b/tox.ini index a5f4fc6..75de963 100644 --- a/tox.ini +++ b/tox.ini @@ -20,4 +20,4 @@ extras = test commands = flake8 - pytest -v + pytest -vv