From 590878b13295794091e704d90e00594911db8882 Mon Sep 17 00:00:00 2001 From: Winston Olson-Duvall Date: Wed, 4 Jan 2023 10:37:16 -0800 Subject: [PATCH 1/6] Fix NetCDF metadata issues --- output_conversion.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/output_conversion.py b/output_conversion.py index 46044fa..88c766d 100644 --- a/output_conversion.py +++ b/output_conversion.py @@ -51,11 +51,12 @@ def main(): logging.debug('Creating global attributes') makeGlobalAttr(nc_ds, args.rfl_file, args.glt_file) - nc_ds.title = "EMIT L2A Surface Reflectance 60 m " + args.version + nc_ds.title = "EMIT L2A Estimated Surface Reflectance 60 m " + args.version nc_ds.summary = nc_ds.summary + \ f"\\n\\nThis file contains L2A estimated surface reflectances \ - and geolocation data. Reflectance estimates are created using an Optimal Estimation technique - see ATBD for \ - details. Reflectance values are reported as fractions (relative to 1). " +and geolocation data. Reflectance estimates are created using an Optimal Estimation technique - see ATBD for \ +details. Reflectance values are reported as fractions (relative to 1). \ +Geolocation data (latitude, longitude, height) and a lookup table to project the data are also included." nc_ds.sync() logging.debug('Creating dimensions') @@ -93,11 +94,12 @@ def main(): logging.debug('Creating global attributes') makeGlobalAttr(nc_ds, args.rfl_unc_file, args.glt_file) - nc_ds.title = "EMIT L2A Surface Reflectance Uncertainty 60 m V001" + nc_ds.title = "EMIT L2A Estimated Surface Reflectance Uncertainty 60 m " + args.version nc_ds.summary = nc_ds.summary + \ f"\\n\\nThis file contains L2A estimated surface reflectance uncertainties \ - and geolocation data. Reflectance uncertainty estimates are created using an Optimal Estimation technique - see ATBD for \ - details. Reflectance uncertainty values are reported as fractions (relative to 1). " +and geolocation data. Reflectance uncertainty estimates are created using an Optimal Estimation technique - see ATBD for \ +details. Reflectance uncertainty values are reported as fractions (relative to 1). \ +Geolocation data (latitude, longitude, height) and a lookup table to project the data are also included." nc_ds.sync() logging.debug('Creating dimensions') @@ -134,10 +136,12 @@ def main(): logging.debug('Creating global attributes') makeGlobalAttr(nc_ds, args.mask_file, args.glt_file) - nc_ds.title = "EMIT L2A Masks 60 m V001" + nc_ds.title = "EMIT L2A Masks 60 m " + args.version nc_ds.summary = nc_ds.summary + \ f"\\n\\nThis file contains masks for L2A estimated surface reflectances \ - and geolocation data. Masks account for clouds, cloud shadows (via buffering), spacecraft interference, and poor atmospheric conditions." +and geolocation data. Masks account for clouds, cloud shadows (via buffering), spacecraft interference, and poor \ +atmospheric conditions. \ +Geolocation data (latitude, longitude, height) and a lookup table to project the data are also included." nc_ds.sync() logging.debug('Creating dimensions') From dfb55c43a44518e37a7db07d19240f598201fe5e Mon Sep 17 00:00:00 2001 From: Phil Brodrick Date: Mon, 9 Jan 2023 09:50:45 -0800 Subject: [PATCH 2/6] grab bbl if specified, if not add something in --- output_conversion.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/output_conversion.py b/output_conversion.py index 88c766d..0149b3d 100644 --- a/output_conversion.py +++ b/output_conversion.py @@ -10,6 +10,7 @@ from emit_utils.file_checks import netcdf_ext, envi_header from spectral.io import envi import logging +import numpy as np def main(): @@ -68,6 +69,18 @@ def main(): add_variable(nc_ds, "sensor_band_parameters/fwhm", "f4", "Full Width at Half Max", "nm", [float(d) for d in rfl_ds.metadata['fwhm']], {"dimensions": ("bands",)}) + # Handle data pre January, where bbl was not set in ENVI header + if 'bbl' not in rfl_ds.metadata['bbl'] or rfl_ds.metadata['bbl'] == '{}': + wl = np.array(nc_ds['sensor_band_parameters']['wavelength']) + bbl = np.ones(len(wl)) + bbl[np.logical_and(wl > 1325, wl < 1435)] = 0 + bbl[np.logical_and(wl > 1770, wl < 1962)] = 0 + else: + bbl = [bool(d) for d in rfl_ds.metadata['bbl']] + + add_variable(nc_ds, "sensor_band_parameters/good_wavelengths", "u1", "Wavelengths where reflectance is useable: 1 = good data, 0 = bad data", "unitless", + bbl, {"dimensions": ("bands",)}) + logging.debug('Creating and writing location data') add_loc(nc_ds, args.loc_file) From 85fe2599d560b34dc6f6fe8876e9718ba553ac00 Mon Sep 17 00:00:00 2001 From: Winston Olson-Duvall Date: Tue, 10 Jan 2023 09:25:01 -0800 Subject: [PATCH 3/6] Fix syntax --- output_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/output_conversion.py b/output_conversion.py index 0149b3d..907060e 100644 --- a/output_conversion.py +++ b/output_conversion.py @@ -70,7 +70,7 @@ def main(): [float(d) for d in rfl_ds.metadata['fwhm']], {"dimensions": ("bands",)}) # Handle data pre January, where bbl was not set in ENVI header - if 'bbl' not in rfl_ds.metadata['bbl'] or rfl_ds.metadata['bbl'] == '{}': + if 'bbl' not in rfl_ds.metadata or rfl_ds.metadata['bbl'] == '{}': wl = np.array(nc_ds['sensor_band_parameters']['wavelength']) bbl = np.ones(len(wl)) bbl[np.logical_and(wl > 1325, wl < 1435)] = 0 From fc5b6132cd687a171949d887b44863e9b279060c Mon Sep 17 00:00:00 2001 From: Winston Olson-Duvall Date: Tue, 10 Jan 2023 13:00:09 -0800 Subject: [PATCH 4/6] Fix key error. --- output_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/output_conversion.py b/output_conversion.py index 907060e..694c426 100644 --- a/output_conversion.py +++ b/output_conversion.py @@ -71,7 +71,7 @@ def main(): # Handle data pre January, where bbl was not set in ENVI header if 'bbl' not in rfl_ds.metadata or rfl_ds.metadata['bbl'] == '{}': - wl = np.array(nc_ds['sensor_band_parameters']['wavelength']) + wl = np.array(nc_ds['sensor_band_parameters']['wavelengths']) bbl = np.ones(len(wl)) bbl[np.logical_and(wl > 1325, wl < 1435)] = 0 bbl[np.logical_and(wl > 1770, wl < 1962)] = 0 From 60ebe714974074ca0cf0150f39969c140793d2ac Mon Sep 17 00:00:00 2001 From: Phil Brodrick Date: Tue, 10 Jan 2023 14:58:47 -0800 Subject: [PATCH 5/6] add bad wavelengths to uncertainty --- output_conversion.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/output_conversion.py b/output_conversion.py index 694c426..9092af6 100644 --- a/output_conversion.py +++ b/output_conversion.py @@ -123,6 +123,8 @@ def main(): [float(d) for d in rfl_ds.metadata['wavelength']], {"dimensions": ("bands",)}) add_variable(nc_ds, "sensor_band_parameters/fwhm", "f4", "Full Width at Half Max", "nm", [float(d) for d in rfl_ds.metadata['fwhm']], {"dimensions": ("bands",)}) + add_variable(nc_ds, "sensor_band_parameters/good_wavelengths", "u1", "Wavelengths where reflectance is useable: 1 = good data, 0 = bad data", "unitless", + bbl, {"dimensions": ("bands",)}) logging.debug('Creating and writing location data') add_loc(nc_ds, args.loc_file) From df3470a5a367dfa241a32632b64a545176d9a2b8 Mon Sep 17 00:00:00 2001 From: Winston Olson-Duvall Date: Thu, 12 Jan 2023 07:54:43 -0800 Subject: [PATCH 6/6] Update change log --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5442885..aa9222f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,20 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). +#### [v1.3.1](https://github.com/emit-sds/emit-sds-l2a/compare/v1.3.0...v1.3.1) + +> 10 January 2023 + +- Bad wavelengths update [`#6`](https://github.com/emit-sds/emit-sds-l2a/pull/6) +- Fix NetCDF metadata issues [`590878b`](https://github.com/emit-sds/emit-sds-l2a/commit/590878b13295794091e704d90e00594911db8882) +- grab bbl if specified, if not add something in [`dfb55c4`](https://github.com/emit-sds/emit-sds-l2a/commit/dfb55c43a44518e37a7db07d19240f598201fe5e) +- add bad wavelengths to uncertainty [`60ebe71`](https://github.com/emit-sds/emit-sds-l2a/commit/60ebe714974074ca0cf0150f39969c140793d2ac) + #### [v1.3.0](https://github.com/emit-sds/emit-sds-l2a/compare/v1.2.0...v1.3.0) -> 3 November 2022 +> 7 November 2022 +- Merge develop into main for v1.3.0 [`#5`](https://github.com/emit-sds/emit-sds-l2a/pull/5) - Analytical atm updates [`#4`](https://github.com/emit-sds/emit-sds-l2a/pull/4) - Rerun updates [`#3`](https://github.com/emit-sds/emit-sds-l2a/pull/3) - updated covariance for oxygen a help [`272d852`](https://github.com/emit-sds/emit-sds-l2a/commit/272d852d89bac2dab5adc597deebf1d16b595d1f)