Merge pull request #20 from emit-sds/develop

Merge develop to main for v1.2.0

CHANGELOG.md

@@ -4,14 +4,24 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v1.2.0](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/compare/v1.1.0...v1.2.0)
+
+> 18 March 2022
+
+- Add raw waterfall script [`37f5a31`](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/commit/37f5a314e622432c0413d84caa0c0fd9183f6cc9)
+- Fix PEP8 [`0ccb1f1`](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/commit/0ccb1f1cd9170e560227d7f59302f5640dd1d305)
+- Incrememnt line count by 2 if from processed partition AND coadd is enabled. [`365ac46`](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/commit/365ac462f0ae1baa5f9e2f9bb48f449c127937ae)
+
 #### [v1.1.0](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/compare/v1.0.0...v1.1.0)
 
 > 28 February 2022
 
+- Merge develop to main for v1.1.0 [`#19`](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/pull/19)
 - CCSDS format change [`#18`](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/pull/18)
+- CCSDS format change [`#17`](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/pull/17)
 - Add pad_byte_flag and use it to get and set data property to include the garbage pad byte or not. [`eb182c3`](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/commit/eb182c36b5f34c70586799cee9d6e9327f67ddcb)
+- Update changelog. [`b037da5`](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/commit/b037da57e32452c67848adf196b5181e9ee6d236)
 - Update version to 1.1.0 [`401c442`](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/commit/401c4420350206989053e70fc876d693ae458d42)
-- Fix PEP8 issue. [`9f57ace`](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/commit/9f57ace79133eab718a96f060c7503154eda2d46)
 
 ### [v1.0.0](https://github.jpl.nasa.gov/emit-sds/emit-sds-l1a/compare/v0.4.1...v1.0.0)
 

emit_sds_l1a/ccsds_packet.py

@@ -414,6 +414,7 @@ def _read_next_packet(self):
                 raise PSCMismatchException(msg)
 
     def _read_frame_start_packet(self):
+        sync_word_warning_count = 0
         while True:
             try:
                 pkt = None
@@ -462,7 +463,11 @@ def _read_frame_start_packet(self):
                     # Save the last chunk of packet data equal to the length of
                     # the HEADER sync word so we can handle the sync word being
                     # split across packets.
-                    logger.warning(f"Unable to find header sync word. Skipping packet {pkt}")
+                    skip_count = 500
+                    if sync_word_warning_count % skip_count == 0:
+                        logger.warning(f"Unable to find header sync word. Skipping packet {pkt}")
+                        logger.warning(f"Silencing up to next {skip_count} skips...")
+                    sync_word_warning_count += 1
                     self._pkt_partial = pkt
                     self._pkt_partial.data = self._pkt_partial.data[-len(self.HEADER_SYNC_WORD):]
 

reassemble_raw_cube.py

@@ -111,8 +111,8 @@ def get_utc_time_from_gps(gps_time):
 
 
 def reassemble_acquisition(acq_data_paths, start_index, stop_index, start_time, stop_time, timing_info, processed_flag,
-                           num_bands, num_lines, image_dir, report_text, failed_decompression_list, uncompressed_list,
-                           missing_frame_nums, logger):
+                           coadd_mode, num_bands, num_lines, image_dir, report_text, failed_decompression_list,
+                           uncompressed_list, missing_frame_nums, logger):
     # Reassemble frames into ENVI image cube filling in missing and cloudy data with data flags
     # First create acquisition_id from frame start_time
     # Assume acquisitions are at least 1 second long
@@ -144,7 +144,7 @@ def reassemble_acquisition(acq_data_paths, start_index, stop_index, start_time,
     cloudy_frame_nums = []
     line = 0
     lt_file = open(line_timestamps_path, "w")
-    lc_increment = 2 if processed_flag == 1 else 1
+    lc_increment = 2 if processed_flag == 1 and coadd_mode == 1 else 1
     lc_lookup = None
     corrupt_lines = []
     for path in acq_data_paths:
@@ -271,7 +271,7 @@ def reassemble_acquisition(acq_data_paths, start_index, stop_index, start_time,
         uncompressed_in_acq.sort()
 
         f.write(f"Total number of frames not requiring decompression in this acquisition "
-                f"(compression flag set to 0): {len(uncompressed_in_acq)}\n")
+                f"(compression flag set to 0 or cloudy flag set to 1): {len(uncompressed_in_acq)}\n")
         f.write("List of frame numbers not requiring decompression (if any):\n")
         if len(uncompressed_in_acq) > 0:
             f.write("\n".join(i for i in uncompressed_in_acq) + "\n")
@@ -409,8 +409,8 @@ def main():
         # Check frame checksum
         logger.debug(f"Frame is valid: {frame.is_valid()}")
 
-        # Decompress if compression flag is set, otherwise, just copy file
-        if frame.compression_flag == 1:
+        # Decompress if compression flag is set and frame is not cloudy, otherwise, just copy file
+        if frame.compression_flag == 1 and frame.cloudy_flag == 0:
             # Decompress frame
             interleave_arg = "--" + args.interleave
             cmd = [args.flexcodec_exe, path, "-a", args.constants_path, "-i", args.init_data_path, "-v",
@@ -435,7 +435,7 @@ def main():
 
         else:
             # Just copy the uncompressed frame and rename it
-            logger.info(f"Found uncompresssed frame at {path}. Copying to {uncomp_frame_path}")
+            logger.info(f"Found uncompresssed or cloudy frame at {path}. Copying to {uncomp_frame_path}")
             shutil.copy2(path, uncomp_frame_path)
             uncompressed_list.append(os.path.basename(path).split(".")[0].split("_")[2])
 
@@ -504,6 +504,7 @@ def main():
     for coadd_mode in coadd_mode_list:
         if not args.test_mode and coadd_mode == 0:
             raise RuntimeError(f"Some frames are not coadded.  See list of coadd_mode flags: {coadd_mode_list}")
+    coadd_mode = coadd_mode_list[0]
 
     # Get number of bands and lines
     num_bands = num_bands_list[0]
@@ -565,6 +566,7 @@ def main():
                                stop_time=start_stop_times[i + frame_chunksize - 1][1],
                                timing_info=timing_info,
                                processed_flag=processed_flag,
+                               coadd_mode=coadd_mode,
                                num_bands=num_bands,
                                num_lines=num_lines,
                                image_dir=image_dir,
@@ -583,6 +585,7 @@ def main():
                            stop_time=start_stop_times[num_frames - 1][1],
                            timing_info=timing_info,
                            processed_flag=processed_flag,
+                           coadd_mode=coadd_mode,
                            num_bands=num_bands,
                            num_lines=num_lines,
                            image_dir=image_dir,

setup.py

@@ -11,7 +11,7 @@
 
 setuptools.setup(
     name="emit_sds_l1a",
-    version="1.1.0",
+    version="1.2.0",
     author="Winston Olson-Duvall",
     author_email="[email protected]",
     description="""

util/raw_waterfall.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Jan 30 14:57:21 2022
+
+@author: bradley
+"""
+
+from pathlib import Path
+import sys
+
+import matplotlib.pyplot as plt
+from mpl_toolkits.axes_grid1 import make_axes_locatable
+import numpy as np
+import spectral.io.envi as envi
+
+filename = sys.argv[1]
+
+data_raw = envi.open(Path(filename).with_suffix('.hdr'))
+waterfall = np.mean(data_raw[:, :, :], axis=2)
+waterfall = waterfall / np.max(waterfall)
+
+plt.figure(figsize=(11, 8.5))
+plt.clf()
+im = plt.imshow(waterfall)
+plt.title('waterfall plot, average over spectral pixels')
+plt.xlabel('spatial pixel')
+plt.ylabel('frame')
+divider = make_axes_locatable(plt.gca())
+cax = divider.append_axes("right", size="5%", pad=0.05)
+plt.colorbar(im, cax=cax)
+plt.savefig(filename.replace('.img', '_waterfall.png'))
+plt.close()