Merge pull request #10 from emit-sds/develop

Merge develop into main for v1.5.0

.gitignore

@@ -2,4 +2,6 @@
 emit_sds_l1a.egg-info
 __pycache__
 .coverage
-.DS_Store
+.DS_Store
+*.log
+tmp

CHANGELOG.md

@@ -4,10 +4,22 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v1.5.0](https://github.com/emit-sds/emit-sds-l1a/compare/v1.4.1...v1.5.0)
+
+> 14 October 2022
+
+- Corrupt frame handling in reassembly [`#9`](https://github.com/emit-sds/emit-sds-l1a/pull/9)
+- Merge IOC hotfixes into develop [`#8`](https://github.com/emit-sds/emit-sds-l1a/pull/8)
+- Save corrupt frames with '9' as acquisition status. During reassembly only use these frames if data is not compressed. [`8ce2620`](https://github.com/emit-sds/emit-sds-l1a/commit/8ce26200972cb2d3ebca109977b7c37abe9da578)
+- Fill in missing gps times in line timestamps file [`e42477c`](https://github.com/emit-sds/emit-sds-l1a/commit/e42477cfb9372bdffb03fea655ba7870907214fb)
+- Add util for checking packet order in HOSC data. [`f2e0519`](https://github.com/emit-sds/emit-sds-l1a/commit/f2e0519b8f8dc3e7d19ce2326990b08b9a5eeb53)
+
 #### [v1.4.1](https://github.com/emit-sds/emit-sds-l1a/compare/v1.4.0...v1.4.1)
 
 > 26 July 2022
 
+- Merge develop into main for v1.4.1 [`#7`](https://github.com/emit-sds/emit-sds-l1a/pull/7)
+- Update change log [`bfc5933`](https://github.com/emit-sds/emit-sds-l1a/commit/bfc59333caa4f8aefd211211748feca0886081e6)
 - Update version to 1.4.1 [`69d7df0`](https://github.com/emit-sds/emit-sds-l1a/commit/69d7df0bb6b9695a0ee11dd0b8ff94738f3ef6b3)
 - Use 7 digit default orbit id [`e073e9e`](https://github.com/emit-sds/emit-sds-l1a/commit/e073e9e3e50428477797de4533e017b2d03cd5a4)
 

depacketize_science_frames.py

@@ -86,7 +86,10 @@ def main():
         try:
             frame_binary = processor.read_frame()
             frame = Frame(frame_binary)
-            frame.save(frames_dir)
+            if frame.corrupt_name in processor.corrupt_frames:
+                frame.save(frames_dir, corrupt=True)
+            else:
+                frame.save(frames_dir, corrupt=False)
             frame_count += 1
         except EOFError:
             break

emit_sds_l1a/ccsds_packet.py

@@ -346,11 +346,8 @@ def truncated_frame(self):
         self._stats["truncated_frame_errors"] += 1
 
     def corrupt_frame(self, frame):
-        name = "_".join([str(frame.dcid).zfill(10), frame.start_time.strftime("%Y%m%dt%H%M%S"),
-                         str(frame.frame_count_in_acq).zfill(5), str(frame.planned_num_frames).zfill(5),
-                         str(frame.acq_status), str(frame.processed_flag)])
-        if name not in self._stats["corrupt_frames"]:
-            self._stats["corrupt_frames"].append(name)
+        if frame.corrupt_name not in self._stats["corrupt_frames"]:
+            self._stats["corrupt_frames"].append(frame.corrupt_name)
 
     def get_data_bytes_read(self):
         return self._stats["data_bytes_read"]
@@ -395,6 +392,7 @@ def __init__(self, stream_path, pkt_format="1.3"):
         logger.debug(f"Initializing SciencePacketProcessor from path {stream_path} using FSW v{pkt_format}")
         self.stream = open(stream_path, "rb")
         self.pkt_format = pkt_format
+        self.corrupt_frames = set()
         self._cur_psc = -1
         self._cur_coarse = -1
         self._cur_fine = -1
@@ -425,7 +423,7 @@ def _read_next_packet(self):
             pkt = ScienceDataPacket(stream=self.stream, pkt_format=self.pkt_format)
             logger.debug(pkt)
             self._stats.ccsds_read(pkt)
-            pkt_hash = str(pkt.coarse_time) + str(pkt.fine_time) + str(pkt.pkt_seq_cnt)
+            pkt_hash = "_".join([str(pkt.coarse_time), str(pkt.fine_time), str(pkt.pkt_seq_cnt)])
 
             # Handle case where packet is not valid
             if not pkt.is_valid:
@@ -657,6 +655,7 @@ def _read_pkt_parts(self, start_pkt):
                         logger.info(f"Inserted garbage packet with {pkt.MAX_DATA_LEN} bytes of data. Accum data is "
                                     f"now {data_accum_len}")
                         self._stats.corrupt_frame(frame)
+                        self.corrupt_frames.add(frame.corrupt_name)
                     elif 0 < remaining_data_len < pkt.MAX_DATA_LEN:
                         if self.pkt_format == "1.2.1":
                             if pkt.pad_byte_flag == 0:
@@ -674,6 +673,7 @@ def _read_pkt_parts(self, start_pkt):
                         logger.info(f"Inserted garbage packet with {remaining_data_len} bytes of data. Accum data is "
                                     f"now {data_accum_len}")
                         self._stats.corrupt_frame(frame)
+                        self.corrupt_frames.add(frame.corrupt_name)
 
             pkt_parts.append(pkt)
             data_accum_len += len(pkt.data)

emit_sds_l1a/frame.py

@@ -158,6 +158,14 @@ def __init__(self, frame_binary):
         self.instrument_mode_desc = "No match" if self.instrument_mode == "no_match" else \
             INSTRUMENT_MODES[self.instrument_mode]["desc"]
 
+        # Frame name and also corrupt name if needed
+        self.name = "_".join([str(self.dcid).zfill(10), self.start_time.strftime("%Y%m%dt%H%M%S"),
+                              str(self.frame_count_in_acq).zfill(5), str(self.planned_num_frames).zfill(5),
+                              str(self.acq_status), str(self.processed_flag)])
+        self.corrupt_name = "_".join([str(self.dcid).zfill(10), self.start_time.strftime("%Y%m%dt%H%M%S"),
+                                      str(self.frame_count_in_acq).zfill(5), str(self.planned_num_frames).zfill(5),
+                                      str(9), str(self.processed_flag)])
+
         logger.debug(f"Initialized frame: {self}")
 
     def __repr__(self):
@@ -218,12 +226,11 @@ def is_valid(self):
                      f"Computed checksum: {self._compute_hdr_checksum()}")
         return is_valid
 
-    def save(self, out_dir):
-        fname = "_".join([str(self.dcid).zfill(10), self.start_time.strftime("%Y%m%dt%H%M%S"),
-                          str(self.frame_count_in_acq).zfill(5), str(self.planned_num_frames).zfill(5),
-                          str(self.acq_status), str(self.processed_flag)])
-
-        out_path = os.path.join(out_dir, fname)
+    def save(self, out_dir, corrupt=False):
+        if corrupt:
+            out_path = os.path.join(out_dir, self.corrupt_name)
+        else:
+            out_path = os.path.join(out_dir, self.name)
 
         logger.info("Writing frame to path %s" % out_path)
         logger.debug("data length is %s" % len(self.data))