Skip to content

Commit

Permalink
Merge pull request #6 from emit-sds/develop
Browse files Browse the repository at this point in the history 
Merge develop into main for v1.4.0
  • Loading branch information
@winstonolson
winstonolson authored Jul 22, 2022
2 parents 65f00ee + 9550cf6 commit ecdefff
Show file tree
Hide file tree
Showing 7 changed files with 223 additions and 97 deletions.
13 changes: 12 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,27 @@ All notable changes to this project will be documented in this file. Dates are d

Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).

#### [v1.4.0](https://github.com/emit-sds/emit-sds-l1a/compare/v1.3.0...v1.4.0)

> 22 July 2022
- Dcid report [`#5`](https://github.com/emit-sds/emit-sds-l1a/pull/5)
- FSW 1.3 CCSDS update and pkt_format switch [`#4`](https://github.com/emit-sds/emit-sds-l1a/pull/4)
- Update ScienceDataPacket and SciencePacketProcessor to use new fsw_ver switch. [`f934b92`](https://github.com/emit-sds/emit-sds-l1a/commit/f934b925858892415d98d5d81bd7f0ea3faa06f1)
- Add dcid_reassembly_report.txt that includes reassembly stats for all acquisitions. [`fc290cd`](https://github.com/emit-sds/emit-sds-l1a/commit/fc290cd720e3762984f4c338898598950f4ece7c)
- Remove checking for pad byte flag. Instead increase secondary header size to 13. Add real_pkt_data_len. Compute num_garbage_bytes. Update data getter and setter. Update crc calculation. Update all places in code where partial or garbage packets are created. [`90263ec`](https://github.com/emit-sds/emit-sds-l1a/commit/90263eca8e457d7a79a57cf39f4de7d49f0aa544)

#### [v1.3.0](https://github.com/emit-sds/emit-sds-l1a/compare/v1.2.0...v1.3.0)

> 6 June 2022
- Merge develop into main for v1.3.0 [`#3`](https://github.com/emit-sds/emit-sds-l1a/pull/3)
- J2000 and BAD timing [`#2`](https://github.com/emit-sds/emit-sds-l1a/pull/2)
- Check number of valid lines and instrument mode [`#1`](https://github.com/emit-sds/emit-sds-l1a/pull/1)
- Pre ioc ccsds updates [`#21`](https://github.com/emit-sds/emit-sds-l1a/pull/21)
- update LICENSE and README [`ec26b1e`](https://github.com/emit-sds/emit-sds-l1a/commit/ec26b1ecb48d0601bab1803671cca1deb8a1896f)
- Check frame instrument modes for consistency and add mode description to report. [`c3244ab`](https://github.com/emit-sds/emit-sds-l1a/commit/c3244ab2132e8786ef162cfd65a7c8581a26e3a9)
- Add utils to check packet sizes below a threshold and to find all the sync words in a stream. [`975ce53`](https://github.com/emit-sds/emit-sds-l1a/commit/975ce53bc98e4cd7ae8c732e421616c3196d3472)
- Update change log [`8078f17`](https://github.com/emit-sds/emit-sds-l1a/commit/8078f177a39a61b265e7611fe530f5afaaaabd18)

#### [v1.2.0](https://github.com/emit-sds/emit-sds-l1a/compare/v1.1.0...v1.2.0)

Expand Down
5 changes: 3 additions & 2 deletions depacketize_science_frames.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ def main():
" * Depacketization summary/report file named depacketize_science_frames_report.txt (default)\n",
formatter_class=RawTextHelpFormatter)
parser.add_argument("stream_path", help="Path to CCSDS stream file")
parser.add_argument("--pkt_format", help="Flight software version to use", default="1.3")
parser.add_argument("--work_dir", help="Path to working directory", default=".")
parser.add_argument("--prev_stream_path", help="Path to previous CCSDS stream file")
parser.add_argument("--prev_bytes_to_read", help="How many bytes to read from the end of the previous stream",
Expand Down Expand Up @@ -77,8 +78,8 @@ def main():
with open(tmp_stream_path, "wb") as f:
f.write(stream)

logger.info(f"Processing stream file {tmp_stream_path}")
processor = SciencePacketProcessor(tmp_stream_path)
logger.info(f"Processing stream file {tmp_stream_path} using packet format from FSW v{args.pkt_format}")
processor = SciencePacketProcessor(tmp_stream_path, pkt_format=args.pkt_format)

frame_count = 0
while True:
Expand Down
157 changes: 108 additions & 49 deletions emit_sds_l1a/ccsds_packet.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,13 +126,13 @@ class ScienceDataPacket(CCSDSPacket):

HEADER_SYNC_WORD = 0x81FFFF81
PRIMARY_HDR_LEN = 6
SEC_HDR_LEN = 11
CRC_LEN = 4

def __init__(self, stream=None, **kwargs):
def __init__(self, stream=None, pkt_format="1.3", **kwargs):
"""Inititialize EngineeringDataPacket
Arguments:
stream - A file object from which to read data (default: None)
pkt_format - The format of the CCSDS packet defined by FSW version (typically 1.2.1 or 1.3)
Keyword Arguments:
- **hdr_data**: A bytes-like object containing 6-bytes
of data that should be processed as a CCSDS Packet header.
Expand All @@ -142,24 +142,32 @@ def __init__(self, stream=None, **kwargs):
enforced if these kwargs are used.
"""
super(ScienceDataPacket, self).__init__(stream=stream, **kwargs)
# logger.debug("SDP primary header: " + str([bin(self.hdr_data[i])[2:].zfill(8) for i in range(self.PRIMARY_HDR_LEN)]))
self.pkt_format = pkt_format
self.SEC_HDR_LEN = 11 if pkt_format == "1.2.1" else 13
self.MAX_DATA_LEN = 1479 if pkt_format == "1.2.1" else 1477

@property
def data(self):
if self.body:
if self.pad_byte_flag == 0:
return self.body[self.SEC_HDR_LEN: -self.CRC_LEN]
if self.pkt_format == "1.2.1":
if self.pad_byte_flag == 0:
return self.body[self.SEC_HDR_LEN: -self.CRC_LEN]
else:
return self.body[self.SEC_HDR_LEN: -(self.CRC_LEN + 1)]
else:
return self.body[self.SEC_HDR_LEN: -(self.CRC_LEN + 1)]
return self.body[self.SEC_HDR_LEN: -(self.CRC_LEN + self.num_garbage_bytes)]
else:
return None

@data.setter
def data(self, data):
if self.pad_byte_flag == 0:
self.body = self.body[:self.SEC_HDR_LEN] + data + self.body[-self.CRC_LEN:]
if self.pkt_format == "1.2.1":
if self.pad_byte_flag == 0:
self.body = self.body[:self.SEC_HDR_LEN] + data + self.body[-self.CRC_LEN:]
else:
self.body = self.body[:self.SEC_HDR_LEN] + data + bytearray(1) + self.body[-self.CRC_LEN:]
else:
self.body = self.body[:self.SEC_HDR_LEN] + data + bytearray(1) + self.body[-self.CRC_LEN:]
self.body = self.body[:self.SEC_HDR_LEN] + data + self.body[-(self.CRC_LEN + self.num_garbage_bytes):]

@property
def coarse_time(self):
Expand All @@ -169,7 +177,7 @@ def coarse_time(self):
else:
logging.error(
f"Insufficient data length {len(self.body)} to extract coarse time "
f"from EngineeringDataPacket. Returning default value: {t}"
f"from ScienceDataPacket. Returning default value: {t}"
)

return t
Expand All @@ -182,7 +190,7 @@ def fine_time(self):
else:
logging.error(
f"Insufficient data length {len(self.body)} to extract fine time "
f"from EngineeringDataPacket. Returning default value: {t}"
f"from ScienceDataPacket. Returning default value: {t}"
)

return t
Expand All @@ -203,25 +211,47 @@ def subheader_id(self):
else:
logging.error(
f"Insufficient data length {len(self.body)} to extract subheader id "
f"from EngineeringDataPacket. Returning default value: {shid}"
f"from ScienceDataPacket. Returning default value: {shid}"
)

return shid

@property
def is_valid(self):
""""""
crc = int.from_bytes(self.body[-self.CRC_LEN:], "big")
calc_crc = zlib.crc32(self.hdr_data + self.body[:-self.CRC_LEN])
return calc_crc == crc
def real_pkt_data_len(self):
if self.pkt_format == "1.2.1":
return None
else:
rpdl = -1
if len(self.body) >= 13:
rpdl = int.from_bytes(self.body[11:13], "big")
else:
logging.error(
f"Insufficient data length {len(self.body)} to extract real packet data length "
f"from ScienceDataPacket. Returning default value: {rpdl}"
)

return rpdl

@property
def num_garbage_bytes(self):
if self.pkt_format == "1.2.1":
return None
else:
return self.pkt_data_len - self.real_pkt_data_len

@property
def payload_data(self):
def is_valid(self):
""""""
if len(self.body) >= self.SEC_HDR_LEN + self.CRC_LEN:
return self.body[self.SEC_HDR_LEN: -self.CRC_LEN]
if self.pkt_format == "1.2.1":
crc = int.from_bytes(self.body[-self.CRC_LEN:], "big")
calc_crc = zlib.crc32(self.hdr_data + self.body[:-self.CRC_LEN])
else:
return bytearray()
if self.num_garbage_bytes == 0:
crc = int.from_bytes(self.body[-self.CRC_LEN:], "big")
else:
crc = int.from_bytes(self.body[-(self.CRC_LEN + self.num_garbage_bytes): -self.num_garbage_bytes], "big")
calc_crc = zlib.crc32(self.hdr_data + self.body[:-(self.CRC_LEN + self.num_garbage_bytes)])
return calc_crc == crc

@property
def is_header_packet(self):
Expand All @@ -245,8 +275,10 @@ def product_length(self):
def __repr__(self):
pkt_str = "<CCSDSPacket: pkt_ver_num={} pkt_type={} apid={} pkt_data_len={} ".format(
self.pkt_ver_num, self.pkt_type, self.apid, self.pkt_data_len)
pkt_str += "coarse_time={} fine_time{} seq_flags={} pkt_seq_cnt={}>".format(
self.coarse_time, self.fine_time, self.seq_flags, self.pkt_seq_cnt)
pkt_str += "coarse_time={} fine_time={} seq_flags={} pkt_seq_cnt={} is_valid={} ".format(
self.coarse_time, self.fine_time, self.seq_flags, self.pkt_seq_cnt, self.is_valid)
pkt_str += "pkt_data_len={} real_pkt_data_len={} num_garbage_bytes={}>".format(
self.pkt_data_len, self.real_pkt_data_len, self.num_garbage_bytes)
return pkt_str


Expand Down Expand Up @@ -357,14 +389,12 @@ def __str__(self):
class SciencePacketProcessor:

HEADER_SYNC_WORD = bytes.fromhex("81FFFF81")
SEC_HDR_LEN = 11
MIN_PROCABLE_PKT_LEN = 8
CRC_LEN = 4
MAX_DATA_LEN = 1479

def __init__(self, stream_path):
logger.debug(f"Initializing SciencePacketProcessor from path {stream_path}")
def __init__(self, stream_path, pkt_format="1.3"):
logger.debug(f"Initializing SciencePacketProcessor from path {stream_path} using FSW v{pkt_format}")
self.stream = open(stream_path, "rb")
self.pkt_format = pkt_format
self._cur_psc = -1
self._cur_coarse = -1
self._cur_fine = -1
Expand Down Expand Up @@ -392,7 +422,8 @@ def read_frame(self):

def _read_next_packet(self):
while True:
pkt = ScienceDataPacket(stream=self.stream)
pkt = ScienceDataPacket(stream=self.stream, pkt_format=self.pkt_format)
logger.debug(pkt)
self._stats.ccsds_read(pkt)
pkt_hash = str(pkt.coarse_time) + str(pkt.fine_time) + str(pkt.pkt_seq_cnt)

Expand Down Expand Up @@ -516,14 +547,19 @@ def _read_pkt_parts(self, start_pkt):
# TODO: This block is probably never executed since the start packet usually contains only the header and
# nothing more
if expected_frame_len < len(start_pkt.data):
logger.info("Creating partial packet - frame length is less than the length of the packet's data.")
# Create a partial and then read in short frame
partial_data = start_pkt.data[expected_frame_len:]
# TODO: Add garbage byte here?
if start_pkt.pad_byte_flag == 0:
body = start_pkt.body[:self.SEC_HDR_LEN] + partial_data + start_pkt.body[-self.CRC_LEN:]
if self.pkt_format == "1.2.1":
if start_pkt.pad_byte_flag == 0:
body = start_pkt.body[:start_pkt.SEC_HDR_LEN] + partial_data + start_pkt.body[-start_pkt.CRC_LEN:]
else:
body = start_pkt.body[:start_pkt.SEC_HDR_LEN] + partial_data + bytearray(1) + \
start_pkt.body[-start_pkt.CRC_LEN:]
else:
body = start_pkt.body[:self.SEC_HDR_LEN] + partial_data + bytearray(1) + start_pkt.body[-self.CRC_LEN:]
partial = ScienceDataPacket(hdr_data=start_pkt.hdr_data, body=body)
body = start_pkt.body[:start_pkt.SEC_HDR_LEN] + partial_data + \
start_pkt.body[-(start_pkt.CRC_LEN + start_pkt.num_garbage_bytes):]
partial = ScienceDataPacket(hdr_data=start_pkt.hdr_data, body=body, pkt_format=self.pkt_format)
self._pkt_partial = partial

start_pkt.data = start_pkt.data[:expected_frame_len]
Expand Down Expand Up @@ -564,22 +600,27 @@ def _read_pkt_parts(self, start_pkt):
logger.debug("Case 2 - accumulated data length exceeds expected length. Trimming last packet to "
"expected size and creating partial packet of remaining bytes.")
# Create new partial
logger.info("Creating partial packet - the accum data length is greater than the expected frame length")
remaining_bytes = data_accum_len - expected_frame_len
partial_data = pkt_parts[-1].data[-remaining_bytes:]
# TODO: Add pad byte?
if pkt_parts[-1].pad_byte_flag == 0:
body = pkt_parts[-1].body[:self.SEC_HDR_LEN] + partial_data + pkt_parts[-1].body[-self.CRC_LEN:]
if self.pkt_format == "1.2.1":
if pkt_parts[-1].pad_byte_flag == 0:
body = pkt_parts[-1].body[:pkt_parts[-1].SEC_HDR_LEN] + partial_data + \
pkt_parts[-1].body[-pkt_parts[-1].CRC_LEN:]
else:
body = pkt_parts[-1].body[:pkt_parts[-1].SEC_HDR_LEN] + partial_data + bytearray(1) + \
pkt_parts[-1].body[-pkt_parts[-1].CRC_LEN:]
else:
body = pkt_parts[-1].body[:self.SEC_HDR_LEN] + partial_data + bytearray(1) + \
pkt_parts[-1].body[-self.CRC_LEN:]
partial = ScienceDataPacket(hdr_data=pkt_parts[-1].hdr_data, body=body)
body = pkt_parts[-1].body[:pkt_parts[-1].SEC_HDR_LEN] + partial_data + \
pkt_parts[-1].body[-(pkt_parts[-1].CRC_LEN + pkt_parts[-1].num_garbage_bytes):]
partial = ScienceDataPacket(hdr_data=pkt_parts[-1].hdr_data, body=body, pkt_format=self.pkt_format)
self._pkt_partial = partial

# Remove extra data from last packet in packet parts
pkt_parts[-1].data = pkt_parts[-1].data[:-remaining_bytes]
return pkt_parts

# If neither of the above end cases is met, then read the next packet
# If neither of the above end cases is met, then read the next packet
try:
pkt = self._read_next_packet()
except PSCMismatchException as e:
Expand All @@ -599,17 +640,35 @@ def _read_pkt_parts(self, start_pkt):
if remaining_data_len == 0:
logger.info(f"Not inserting any more garbage packets because end of frame.")
break
elif remaining_data_len >= self.MAX_DATA_LEN:
body = pkt.body[:self.SEC_HDR_LEN] + bytearray(self.MAX_DATA_LEN) + pkt.body[-self.CRC_LEN:]
garbage_pkt = ScienceDataPacket(hdr_data=pkt.hdr_data, body=body)
elif remaining_data_len >= pkt.MAX_DATA_LEN:
if self.pkt_format == "1.2.1":
if pkt.pad_byte_flag == 0:
body = pkt.body[:pkt.SEC_HDR_LEN] + bytearray(pkt.MAX_DATA_LEN) + \
pkt.body[-pkt.CRC_LEN:]
else:
body = pkt.body[:pkt.SEC_HDR_LEN] + bytearray(pkt.MAX_DATA_LEN) + bytearray(1) + \
pkt.body[-pkt.CRC_LEN:]
else:
body = pkt.body[:pkt.SEC_HDR_LEN] + bytearray(pkt.MAX_DATA_LEN) + \
pkt.body[-(pkt.CRC_LEN + pkt.num_garbage_bytes):]
garbage_pkt = ScienceDataPacket(hdr_data=pkt.hdr_data, body=body, pkt_format=self.pkt_format)
pkt_parts.append(garbage_pkt)
data_accum_len += self.MAX_DATA_LEN
logger.info(f"Inserted garbage packet with {self.MAX_DATA_LEN} bytes of data. Accum data is "
data_accum_len += pkt.MAX_DATA_LEN
logger.info(f"Inserted garbage packet with {pkt.MAX_DATA_LEN} bytes of data. Accum data is "
f"now {data_accum_len}")
self._stats.corrupt_frame(frame)
elif 0 < remaining_data_len < self.MAX_DATA_LEN:
body = pkt.body[:self.SEC_HDR_LEN] + bytearray(remaining_data_len) + pkt.body[-self.CRC_LEN:]
garbage_pkt = ScienceDataPacket(hdr_data=pkt.hdr_data, body=body)
elif 0 < remaining_data_len < pkt.MAX_DATA_LEN:
if self.pkt_format == "1.2.1":
if pkt.pad_byte_flag == 0:
body = pkt.body[:pkt.SEC_HDR_LEN] + bytearray(remaining_data_len) + \
pkt.body[-pkt.CRC_LEN:]
else:
body = pkt.body[:pkt.SEC_HDR_LEN] + bytearray(remaining_data_len) + bytearray(1) + \
pkt.body[-pkt.CRC_LEN:]
else:
body = pkt.body[:pkt.SEC_HDR_LEN] + bytearray(remaining_data_len) + \
pkt.body[-(pkt.CRC_LEN + pkt.num_garbage_bytes):]
garbage_pkt = ScienceDataPacket(hdr_data=pkt.hdr_data, body=body, pkt_format=self.pkt_format)
pkt_parts.append(garbage_pkt)
data_accum_len += remaining_data_len
logger.info(f"Inserted garbage packet with {remaining_data_len} bytes of data. Accum data is "
Expand Down
9 changes: 9 additions & 0 deletions emit_sds_l1a/frame.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,15 @@
0x00, 0x02, 0x7c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x35, 0x20, 0x20, 0x20,
0x20, 0x30, 0x38, 0x37, 0x20, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x9a, 0xfe, 0x8c, 0x8c, 0x8c,
0x8c, 0xef])
},
"warm_img_row0_row327_not_flight": {
"desc": "Older version of Nominal Warm FPA used in testing",
"roic_values":
bytearray([0xc3, 0x34, 0x00, 0x00, 0x47, 0x01, 0x01, 0x00, 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x01, 0x02, 0x8c, 0x02, 0x02, 0x0e,
0x00, 0x02, 0x7c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x35, 0x20, 0x20, 0x20,
0x20, 0x30, 0x38, 0x37, 0x20, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x9a, 0xfe, 0x8c, 0x8c, 0x8c,
0x8c, 0xef])
}
}

Expand Down
Loading

0 comments on commit ecdefff

Please sign in to comment.