BINEX: fix CRC16 calculations (#276)

* CRC verification * Fix CRC16 calc * Update docs * ClosedSourceMeta: - the decoding process will need it --------- Signed-off-by: Guillaume W. Bres <[email protected]>
georust · Nov 2, 2024 · 3a68b28 · 3a68b28
1 parent d6ed7be
commit 3a68b28
Show file tree

Hide file tree

Showing 14 changed files with 564 additions and 183 deletions.
diff --git a/binex/Cargo.toml b/binex/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "binex"
-version = "0.2.0"
+version = "0.3.0"
 license = "MIT OR Apache-2.0"
 authors = ["Guillaume W. Bres <[email protected]>"]
 description = "BINEX Binary RINEX encoder and decoder"

diff --git a/binex/README.md b/binex/README.md
@@ -1,9 +1,7 @@
 # BINEX
 
 [![Rust](https://github.com/georust/rinex/actions/workflows/rust.yml/badge.svg)](https://github.com/georust/rinex/actions/workflows/rust.yml)
-[![Rust](https://github.com/georust/rinex/actions/workflows/daily.yml/badge.svg)](https://github.com/georust/rinex/actions/workflows/daily.yml)
-[![crates.io](https://img.shields.io/crates/v/binex.svg)](https://crates.io/crates/binex)
-[![crates.io](https://docs.rs/binex/badge.svg)](https://docs.rs/binex/badge.svg)
+[![Rust](https://github.com/georust/rinex/actions/workflows/daily.yml/badge.svg)](https://github.com/georust/rinex/actions/workflows/daily.yml) [![crates.io](https://img.shields.io/crates/v/binex.svg)](https://crates.io/crates/binex) [![crates.io](https://docs.rs/binex/badge.svg)](https://docs.rs/binex/badge.svg)
 
 BINEX is a simple library to decode and encode BINEX messages.  
 BINEX stands for BINary EXchange and is the "real time" stream oriented
@@ -16,14 +14,70 @@ hardware orientated (at the GNSS receiver firmware level).
 This library allows easy message encoding and decoding, and aims at providing seamless
 convertion from RINEX back and forth.
 
-## Message Decoding
+You have two scenarios to approach a BINEX stream:
 
-Use the BINEX `Decoder` to decode messages from a `Readable` interface:
+* use our Decoder object, which works on I/O interface directly
+and can represent a stream of continuous of either Messages (open source)
+or undisclosed elements. (private prototypes)
+
+* or use Message::decode to work on your own buffer directly.
+
+Message Decoding
+================
+
+Use the BINEX `Decoder` to decode a `Readable` interface streaming
+BINEX messages. Decoder exposes both open source Messages that
+were fully interprated and closed source Messages (undisclosed prototypes)
+that it cannot interprate:
 
 ```rust
+use std::fs::File;
+use binex::prelude::{Decoder, StreamElement, Provider, Error};
+
+let fd = File::open("../test_resources/BIN/mfle20190130.bnx")
+    .unwrap();
+
+let mut decoder = Decoder::new(fd);
+
+loop {
+    match decoder.next() {
+        Some(Ok(StreamElement::OpenSource(msg))) => {
+            // fully interprated element
+        },
+        Some(Ok(StreamElement::ClosedSource(element))) => {
+            // verify this is your organization
+            if element.meta.provider == Provider::JPL {
+                // grab fields that you probably need to decode
+                let mid = element.meta.mid;
+                let mlen = element.meta.mlen;
+                let big_endian = element.meta.big_endian;
+                let is_reversed = element.meta.reversed;
+                let enhanced_crc = element.meta.enhanced_crc;
+
+                // now, proceed to interpretation of this element,
+                // using undisclosed method
+                element.interprate(&|data| {
+                    match mid {
+                        _ => {},
+                    }
+                });
+            }
+        },
+        Some(Err(e)) => {
+            // it is possible that some frames may not
+            // be supported yet.
+            // Any I/O error should not happen.
+        },
+        None => {
+            // end of stream
+            break;
+        },
+    }
+}
 ```
 
-## Message forging
+Message Forging
+===============
 
 The BINEX library allows easy message forging. Each message can be easily encoded and then
 streamed into a `Writable` interface:

diff --git a/binex/src/decoder.rs b/binex/src/decoder.rs
@@ -236,6 +236,29 @@ impl<'a, R: Read> Iterator for Decoder<'a, R> {
                             return Some(Err(Error::IncompleteMessage(mlen)));
                         }
                     },
+                    Error::ClosedSourceMessage(meta) => {
+                        // determine whether
+                        // - this element is self sustained (ie., fully described by this meta)
+                        // - the followup of previous elements
+                        // - or the last element of a serie
+                        if self.rd_ptr + meta.mlen < 4096 {
+                            // content is fully wrapped in buffer: expose as is
+                            // self.past_element = Some(ClosedSourceElement {
+                            //     provider: meta.provider,
+                            //     size: meta.mlen,
+                            //     total: meta.mlen,
+                            //     raw: self.buf[self.rd_ptr..self.rd_ptr +meta.mlen],
+                            // });
+                        } else {
+                            // content is not fully wrapped up here;
+                            // initiate or continue a serie of undisclosed element
+                        }
+                        return Some(Err(Error::IncompleteMessage(meta.mlen)));
+                    },
+                    Error::UnknownMessage => {
+                        // panic!("unknown message\nrd_ptr={}\nbuf={:?}", self.rd_ptr, &self.buf[self.rd_ptr-1..self.rd_ptr+4]);
+                        self.rd_ptr += 1;
+                    },
                     _ => {
                         // bad content that does not look like valid BINEX.
                         // This is very inefficient. If returned error would increment

diff --git a/binex/src/lib.rs b/binex/src/lib.rs
@@ -19,7 +19,7 @@ pub mod prelude {
             MonumentGeoMetadata, MonumentGeoRecord, Record, SBASEphemeris, TimeResolution,
         },
         stream::{ClosedSourceElement, Provider, StreamElement},
-        Error,
+        ClosedSourceMeta, Error,
     };
     // re-export
     pub use hifitime::Epoch;
@@ -32,12 +32,18 @@ use crate::stream::Provider;
 pub struct ClosedSourceMeta {
     // decoded MID (as is)
     pub mid: u32,
-    // decoded MLEN (as is)
-    pub mlen: u32,
-    // payload offset in buffer
-    pub offset: usize,
-    // [Provider] of this message. Only this organization may continue the decoding process.
+    /// decoded MLEN (as is)
+    pub mlen: usize,
+    /// Whether this item is reversed or not
+    pub reversed: bool,
+    /// Whether this item uses enhanced CRC or not
+    pub enhanced_crc: bool,
+    /// Whether this is big endian encoded or not
+    pub big_endian: bool,
+    /// [Provider] of this message. Only this organization may continue the decoding process.
     pub provider: Provider,
+    // payload offset in buffer
+    offset: usize,
 }
 
 #[derive(Debug)]
@@ -67,7 +73,7 @@ pub enum Error {
     CorrupctBadCRC,
     /// Incomplete message: need more data to complete
     IncompleteMessage(usize),
-    /// Library limitation: not all open source [Message]s supported yet
+    /// Library limitation: not all open source Messages supported yet
     NonSupportedMesssage(usize),
     /// Library limtation: should never happen, because this library
     /// will be designed to parse all open source [Message]s.

diff --git a/binex/src/message/checksum.rs b/binex/src/message/checksum.rs
@@ -112,17 +112,17 @@ impl Checksum {
     }
     /// Helper to decode checksum value as unsigned 128,
     /// which covers all scenarios
-    pub fn decode(&self, slice: &[u8], len: usize, big_endian: bool) -> u128 {
-        if len == 1 {
+    pub fn decode(&self, slice: &[u8], ck_len: usize, big_endian: bool) -> u128 {
+        if ck_len == 1 {
             slice[0] as u128
-        } else if len == 2 {
+        } else if ck_len == 2 {
             let val_u16 = if big_endian {
                 u16::from_be_bytes([slice[0], slice[1]])
             } else {
                 u16::from_le_bytes([slice[0], slice[1]])
             };
             val_u16 as u128
-        } else if len == 4 {
+        } else if ck_len == 4 {
             let val_u32 = if big_endian {
                 u32::from_be_bytes([slice[0], slice[1], slice[2], slice[3]])
             } else {
@@ -134,12 +134,12 @@ impl Checksum {
         }
     }
     /// Calculates expected Checksum for this msg
-    pub fn calc(&self, bytes: &[u8], mlen: usize) -> u128 {
+    pub fn calc(&self, bytes: &[u8], size: usize) -> u128 {
         match self {
-            Self::XOR8 => Self::xor8_calc(bytes, mlen),
-            Self::XOR16 => Self::xor16_calc(bytes),
-            Self::XOR32 => Self::xor32_calc(bytes),
-            Self::MD5 => Self::md5_calc(bytes),
+            Self::XOR8 => Self::xor8_calc(bytes, size),
+            Self::XOR16 => Self::xor16_calc(bytes, size),
+            Self::XOR32 => Self::xor32_calc(bytes, size),
+            Self::MD5 => Self::md5_calc(bytes, size),
         }
     }
     /// Calculates expected Checksum using XOR8 algorithm
@@ -151,29 +151,29 @@ impl Checksum {
         xor as u128
     }
     /// Calculates expected Checksum using XOR16 algorithm
-    fn xor16_calc(bytes: &[u8]) -> u128 {
-        let mut crc = 0xffff_u16;
-        for byte in bytes.iter() {
-            let tmp = (*byte as u16) ^ crc;
-            crc >>= 8;
-            crc ^= CRC16_TABLE[(tmp as usize) % 256];
+    fn xor16_calc(bytes: &[u8], size: usize) -> u128 {
+        let mut crc = 0_u16;
+        for i in 0..size {
+            let index = (((crc >> 8) ^ bytes[i] as u16) & 0xff) as usize;
+            crc = (crc << 8) ^ CRC16_TABLE[index];
+            crc &= 0xffff;
         }
         crc as u128
     }
     /// Calculates expected Checksum using XO32 algorithm
-    fn xor32_calc(bytes: &[u8]) -> u128 {
-        let mut crc = 0xffffffff_u32;
-        for byte in bytes.iter() {
-            let tmp = (*byte as u32) ^ crc;
-            crc >>= 8;
-            crc ^= CRC32_TABLE[(tmp as usize) % 256];
+    fn xor32_calc(bytes: &[u8], size: usize) -> u128 {
+        let mut crc = 0_u32;
+        for i in 0..size {
+            let index = (((crc >> 24) ^ bytes[i] as u32) & 0xff) as usize;
+            crc = (crc << 8) ^ CRC32_TABLE[index];
+            crc &= 0xffffffff;
         }
         crc as u128
     }
     /// Calculates expected Checksum using MD5 algorithm
-    fn md5_calc(bytes: &[u8]) -> u128 {
+    fn md5_calc(bytes: &[u8], size: usize) -> u128 {
         let mut hasher = Md5::new();
-        hasher.update(bytes);
+        hasher.update(&bytes[..size]);
         let md5 = hasher.finalize();
         u128::from_le_bytes(md5.into())
     }
@@ -186,5 +186,91 @@ mod test {
     fn test_xor8() {
         let buf = [0, 1, 2, 3, 4];
         assert_eq!(Checksum::XOR8.calc(&buf, 5), 4);
+
+        let buf = [
+            0x00, 0x1f, 0x01, 0x39, 0x87, 0x20, 0x00, 0x00, 0x00, 0x17, 0x42, 0x49, 0x4e, 0x45,
+            0x58, 0x20, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x20, 0x52, 0x65, 0x73, 0x74, 0x61,
+            0x72, 0x74, 0x65, 0x64, 0x21,
+        ];
+        assert_eq!(Checksum::XOR8.calc(&buf, buf.len()), 0x84);
+    }
+
+    #[test]
+    fn test_xor16() {
+        // e2, 01 81
+        let buf = [
+            0x01, 0x81, 0x00, 0x01, 0x1d, 0x07, 0xf6, 0x00, 0x03, 0xd8, 0x72, 0x00, 0x03, 0xf4,
+            0x80, 0x31, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0xac,
+            0xdc, 0x00, 0x00, 0xb8, 0x38, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x20, 0x30, 0xd5, 0x5c,
+            0x00, 0xbf, 0xf8, 0x96, 0x4c, 0x65, 0x6e, 0xda, 0x41, 0x3f, 0x6d, 0x97, 0xd5, 0xd0,
+            0x00, 0x00, 0x00, 0x40, 0xb4, 0x21, 0xa2, 0x39, 0x40, 0x00, 0x00, 0x32, 0x20, 0x00,
+            0x00, 0x43, 0x44, 0xf8, 0x00, 0xb3, 0x18, 0x00, 0x00, 0x42, 0x78, 0x60, 0x00, 0x36,
+            0x49, 0xa0, 0x00, 0x37, 0x16, 0x60, 0x00, 0x40, 0x02, 0xa8, 0x2c, 0x0b, 0x2a, 0x18,
+            0x0c, 0xc0, 0x08, 0x23, 0xb8, 0x97, 0xbd, 0xf9, 0x99, 0x3f, 0xee, 0x23, 0x55, 0xce,
+            0x2e, 0x11, 0x70, 0xb1, 0x31, 0xa4, 0x00, 0xad, 0xac, 0x00, 0x00, 0x41, 0xa0, 0x00,
+            0x00, 0x00, 0x00, 0x02, 0x04,
+        ];
+
+        assert_eq!(Checksum::XOR16.calc(&buf, buf.len()), 0x7d49);
+
+        let buf = [
+            0x01, 0x81, 0x00, 0x01, 0x07, 0x07, 0xf6, 0x00, 0x03, 0xd8, 0x72, 0x00, 0x03, 0xf4,
+            0x80, 0x31, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, 0x00, 0xab,
+            0xc0, 0x00, 0x00, 0xb9, 0x09, 0x3b, 0x60, 0x00, 0x00, 0x00, 0x1d, 0x30, 0xc3, 0x30,
+            0x00, 0x3f, 0xf9, 0xa2, 0xc9, 0x26, 0x53, 0xc2, 0x7b, 0x3f, 0x71, 0x2c, 0xe0, 0xd8,
+            0x00, 0x00, 0x00, 0x40, 0xb4, 0x21, 0xb0, 0xf1, 0x60, 0x00, 0x00, 0x33, 0xa0, 0x00,
+            0x00, 0x43, 0x98, 0x64, 0x00, 0xb2, 0x60, 0x00, 0x00, 0xc2, 0x2f, 0xa0, 0x00, 0xb6,
+            0x0c, 0xe0, 0x00, 0x36, 0x83, 0xc0, 0x00, 0xbf, 0xfe, 0xa8, 0x9a, 0xfb, 0x49, 0x69,
+            0xb2, 0xbf, 0xd7, 0x73, 0x3f, 0x12, 0x4a, 0xa8, 0x69, 0x3f, 0xef, 0x09, 0xab, 0xae,
+            0x21, 0x65, 0xd4, 0xb1, 0x33, 0xe8, 0x00, 0xae, 0xa1, 0xc0, 0x00, 0x41, 0xa0, 0x00,
+            0x00, 0x00, 0x00, 0x02, 0x04,
+        ];
+
+        assert_eq!(Checksum::XOR16.calc(&buf, buf.len()), 0x6c23);
+
+        let buf = [
+            0x01, 0x81, 0x00, 0x01, 0x06, 0x07, 0xf6, 0x00, 0x03, 0xd8, 0x72, 0x00, 0x03, 0xf4,
+            0x80, 0xb2, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, 0x00, 0xac,
+            0xfc, 0x00, 0x00, 0x38, 0x31, 0xab, 0x80, 0x00, 0x00, 0x00, 0x53, 0x30, 0xc5, 0x24,
+            0x00, 0xbf, 0xf8, 0xbb, 0x57, 0x3d, 0x09, 0x5f, 0x9d, 0x3f, 0x88, 0xee, 0x38, 0x68,
+            0x00, 0x00, 0x00, 0x40, 0xb4, 0x21, 0x9d, 0xac, 0xc0, 0x00, 0x00, 0x34, 0x5a, 0x00,
+            0x00, 0x43, 0x48, 0x50, 0x00, 0xb3, 0xe4, 0x00, 0x00, 0x42, 0x67, 0x40, 0x00, 0x36,
+            0x42, 0x80, 0x00, 0x37, 0x16, 0xe8, 0x00, 0x40, 0x02, 0x6a, 0xdc, 0xf8, 0x6b, 0x10,
+            0x56, 0xc0, 0x03, 0xd3, 0x59, 0xfa, 0x22, 0x6d, 0x54, 0x3f, 0xee, 0x99, 0xf5, 0x34,
+            0x32, 0xf3, 0xdd, 0xb1, 0x2b, 0xf6, 0x00, 0xac, 0xe8, 0x00, 0x00, 0x41, 0xa0, 0x00,
+            0x00, 0x00, 0x00, 0x02, 0x04,
+        ];
+
+        assert_eq!(Checksum::XOR16.calc(&buf, buf.len()), 0x1919);
+
+        let buf = [
+            0x01, 0x81, 0x00, 0x01, 0x11, 0x07, 0xf6, 0x00, 0x03, 0xe5, 0x74, 0x00, 0x03, 0xf4,
+            0x80, 0xb1, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x00, 0x2c,
+            0x84, 0x00, 0x00, 0x37, 0xae, 0x23, 0x00, 0x00, 0x00, 0x00, 0x68, 0x30, 0xc9, 0xa4,
+            0x00, 0xbf, 0xf2, 0x1b, 0xb8, 0xd2, 0xf3, 0x07, 0xe2, 0x3f, 0x8e, 0xbe, 0xca, 0x08,
+            0x00, 0x00, 0x00, 0x40, 0xb4, 0x21, 0xbe, 0x8a, 0x80, 0x00, 0x00, 0xb3, 0xc0, 0x00,
+            0x00, 0x43, 0x53, 0x30, 0x00, 0x34, 0xb4, 0x00, 0x00, 0xc2, 0x64, 0xa0, 0x00, 0xb6,
+            0x3f, 0xa0, 0x00, 0x37, 0x0e, 0xf8, 0x00, 0xbf, 0xec, 0xde, 0x8f, 0x8b, 0x25, 0x86,
+            0x86, 0x3f, 0xf5, 0xf8, 0xf4, 0xf0, 0x6d, 0x38, 0x8e, 0x3f, 0xee, 0x7c, 0xb1, 0xdf,
+            0xad, 0x10, 0xdf, 0xb1, 0x34, 0x46, 0x00, 0xaa, 0x80, 0x00, 0x00, 0x41, 0xa0, 0x00,
+            0x00, 0x00, 0x00, 0x02, 0x04,
+        ];
+
+        assert_eq!(Checksum::XOR16.calc(&buf, buf.len()), 0x72d8);
+
+        let buf = [
+            0x01, 0x81, 0x00, 0x01, 0x00, 0x07, 0xf6, 0x00, 0x03, 0xf2, 0x6a, 0x00, 0x03, 0xf4,
+            0x80, 0x31, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, 0x00, 0x00, 0xac,
+            0xec, 0x00, 0x00, 0xb9, 0x21, 0x70, 0x60, 0x00, 0x00, 0x00, 0x5b, 0x30, 0xb1, 0xd4,
+            0x00, 0xbf, 0xed, 0x6f, 0x04, 0x75, 0x35, 0xbf, 0x7c, 0x3f, 0x81, 0x09, 0xb5, 0x7c,
+            0x00, 0x00, 0x00, 0x40, 0xb4, 0x21, 0xa9, 0xec, 0xa0, 0x00, 0x00, 0x34, 0x3c, 0x00,
+            0x00, 0x43, 0x55, 0xd8, 0x00, 0x33, 0xec, 0x00, 0x00, 0xc2, 0x6f, 0xc0, 0x00, 0xb6,
+            0x55, 0x80, 0x00, 0x37, 0x16, 0x80, 0x00, 0xbf, 0xeb, 0x2f, 0xea, 0x1a, 0x2d, 0x94,
+            0x4f, 0x3f, 0xe5, 0xf2, 0x7f, 0x44, 0xff, 0xc4, 0xc1, 0x3f, 0xef, 0x2d, 0x2d, 0x85,
+            0x03, 0xc4, 0xfb, 0xb1, 0x2c, 0x40, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x41, 0xa0, 0x00,
+            0x00, 0x00, 0x00, 0x02, 0x04,
+        ];
+
+        assert_eq!(Checksum::XOR16.calc(&buf, buf.len()), 0x5376);
     }
 }