From 53b90e9c96ea03372d6fc00d9662e199cd4c5f18 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Wed, 1 Jan 2025 17:14:08 -0500 Subject: [PATCH 1/8] Update TLS file paths, installation guide, and dependencies Simplified TLS file paths by removing unnecessary relative segments. Improved the README installation guide for clarity and updated instructions for key generation. Bumped Rust version to 1.83.0, updated crate dependencies, and removed unused `zeroize`. --- Cargo.toml | 6 ++---- README.md | 22 +++++++++++++--------- src/bin/main.rs | 6 +++--- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2ed1c77..e046e51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "arkavo-rs" version = "0.9.1" edition = "2021" -rust-version = "1.80.0" +rust-version = "1.83.0" [lib] name = "nanotdf" @@ -32,7 +32,6 @@ aes-gcm = "0.10.3" p256 = { version = "=0.13.2", features = ["ecdh"] } once_cell = "1.19.0" rand_core = "0.6.4" -zeroize = "1.8.1" sha2 = "0.10.8" hkdf = "0.12.4" tokio-native-tls = "0.3.1" @@ -41,8 +40,7 @@ env_logger = "0.11.5" log = "0.4.22" ink = "5.0.0" jsonwebtoken = "9.3.0" -async-nats = "0.36.0" -serde_json = "1.0.128" +async-nats = "0.38.0" redis = { version = "0.27.2", features = ["tokio-comp"] } flatbuffers = "24.3.25" scale = { package = "parity-scale-codec", version = "3.6.12", default-features = false, features = ["derive"] } diff --git a/README.md b/README.md index a8f761a..00ae12e 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,7 @@ flatc --binary --rust idl/metadata.fbs ### Installation -1. Clone the repository: - - ```shell - git clone https://github.com/arkavo-org/backend-rust.git - cd backend-rust - ``` - -2. Build the project to download and compile the dependencies: +1. Build the project to download and compile the dependencies: ```shell cargo build @@ -64,7 +57,18 @@ flatc --binary --rust idl/metadata.fbs openssl ec -in recipient_private_key.pem -text -noout ``` -2. Generating Self-Signed Certificate +2. Ensure you have a valid EC private key in PEM format named `recipient_private_key.pem`. + + ```shell + openssl ecparam -genkey -name prime256v1 -noout -out recipient_private_key.pem + ``` + + Validate + ```shell + openssl ec -in recipient_private_key.pem -text -noout + ``` + +3. Generating Self-Signed Certificate For development purposes, you can generate a self-signed certificate using OpenSSL. Run the following command in your terminal: diff --git a/src/bin/main.rs b/src/bin/main.rs index 6eb1ffc..a5c04e7 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -1162,21 +1162,21 @@ fn load_config() -> Result> { tls_enabled: env::var("TLS_CERT_PATH").is_ok(), tls_cert_path: env::var("TLS_CERT_PATH").unwrap_or_else(|_| { current_dir - .join("../../fullchain.pem") + .join("fullchain.pem") .to_str() .unwrap() .to_string() }), tls_key_path: env::var("TLS_KEY_PATH").unwrap_or_else(|_| { current_dir - .join("../../privkey.pem") + .join("privkey.pem") .to_str() .unwrap() .to_string() }), kas_key_path: env::var("KAS_KEY_PATH").unwrap_or_else(|_| { current_dir - .join("../../recipient_private_key.pem") + .join("recipient_private_key.pem") .to_str() .unwrap() .to_string() From ca80981f637a303cddb6f562a3d25cfe781da137 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Wed, 1 Jan 2025 19:06:39 -0500 Subject: [PATCH 2/8] Validate payload size and add detailed logging. Added a size validation check for event payloads to ensure they do not exceed the maximum allowed size. Introduced improved logging to display the first 20 bytes of the payload in hexadecimal format for debugging purposes. Gracefully handle parsing failures with error messages and early returns. --- src/bin/main.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/bin/main.rs b/src/bin/main.rs index a5c04e7..e3a8af9 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -875,6 +875,21 @@ async fn handle_event( nats_connection: Arc, ) -> Option { let start_time = Instant::now(); + println!( + "Payload (first 20 bytes in hex, space-delimited): {}", + payload + .iter() + .take(20) + .map(|byte| format!("{:02x}", byte)) + .collect::>() + .join(" ") + ); + // Size validation for type 0x06 + const MAX_EVENT_SIZE: usize = 2000; // Adjust this value as needed + if payload.len() > MAX_EVENT_SIZE { + error!("Event payload exceeds maximum allowed size of {} bytes", MAX_EVENT_SIZE); + return None; + } let mut event_data: Option> = None; if let Ok(event) = root::(payload) { println!("Event Action: {:?}", event.action()); @@ -920,6 +935,9 @@ async fn handle_event( } }; // TODO if cache miss then route to device + } else { + error!("Failed to parse user event from payload"); + return None; } } EventData::CacheEvent => { From f4a697d114856366be894980775d13fb724eb90f Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Wed, 1 Jan 2025 20:52:25 -0500 Subject: [PATCH 3/8] Adjust message type based on payload size in NATS handler. Introduce a payload size check to dynamically set the message type, addressing inconsistencies with large messages. This mitigates potential issues with handling message size and aligns processing with expected policies. --- src/bin/main.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/bin/main.rs b/src/bin/main.rs index e3a8af9..5a489a3 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -856,12 +856,22 @@ async fn handle_nats_subscription( tokio::time::sleep(NATS_RETRY_INTERVAL).await; } } + async fn handle_nats_event( msg: NatsMessage, connection_state: Arc, ) -> Result<(), Box> { + // workaround, not sure why video is switching to 0x06 + // may need to check the policy + const MAX_EVENT_SIZE: usize = 2000; + let message_type = if msg.payload.len() > MAX_EVENT_SIZE { + MessageType::Nats // 0x05 for large messages + } else { + MessageType::Event // 0x06 for small messages + }; + let ws_message = Message::Binary( - vec![MessageType::Event as u8] + vec![message_type as u8] .into_iter() .chain(msg.payload) .collect(), @@ -869,6 +879,7 @@ async fn handle_nats_event( connection_state.outgoing_tx.send(ws_message)?; Ok(()) } + async fn handle_event( server_state: &Arc, payload: &[u8], From 757c9c4e04ca9f4e59027628a4600e29ceb5b1a2 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Wed, 1 Jan 2025 22:04:53 -0500 Subject: [PATCH 4/8] Refactor NATS message handler function name. Renamed `handle_nats_event` to `handle_nats` for clarity and updated related logic. Simplified message type determination by replacing redundant comments and conditions. --- src/bin/main.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/bin/main.rs b/src/bin/main.rs index 5a489a3..450855d 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -840,7 +840,7 @@ async fn handle_nats_subscription( Ok(mut subscription) => { info!("Subscribed to NATS subject: {}", subject); while let Some(msg) = subscription.next().await { - if let Err(e) = handle_nats_event(msg, connection_state.clone()).await { + if let Err(e) = handle_nats(msg, connection_state.clone()).await { error!("Error handling NATS message: {}", e); } } @@ -857,17 +857,15 @@ async fn handle_nats_subscription( } } -async fn handle_nats_event( +async fn handle_nats( msg: NatsMessage, connection_state: Arc, ) -> Result<(), Box> { - // workaround, not sure why video is switching to 0x06 - // may need to check the policy - const MAX_EVENT_SIZE: usize = 2000; - let message_type = if msg.payload.len() > MAX_EVENT_SIZE { - MessageType::Nats // 0x05 for large messages + // it nanotdf, then do a message, otherwise it is a Flatbuffers event + let message_type = if msg.payload[0..3].iter().eq(&[0x4C, 0x31, 0x4C]) { + MessageType::Nats } else { - MessageType::Event // 0x06 for small messages + MessageType::Event }; let ws_message = Message::Binary( From b51aa2e9174b3ae7222a4f3102eed6c9a8bd1591 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Thu, 2 Jan 2025 21:38:42 -0500 Subject: [PATCH 5/8] Update worker threads config and clean up generated schemas Set `worker_threads` to 4 in the Tokio runtime initialization for better control over thread usage. Removed unnecessary attributes and redundant imports in generated FlatBuffers schema files, improving code maintainability and readability. --- src/bin/main.rs | 7 +++++-- src/bin/schemas/entity_generated.rs | 1 - src/bin/schemas/event_generated.rs | 12 ++++++++++-- src/bin/schemas/metadata_generated.rs | 13 +------------ 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/bin/main.rs b/src/bin/main.rs index 450855d..dfc7e94 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -154,7 +154,7 @@ static KAS_KEYS: OnceCell> = OnceCell::new(); trait AsyncStream: AsyncRead + AsyncWrite + Unpin + Send {} impl AsyncStream for T where T: AsyncRead + AsyncWrite + Unpin + Send {} -#[tokio::main(flavor = "multi_thread")] +#[tokio::main(flavor = "multi_thread", worker_threads = 4)] async fn main() -> Result<(), Box> { // Initialize logging env_logger::init(); @@ -896,7 +896,10 @@ async fn handle_event( // Size validation for type 0x06 const MAX_EVENT_SIZE: usize = 2000; // Adjust this value as needed if payload.len() > MAX_EVENT_SIZE { - error!("Event payload exceeds maximum allowed size of {} bytes", MAX_EVENT_SIZE); + error!( + "Event payload exceeds maximum allowed size of {} bytes", + MAX_EVENT_SIZE + ); return None; } let mut event_data: Option> = None; diff --git a/src/bin/schemas/entity_generated.rs b/src/bin/schemas/entity_generated.rs index 857fd2c..95ea727 100644 --- a/src/bin/schemas/entity_generated.rs +++ b/src/bin/schemas/entity_generated.rs @@ -1,5 +1,4 @@ // automatically generated by the FlatBuffers compiler, do not modify -#![allow(clippy::extra_unused_lifetimes)] // @generated diff --git a/src/bin/schemas/event_generated.rs b/src/bin/schemas/event_generated.rs index 91d4731..6282afb 100644 --- a/src/bin/schemas/event_generated.rs +++ b/src/bin/schemas/event_generated.rs @@ -1,13 +1,17 @@ // automatically generated by the FlatBuffers compiler, do not modify -#![allow(clippy::extra_unused_lifetimes)] + // @generated extern crate flatbuffers; #[allow(unused_imports, dead_code)] pub mod arkavo { + + use core::cmp::Ordering; + use core::mem; + extern crate flatbuffers; - use self::flatbuffers::Follow; + use self::flatbuffers::{EndianScalar, Follow}; #[deprecated( since = "2.0.0", @@ -677,6 +681,7 @@ pub mod arkavo { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; v.visit_table(pos)? .visit_field::("source_type", Self::VT_SOURCE_TYPE, false)? .visit_field::("target_type", Self::VT_TARGET_TYPE, false)? @@ -895,6 +900,7 @@ pub mod arkavo { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; v.visit_table(pos)? .visit_field::>>( "target_id", @@ -1140,6 +1146,7 @@ pub mod arkavo { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; v.visit_table(pos)? .visit_field::("target_type", Self::VT_TARGET_TYPE, false)? .visit_field::>>( @@ -1433,6 +1440,7 @@ pub mod arkavo { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; v.visit_table(pos)? .visit_field::("action", Self::VT_ACTION, false)? .visit_field::("timestamp", Self::VT_TIMESTAMP, false)? diff --git a/src/bin/schemas/metadata_generated.rs b/src/bin/schemas/metadata_generated.rs index 0c8b731..30c896b 100644 --- a/src/bin/schemas/metadata_generated.rs +++ b/src/bin/schemas/metadata_generated.rs @@ -1,18 +1,13 @@ // automatically generated by the FlatBuffers compiler, do not modify -#![allow(clippy::extra_unused_lifetimes)] -// @generated extern crate flatbuffers; #[allow(unused_imports, dead_code)] pub mod arkavo { - use crate::schemas::entity_generated::arkavo::MediaType; - use core::cmp::Ordering; - use core::mem; extern crate flatbuffers; - use self::flatbuffers::{EndianScalar, Follow}; + use self::flatbuffers::Follow; #[deprecated( since = "2.0.0", @@ -627,7 +622,6 @@ pub mod arkavo { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; v.visit_table(pos)? .visit_field::("violent", Self::VT_VIOLENT, false)? .visit_field::("sexual", Self::VT_SEXUAL, false)? @@ -904,7 +898,6 @@ pub mod arkavo { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; v.visit_table(pos)? .visit_field::("educational", Self::VT_EDUCATIONAL, false)? .visit_field::("entertainment", Self::VT_ENTERTAINMENT, false)? @@ -1114,7 +1107,6 @@ pub mod arkavo { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; v.visit_table(pos)? .visit_field::("type_", Self::VT_TYPE_, false)? .visit_field::>( @@ -1279,7 +1271,6 @@ pub mod arkavo { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; v.visit_table(pos)? .visit_field::("type_", Self::VT_TYPE_, false)? .visit_field::>( @@ -1446,7 +1437,6 @@ pub mod arkavo { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; v.visit_table(pos)? .visit_field::("media_type", Self::VT_MEDIA_TYPE, false)? .visit_field::("data_encoding", Self::VT_DATA_ENCODING, false)? @@ -1691,7 +1681,6 @@ pub mod arkavo { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; v.visit_table(pos)? .visit_field::("created", Self::VT_CREATED, false)? .visit_field::>>( From 2accdb17c63f5b39c5d6014e1efb95fd977c26d5 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Thu, 2 Jan 2025 21:42:51 -0500 Subject: [PATCH 6/8] Optimize build performance and update dependencies Increased `codegen-units` to 16 in release profile for faster builds. Updated `flatbuffers` to version 24.12.23 to include the latest enhancements and fixes. Ensured other dependencies remain consistent for compatibility. --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e046e51..a89ce6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ path = "src/bin/main.rs" [profile.release] opt-level = 3 lto = true -codegen-units = 1 +codegen-units = 16 [profile.bench] lto = true @@ -42,7 +42,7 @@ ink = "5.0.0" jsonwebtoken = "9.3.0" async-nats = "0.38.0" redis = { version = "0.27.2", features = ["tokio-comp"] } -flatbuffers = "24.3.25" +flatbuffers = "24.12.23" scale = { package = "parity-scale-codec", version = "3.6.12", default-features = false, features = ["derive"] } scale-info = { version = "2.11.3", default-features = false, features = ["derive"], optional = true } bs58 = "0.5.1" From e140cc10d8977cb1873b2d6febc304337e60472a Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Thu, 2 Jan 2025 22:00:15 -0500 Subject: [PATCH 7/8] Enable metadata debugging in embedded policy handling Added print statements to log metadata buffer size, contents, and processed metadata during embedded policy handling. These changes improve visibility and aid in debugging metadata-related issues. --- src/bin/main.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/bin/main.rs b/src/bin/main.rs index dfc7e94..f7e252d 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -523,8 +523,10 @@ async fn handle_rewrap( locator = policy.get_locator().clone(); } PolicyType::Embedded => { - // println!("embedded policy"); + println!("embedded policy"); if let Some(body) = &policy.body { + println!("Metadata buffer size: {}", body.len()); + println!("Metadata buffer contents: {:?}", body); metadata = match root_as_metadata(body) { Ok(metadata) => Some(metadata), Err(e) => { @@ -533,7 +535,7 @@ async fn handle_rewrap( } }; // TODO add contracts - // println!("metadata: {:#?}", metadata); + println!("metadata: {:#?}", metadata); } // add content rating contract let rl = ResourceLocator { From 5e8765f0a39379cf21d615500353ff040c19cd99 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Fri, 3 Jan 2025 18:26:31 -0500 Subject: [PATCH 8/8] Add Clippy allow directive to generated schema files This change adds `#![allow(clippy::all)]` to suppress Clippy lints in files generated by the FlatBuffers compiler. These files are auto-generated, so manual linting is unnecessary and could lead to noise during development. --- src/bin/schemas/entity_generated.rs | 1 + src/bin/schemas/event_generated.rs | 1 + src/bin/schemas/metadata_generated.rs | 1 + 3 files changed, 3 insertions(+) diff --git a/src/bin/schemas/entity_generated.rs b/src/bin/schemas/entity_generated.rs index 95ea727..5251d16 100644 --- a/src/bin/schemas/entity_generated.rs +++ b/src/bin/schemas/entity_generated.rs @@ -1,3 +1,4 @@ +#![allow(clippy::all)] // automatically generated by the FlatBuffers compiler, do not modify // @generated diff --git a/src/bin/schemas/event_generated.rs b/src/bin/schemas/event_generated.rs index 6282afb..5a7b288 100644 --- a/src/bin/schemas/event_generated.rs +++ b/src/bin/schemas/event_generated.rs @@ -1,3 +1,4 @@ +#![allow(clippy::all)] // automatically generated by the FlatBuffers compiler, do not modify // @generated diff --git a/src/bin/schemas/metadata_generated.rs b/src/bin/schemas/metadata_generated.rs index 30c896b..66e4542 100644 --- a/src/bin/schemas/metadata_generated.rs +++ b/src/bin/schemas/metadata_generated.rs @@ -1,3 +1,4 @@ +#![allow(clippy::all)] // automatically generated by the FlatBuffers compiler, do not modify extern crate flatbuffers;