From bdb22ed3f3ca69537f5972a762444023e2919110 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 13 Nov 2024 12:13:54 +0100 Subject: [PATCH 01/47] fix typo --- crates/store/re_protos/src/codec.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/store/re_protos/src/codec.rs b/crates/store/re_protos/src/codec.rs index 375722bfac8d..12a4b0a08382 100644 --- a/crates/store/re_protos/src/codec.rs +++ b/crates/store/re_protos/src/codec.rs @@ -32,16 +32,16 @@ pub enum CodecError { } #[derive(Clone, Copy, PartialEq, Eq, Hash, Default)] -pub struct MessageHader(pub u8); +pub struct MessageHeader(pub u8); -impl MessageHader { +impl MessageHeader { pub const NO_DATA: Self = Self(1); pub const RECORD_BATCH: Self = Self(2); pub const SIZE_BYTES: usize = 1; } -impl MessageHader { +impl MessageHeader { fn decode(read: &mut impl std::io::Read) -> Result { let mut buffer = [0_u8; Self::SIZE_BYTES]; read.read_exact(&mut buffer) @@ -72,12 +72,12 @@ impl TransportMessageV0 { match self { Self::NoData => { let mut data: Vec = Vec::new(); - MessageHader::NO_DATA.encode(&mut data)?; + MessageHeader::NO_DATA.encode(&mut data)?; Ok(data) } Self::RecordBatch(chunk) => { let mut data: Vec = Vec::new(); - MessageHader::RECORD_BATCH.encode(&mut data)?; + MessageHeader::RECORD_BATCH.encode(&mut data)?; write_arrow_to_bytes(&mut data, &chunk.schema, &chunk.data)?; @@ -88,11 +88,11 @@ impl TransportMessageV0 { fn from_bytes(data: &[u8]) -> Result { let mut reader = std::io::Cursor::new(data); - let header = MessageHader::decode(&mut reader)?; + let header = MessageHeader::decode(&mut reader)?; match header { - MessageHader::NO_DATA => Ok(Self::NoData), - MessageHader::RECORD_BATCH => { + MessageHeader::NO_DATA => Ok(Self::NoData), + MessageHeader::RECORD_BATCH => { let (schema, data) = read_arrow_from_bytes(&mut reader)?; let tc = TransportChunk { From 8b51ba81c45d03892bdc8fee680d97c8bf1a4495 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 26 Nov 2024 17:40:27 +0100 Subject: [PATCH 02/47] temp --- Cargo.lock | 10 +- crates/build/re_protos_builder/README.md | 2 +- .../src/bin/build_re_remote_store_types.rs | 14 +- crates/build/re_protos_builder/src/lib.rs | 18 +- crates/store/re_chunk_store/Cargo.toml | 3 +- crates/store/re_chunk_store/src/dataframe.rs | 19 +- crates/store/re_chunk_store/src/lib.rs | 8 +- crates/store/re_chunk_store/src/store.rs | 4 +- crates/store/re_dataframe/Cargo.toml | 3 +- crates/store/re_dataframe/src/engine.rs | 3 +- crates/store/re_dataframe/src/lib.rs | 3 +- crates/store/re_grpc_client/Cargo.toml | 3 +- crates/store/re_grpc_client/src/lib.rs | 7 +- crates/store/re_log_encoding/Cargo.toml | 2 + .../store/re_log_encoding/src/decoder/mod.rs | 39 +- crates/store/re_log_encoding/src/encoder.rs | 30 +- crates/store/re_log_encoding/src/lib.rs | 15 +- crates/store/re_log_types/Cargo.toml | 2 + crates/store/re_log_types/src/lib.rs | 70 ++ crates/store/re_protos/Cargo.toml | 4 - crates/store/re_protos/README.md | 2 +- .../re_protos/proto/rerun/v0/common.proto | 37 +- .../proto/rerun/v0/remote_store.proto | 43 +- crates/store/re_protos/src/lib.rs | 621 ++++++----------- .../re_protos/src/v0/rerun.remote_store.v0.rs | 639 +++++++----------- crates/top/rerun/src/lib.rs | 3 +- pixi.toml | 6 +- rerun_py/Cargo.toml | 1 + rerun_py/src/dataframe.rs | 3 +- 29 files changed, 716 insertions(+), 898 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5d7e947a821..d7dc5fe19468 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5513,6 +5513,7 @@ dependencies = [ "re_log", "re_log_encoding", "re_log_types", + "re_protos", "re_tracing", "re_types", "re_types_core", @@ -5679,6 +5680,7 @@ dependencies = [ "re_chunk", "re_chunk_store", "re_log", + "re_log_encoding", "re_log_types", "re_query", "re_tracing", @@ -5781,6 +5783,7 @@ dependencies = [ "re_chunk", "re_error", "re_log", + "re_log_encoding", "re_log_types", "re_protos", "re_smart_channel", @@ -5826,10 +5829,12 @@ dependencies = [ "lz4_flex", "mimalloc", "parking_lot", + "re_arrow2", "re_build_info", "re_chunk", "re_log", "re_log_types", + "re_protos", "re_smart_channel", "re_tracing", "re_types", @@ -5866,6 +5871,7 @@ dependencies = [ "re_build_info", "re_format", "re_log", + "re_protos", "re_string_interner", "re_tracing", "re_tuid", @@ -5931,9 +5937,6 @@ name = "re_protos" version = "0.21.0-alpha.1+dev" dependencies = [ "prost", - "re_arrow2", - "re_dataframe", - "re_log_types", "thiserror", "tonic", "tonic-web-wasm-client", @@ -7119,6 +7122,7 @@ dependencies = [ "re_chunk_store", "re_dataframe", "re_log", + "re_log_encoding", "re_log_types", "re_memory", "re_protos", diff --git a/crates/build/re_protos_builder/README.md b/crates/build/re_protos_builder/README.md index 25efb2a62361..a14aebe934e0 100644 --- a/crates/build/re_protos_builder/README.md +++ b/crates/build/re_protos_builder/README.md @@ -9,4 +9,4 @@ Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates. Code generation for Rerun's Protobuf and gRPC definitions. -You can generate the code with `pixi run codegen-rstore`. +You can generate the code with `pixi run codegen-protos`. diff --git a/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs b/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs index 9afb8c37f9bf..dc2e15c4d23b 100644 --- a/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs +++ b/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs @@ -1,15 +1,15 @@ //! This binary runs the remote store gRPC service codegen manually. //! -//! It is easiest to call this using `pixi run codegen-rstore`, +//! It is easiest to call this using `pixi run codegen-protos`, //! which will set up the necessary tools. #![allow(clippy::unwrap_used)] use camino::Utf8Path; -const PROTOBUF_DEFINITIONS_DIR_PATH: &str = "crates/store/re_protos/proto"; -const PROTOBUF_REMOTE_STORE_V0_RELATIVE_PATH: &str = "rerun/v0/remote_store.proto"; -const RUST_V0_OUTPUT_DIR_PATH: &str = "crates/store/re_protos/src/v0"; +const PROTOS_DIR: &str = "crates/store/re_protos/proto"; +const INPUT_V0: &[&str] = &["rerun/v0/remote_store.proto", "rerun/v0/log_msg.proto"]; +const OUTPUT_V0_RUST: &str = "crates/store/re_protos/src/v0"; fn main() { re_log::setup_logging(); @@ -26,8 +26,8 @@ fn main() { "failed to find workspace root" ); - let definitions_dir_path = workspace_dir.join(PROTOBUF_DEFINITIONS_DIR_PATH); - let rust_generated_output_dir_path = workspace_dir.join(RUST_V0_OUTPUT_DIR_PATH); + let definitions_dir_path = workspace_dir.join(PROTOS_DIR); + let rust_generated_output_dir_path = workspace_dir.join(OUTPUT_V0_RUST); re_log::info!( definitions=?definitions_dir_path, @@ -37,7 +37,7 @@ fn main() { re_protos_builder::generate_rust_code( definitions_dir_path, - &[PROTOBUF_REMOTE_STORE_V0_RELATIVE_PATH], + INPUT_V0, rust_generated_output_dir_path, ); } diff --git a/crates/build/re_protos_builder/src/lib.rs b/crates/build/re_protos_builder/src/lib.rs index 30315aa559ec..a1d2f744e1a0 100644 --- a/crates/build/re_protos_builder/src/lib.rs +++ b/crates/build/re_protos_builder/src/lib.rs @@ -3,24 +3,34 @@ //! definitions in the same file. //! -#![allow(clippy::unwrap_used)] +#![allow(clippy::unwrap_used, clippy::exit)] use std::path::Path; /// Generate rust from protobuf definitions. We rely on `tonic_build` to do the heavy lifting. /// `tonic_build` relies on `prost` which itself relies on `protoc`. /// -/// Note: make sure to invoke this via `pixi run codegen-rstore` in order to use the right `protoc` version. +/// Note: make sure to invoke this via `pixi run codegen-protos` in order to use the right `protoc` version. pub fn generate_rust_code( definitions_dir: impl AsRef, proto_paths: &[impl AsRef], output_dir: impl AsRef, ) { - tonic_build::configure() + if let Err(err) = tonic_build::configure() .out_dir(output_dir.as_ref()) .build_client(true) .build_server(true) .build_transport(false) // Small convenience, but doesn't work on web .compile_protos(proto_paths, &[definitions_dir]) - .unwrap(); + { + match err.kind() { + std::io::ErrorKind::Other => { + eprintln!("Failed to generate protobuf types:\n{err}"); + std::process::exit(1); + } + _ => { + panic!("{err:?}"); + } + } + } } diff --git a/crates/store/re_chunk_store/Cargo.toml b/crates/store/re_chunk_store/Cargo.toml index 4e9a6a9ab126..dde2529fa031 100644 --- a/crates/store/re_chunk_store/Cargo.toml +++ b/crates/store/re_chunk_store/Cargo.toml @@ -25,7 +25,6 @@ default = [] ## Enables `parking_lot`'s deadlock detection background thread. deadlock_detection = ["parking_lot/deadlock_detection"] - [dependencies] # Rerun dependencies: re_chunk.workspace = true @@ -33,9 +32,11 @@ re_format.workspace = true re_log = { workspace = true, features = ["setup"] } re_log_encoding = { workspace = true, features = ["decoder"] } re_log_types.workspace = true +re_protos.workspace = true re_tracing.workspace = true re_types_core.workspace = true + # External dependencies: ahash.workspace = true anyhow.workspace = true diff --git a/crates/store/re_chunk_store/src/dataframe.rs b/crates/store/re_chunk_store/src/dataframe.rs index eda6c6190153..051ec277e528 100644 --- a/crates/store/re_chunk_store/src/dataframe.rs +++ b/crates/store/re_chunk_store/src/dataframe.rs @@ -1,6 +1,8 @@ //! All the APIs used specifically for `re_dataframe`. use std::collections::{BTreeMap, BTreeSet}; +use std::ops::Deref; +use std::ops::DerefMut; use arrow2::{ array::ListArray as ArrowListArray, @@ -472,7 +474,22 @@ impl std::fmt::Display for SparseFillStrategy { /// // TODO(cmc): we need to be able to build that easily in a command-line context, otherwise it's just // very annoying. E.g. `--with /world/points:[rr.Position3D, rr.Radius] --with /cam:[rr.Pinhole]`. -pub type ViewContentsSelector = BTreeMap>; +#[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] +pub struct ViewContentsSelector(pub BTreeMap>>); + +impl Deref for ViewContentsSelector { + type Target = BTreeMap>>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for ViewContentsSelector { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} // TODO(cmc): Ultimately, this shouldn't be hardcoded to `Timeline`, but to a generic `I: Index`. // `Index` in this case should also be implemented on tuples (`(I1, I2, ...)`). diff --git a/crates/store/re_chunk_store/src/lib.rs b/crates/store/re_chunk_store/src/lib.rs index d4c69adb3ae1..9ed27a730fe9 100644 --- a/crates/store/re_chunk_store/src/lib.rs +++ b/crates/store/re_chunk_store/src/lib.rs @@ -24,6 +24,8 @@ mod store; mod subscribers; mod writes; +mod protobuf_conversions; + pub use self::dataframe::{ ColumnDescriptor, ColumnSelector, ComponentColumnDescriptor, ComponentColumnSelector, Index, IndexRange, IndexValue, QueryExpression, SparseFillStrategy, TimeColumnDescriptor, @@ -47,8 +49,8 @@ pub use re_chunk::{ Chunk, ChunkId, ChunkShared, LatestAtQuery, RangeQuery, RangeQueryOptions, RowId, UnitChunkShared, }; -#[doc(no_inline)] -pub use re_log_encoding::decoder::VersionPolicy; +// #[doc(no_inline)] +// pub use re_log_encoding::decoder::VersionPolicy; #[doc(no_inline)] pub use re_log_types::{ResolvedTimeRange, TimeInt, TimeType, Timeline}; @@ -56,7 +58,7 @@ pub mod external { pub use arrow2; pub use re_chunk; - pub use re_log_encoding; + // pub use re_log_encoding; } // --- diff --git a/crates/store/re_chunk_store/src/store.rs b/crates/store/re_chunk_store/src/store.rs index 705dca3f12e2..fa17ebec4481 100644 --- a/crates/store/re_chunk_store/src/store.rs +++ b/crates/store/re_chunk_store/src/store.rs @@ -683,7 +683,7 @@ impl ChunkStore { pub fn from_rrd_filepath( store_config: &ChunkStoreConfig, path_to_rrd: impl AsRef, - version_policy: crate::VersionPolicy, + version_policy: re_log_encoding::VersionPolicy, ) -> anyhow::Result> { let path_to_rrd = path_to_rrd.as_ref(); @@ -747,7 +747,7 @@ impl ChunkStore { pub fn handle_from_rrd_filepath( store_config: &ChunkStoreConfig, path_to_rrd: impl AsRef, - version_policy: crate::VersionPolicy, + version_policy: re_log_encoding::VersionPolicy, ) -> anyhow::Result> { Ok( Self::from_rrd_filepath(store_config, path_to_rrd, version_policy)? diff --git a/crates/store/re_dataframe/Cargo.toml b/crates/store/re_dataframe/Cargo.toml index eb15df0a9b11..eaaca1a0a58d 100644 --- a/crates/store/re_dataframe/Cargo.toml +++ b/crates/store/re_dataframe/Cargo.toml @@ -24,16 +24,17 @@ all-features = true [features] default = [] - [dependencies] # Rerun dependencies: re_chunk.workspace = true re_chunk_store.workspace = true re_log.workspace = true +re_log_encoding.workspace = true re_log_types.workspace = true re_query.workspace = true re_tracing.workspace = true re_types_core.workspace = true + # External dependencies: anyhow.workspace = true arrow2.workspace = true diff --git a/crates/store/re_dataframe/src/engine.rs b/crates/store/re_dataframe/src/engine.rs index 8a22a4cdaec0..3fc2bb875944 100644 --- a/crates/store/re_dataframe/src/engine.rs +++ b/crates/store/re_dataframe/src/engine.rs @@ -3,7 +3,6 @@ use std::collections::BTreeMap; use re_chunk::{EntityPath, TransportChunk}; use re_chunk_store::{ ChunkStore, ChunkStoreConfig, ChunkStoreHandle, ColumnDescriptor, QueryExpression, - VersionPolicy, }; use re_log_types::{EntityPathFilter, StoreId}; use re_query::{QueryCache, QueryCacheHandle, StorageEngine, StorageEngineLike}; @@ -59,7 +58,7 @@ impl QueryEngine { pub fn from_rrd_filepath( store_config: &ChunkStoreConfig, path_to_rrd: impl AsRef, - version_policy: VersionPolicy, + version_policy: re_log_encoding::VersionPolicy, ) -> anyhow::Result> { Ok( ChunkStore::handle_from_rrd_filepath(store_config, path_to_rrd, version_policy)? diff --git a/crates/store/re_dataframe/src/lib.rs b/crates/store/re_dataframe/src/lib.rs index 40702bc590ac..064727f9b612 100644 --- a/crates/store/re_dataframe/src/lib.rs +++ b/crates/store/re_dataframe/src/lib.rs @@ -13,8 +13,7 @@ pub use self::external::re_chunk::{util::concatenate_record_batches, TransportCh #[doc(no_inline)] pub use self::external::re_chunk_store::{ ChunkStoreConfig, ChunkStoreHandle, ColumnSelector, ComponentColumnSelector, Index, IndexRange, - IndexValue, QueryExpression, SparseFillStrategy, TimeColumnSelector, VersionPolicy, - ViewContentsSelector, + IndexValue, QueryExpression, SparseFillStrategy, TimeColumnSelector, ViewContentsSelector, }; #[doc(no_inline)] pub use self::external::re_log_types::{ diff --git a/crates/store/re_grpc_client/Cargo.toml b/crates/store/re_grpc_client/Cargo.toml index c3e685a2a893..58fea586e418 100644 --- a/crates/store/re_grpc_client/Cargo.toml +++ b/crates/store/re_grpc_client/Cargo.toml @@ -22,8 +22,9 @@ all-features = true [dependencies] re_chunk.workspace = true re_error.workspace = true -re_log_types.workspace = true re_log.workspace = true +re_log_encoding.workspace = true +re_log_types.workspace = true re_protos.workspace = true re_smart_channel.workspace = true diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs index 767ed1dd28f3..5576f15a4bd2 100644 --- a/crates/store/re_grpc_client/src/lib.rs +++ b/crates/store/re_grpc_client/src/lib.rs @@ -9,14 +9,13 @@ pub use address::{Address, InvalidAddressError}; use std::{error::Error, str::FromStr}; use re_chunk::Chunk; +use re_log_encoding::codec::{decode, CodecError}; use re_log_types::{ ApplicationId, LogMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; use re_protos::{ - codec::{decode, CodecError}, - v0::{ - storage_node_client::StorageNodeClient, EncoderVersion, FetchRecordingRequest, RecordingId, - }, + common::v0::{EncoderVersion, RecordingId}, + remote_store::v0::{storage_node_client::StorageNodeClient, FetchRecordingRequest}, }; // ---------------------------------------------------------------------------- diff --git a/crates/store/re_log_encoding/Cargo.toml b/crates/store/re_log_encoding/Cargo.toml index 721b42462c08..5fd58ed1ae61 100644 --- a/crates/store/re_log_encoding/Cargo.toml +++ b/crates/store/re_log_encoding/Cargo.toml @@ -46,10 +46,12 @@ re_build_info.workspace = true re_chunk.workspace = true re_log_types.workspace = true re_log.workspace = true +re_protos.workspace = true re_smart_channel.workspace = true re_tracing.workspace = true # External: +arrow2.workspace = true parking_lot.workspace = true thiserror.workspace = true diff --git a/crates/store/re_log_encoding/src/decoder/mod.rs b/crates/store/re_log_encoding/src/decoder/mod.rs index 3483bd4b39fb..9dd4150c38c3 100644 --- a/crates/store/re_log_encoding/src/decoder/mod.rs +++ b/crates/store/re_log_encoding/src/decoder/mod.rs @@ -8,6 +8,7 @@ use std::io::Read; use re_build_info::CrateVersion; use re_log_types::LogMsg; +use crate::codec; use crate::FileHeader; use crate::MessageHeader; use crate::OLD_RRD_HEADERS; @@ -89,6 +90,9 @@ pub enum DecodeError { #[error("MsgPack error: {0}")] MsgPack(#[from] rmp_serde::decode::Error), + + #[error("Codec error: {0}")] + Codec(#[from] codec::CodecError), } // ---------------------------------------------------------------------------- @@ -117,7 +121,7 @@ pub fn read_options( let FileHeader { magic, version, - options, + mut options, } = FileHeader::decode(&mut read)?; if OLD_RRD_HEADERS.contains(&magic) { @@ -130,6 +134,9 @@ pub fn read_options( match options.serializer { Serializer::MsgPack => {} + Serializer::Protobuf => { + options.compression = Compression::Off; + } } Ok((CrateVersion::from_bytes(version), options)) @@ -152,7 +159,7 @@ impl std::io::Read for Reader { pub struct Decoder { version: CrateVersion, - compression: Compression, + options: EncodingOptions, read: Reader, uncompressed: Vec, // scratch space compressed: Vec, // scratch space @@ -179,11 +186,10 @@ impl Decoder { read.read_exact(&mut data).map_err(DecodeError::Read)?; let (version, options) = read_options(version_policy, &data)?; - let compression = options.compression; Ok(Self { version, - compression, + options, read: Reader::Raw(read), uncompressed: vec![], compressed: vec![], @@ -217,11 +223,10 @@ impl Decoder { read.read_exact(&mut data).map_err(DecodeError::Read)?; let (version, options) = read_options(version_policy, &data)?; - let compression = options.compression; Ok(Self { version, - compression, + options, read: Reader::Buffered(read), uncompressed: vec![], compressed: vec![], @@ -284,10 +289,9 @@ impl Iterator for Decoder { Ok(opts) => opts, Err(err) => return Some(Err(err)), }; - let compression = options.compression; self.version = CrateVersion::max(self.version, version); - self.compression = compression; + self.options = options; self.size_bytes += FileHeader::SIZE as u64; } @@ -322,7 +326,7 @@ impl Iterator for Decoder { self.uncompressed .resize(self.uncompressed.len().max(uncompressed_len), 0); - match self.compression { + match self.options.compression { Compression::Off => { re_tracing::profile_scope!("read uncompressed"); if let Err(err) = self @@ -357,8 +361,19 @@ impl Iterator for Decoder { } } - re_tracing::profile_scope!("MsgPack deser"); - match rmp_serde::from_slice(&self.uncompressed[..uncompressed_len]) { + let data = &self.uncompressed[..uncompressed_len]; + let result = match self.options.serializer { + Serializer::MsgPack => { + re_tracing::profile_scope!("MsgPack deser"); + rmp_serde::from_slice(data).map_err(|e| e.into()) + } + Serializer::Protobuf => { + re_tracing::profile_scope!("Protobuf deser"); + codec::decode_log_msg(data).map_err(|e| e.into()) + } + }; + + match result { Ok(re_log_types::LogMsg::SetStoreInfo(mut msg)) => { // Propagate the protocol version from the header into the `StoreInfo` so that all // parts of the app can easily access it. @@ -366,7 +381,7 @@ impl Iterator for Decoder { Some(Ok(re_log_types::LogMsg::SetStoreInfo(msg))) } Ok(msg) => Some(Ok(msg)), - Err(err) => Some(Err(err.into())), + Err(err) => Some(Err(err)), } } } diff --git a/crates/store/re_log_encoding/src/encoder.rs b/crates/store/re_log_encoding/src/encoder.rs index edc966ce0d19..73127fc2c1a0 100644 --- a/crates/store/re_log_encoding/src/encoder.rs +++ b/crates/store/re_log_encoding/src/encoder.rs @@ -4,8 +4,10 @@ use re_build_info::CrateVersion; use re_chunk::{ChunkError, ChunkResult}; use re_log_types::LogMsg; +use crate::codec; use crate::FileHeader; use crate::MessageHeader; +use crate::Serializer; use crate::{Compression, EncodingOptions}; // ---------------------------------------------------------------------------- @@ -22,6 +24,9 @@ pub enum EncodeError { #[error("MsgPack error: {0}")] MsgPack(#[from] rmp_serde::encode::Error), + #[error("{0}")] + Codec(#[from] codec::CodecError), + #[error("Chunk error: {0}")] Chunk(#[from] ChunkError), @@ -109,6 +114,7 @@ impl std::ops::Drop for DroppableEncoder { /// Prefer [`DroppableEncoder`] if possible, make sure to call [`Encoder::finish`] when appropriate /// otherwise. pub struct Encoder { + serializer: Serializer, compression: Compression, write: W, uncompressed: Vec, @@ -128,15 +134,18 @@ impl Encoder { } .encode(&mut write)?; - match options.serializer { - crate::Serializer::MsgPack => {} - } + let serializer = options.serializer; + let compression = match serializer { + Serializer::MsgPack => options.compression, + Serializer::Protobuf => Compression::Off, + }; Ok(Self { - compression: options.compression, + serializer, + compression, write, - uncompressed: vec![], - compressed: vec![], + uncompressed: Vec::new(), + compressed: Vec::new(), }) } @@ -145,7 +154,14 @@ impl Encoder { re_tracing::profile_function!(); self.uncompressed.clear(); - rmp_serde::encode::write_named(&mut self.uncompressed, message)?; + match self.serializer { + Serializer::MsgPack => { + rmp_serde::encode::write_named(&mut self.uncompressed, message)?; + } + Serializer::Protobuf => { + crate::codec::encode_log_msg(message, &mut self.uncompressed)?; + } + } match self.compression { Compression::Off => { diff --git a/crates/store/re_log_encoding/src/lib.rs b/crates/store/re_log_encoding/src/lib.rs index 2f62a6675d75..064ad9ad575f 100644 --- a/crates/store/re_log_encoding/src/lib.rs +++ b/crates/store/re_log_encoding/src/lib.rs @@ -2,9 +2,14 @@ #[cfg(feature = "decoder")] pub mod decoder; +#[cfg(feature = "decoder")] +pub use decoder::VersionPolicy; + #[cfg(feature = "encoder")] pub mod encoder; +pub mod codec; + #[cfg(feature = "encoder")] #[cfg(not(target_arch = "wasm32"))] mod file_sink; @@ -21,10 +26,10 @@ pub use file_sink::{FileSink, FileSinkError}; // ---------------------------------------------------------------------------- #[cfg(any(feature = "encoder", feature = "decoder"))] -const RRD_HEADER: &[u8; 4] = b"RRF2"; +const RRD_HEADER: &[u8; 4] = b"RRF3"; #[cfg(feature = "decoder")] -const OLD_RRD_HEADERS: &[[u8; 4]] = &[*b"RRF0", *b"RRF1"]; +const OLD_RRD_HEADERS: &[[u8; 4]] = &[*b"RRF0", *b"RRF1", *b"RRF2"]; // ---------------------------------------------------------------------------- @@ -43,6 +48,7 @@ pub enum Compression { #[repr(u8)] pub enum Serializer { MsgPack = 1, + Protobuf = 2, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -60,6 +66,10 @@ impl EncodingOptions { compression: Compression::LZ4, serializer: Serializer::MsgPack, }; + pub const PROTOBUF: Self = Self { + compression: Compression::Off, + serializer: Serializer::Protobuf, + }; pub fn from_bytes(bytes: [u8; 4]) -> Result { match bytes { @@ -71,6 +81,7 @@ impl EncodingOptions { }; let serializer = match serializer { 1 => Serializer::MsgPack, + 2 => Serializer::Protobuf, _ => return Err(OptionsError::UnknownSerializer(serializer)), }; Ok(Self { diff --git a/crates/store/re_log_types/Cargo.toml b/crates/store/re_log_types/Cargo.toml index 9ac3b184401b..969cb3718ea5 100644 --- a/crates/store/re_log_types/Cargo.toml +++ b/crates/store/re_log_types/Cargo.toml @@ -45,11 +45,13 @@ serde = [ re_build_info.workspace = true re_format.workspace = true re_log.workspace = true +re_protos.workspace = true re_string_interner.workspace = true re_tracing.workspace = true re_tuid.workspace = true re_types_core.workspace = true + # External ahash.workspace = true anyhow.workspace = true diff --git a/crates/store/re_log_types/src/lib.rs b/crates/store/re_log_types/src/lib.rs index 5c3f52b88b87..50d1fa212efb 100644 --- a/crates/store/re_log_types/src/lib.rs +++ b/crates/store/re_log_types/src/lib.rs @@ -32,6 +32,8 @@ mod time; mod time_real; mod vec_deque_ext; +mod protobuf_conversions; + use std::sync::Arc; use re_build_info::CrateVersion; @@ -405,6 +407,74 @@ impl std::fmt::Display for PythonVersion { } } +impl std::str::FromStr for PythonVersion { + type Err = PythonVersionParseError; + + fn from_str(s: &str) -> Result { + let (major, rest) = s + .split_once('.') + .ok_or(PythonVersionParseError::MissingMajor)?; + if major.is_empty() { + return Err(PythonVersionParseError::MissingMajor); + } + let (minor, rest) = rest + .split_once('.') + .ok_or(PythonVersionParseError::MissingMinor)?; + if minor.is_empty() { + return Err(PythonVersionParseError::MissingMinor); + } + let pos = rest.bytes().position(|v| !v.is_ascii_digit()); + let (patch, suffix) = match pos { + Some(pos) => rest.split_at(pos), + None => (rest, ""), + }; + if patch.is_empty() { + return Err(PythonVersionParseError::MissingPatch); + } + Ok(Self { + major: major + .parse() + .map_err(|_| PythonVersionParseError::InvalidMajor)?, + minor: minor + .parse() + .map_err(|_| PythonVersionParseError::InvalidMinor)?, + patch: patch + .parse() + .map_err(|_| PythonVersionParseError::InvalidPatch)?, + suffix: suffix.into(), + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum PythonVersionParseError { + #[error("missing major version")] + MissingMajor, + #[error("missing minor version")] + MissingMinor, + #[error("missing patch version")] + MissingPatch, + #[error("invalid major version")] + InvalidMajor, + #[error("invalid minor version")] + InvalidMinor, + #[error("invalid patch version")] + InvalidPatch, +} + +impl PythonVersionParseError { + pub fn as_str(&self) -> &'static str { + match self { + Self::MissingMajor => "missing major version", + Self::MissingMinor => "missing minor version", + Self::MissingPatch => "missing patch version", + Self::InvalidMajor => "invalid major version", + Self::InvalidMinor => "invalid minor version", + Self::InvalidPatch => "invalid patch version", + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub enum FileSource { diff --git a/crates/store/re_protos/Cargo.toml b/crates/store/re_protos/Cargo.toml index 66cb898e1cc1..74a75d7759c4 100644 --- a/crates/store/re_protos/Cargo.toml +++ b/crates/store/re_protos/Cargo.toml @@ -13,11 +13,7 @@ rust-version.workspace = true version.workspace = true [dependencies] -re_log_types.workspace = true -re_dataframe.workspace = true - # External -arrow2 = { workspace = true, features = ["io_ipc"] } prost.workspace = true thiserror.workspace = true diff --git a/crates/store/re_protos/README.md b/crates/store/re_protos/README.md index 9e6a1c97637f..e96cb055def6 100644 --- a/crates/store/re_protos/README.md +++ b/crates/store/re_protos/README.md @@ -11,4 +11,4 @@ Rerun remote store node gRPC API service types (client and server). This crate includes both the language-agnostic definitions (protobuf) as well as the generated code. -The code is generated with `pixi run codegen-rstore`. +The code is generated with `pixi run codegen-protos`. diff --git a/crates/store/re_protos/proto/rerun/v0/common.proto b/crates/store/re_protos/proto/rerun/v0/common.proto index 0f5d979cd81c..bf4c8d43a33f 100644 --- a/crates/store/re_protos/proto/rerun/v0/common.proto +++ b/crates/store/re_protos/proto/rerun/v0/common.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package rerun.remote_store.v0; +package rerun.common.v0; // unique recording identifier. At this point in time it is the same id as the ChunkStore's StoreId message RecordingId { @@ -161,3 +161,38 @@ enum SparseFillStrategy { NONE = 0; LATEST_AT_GLOBAL = 1; } + +// Error codes for application level errors +enum ErrorCode { + // unused + _UNUSED = 0; + + // object store access error + OBJECT_STORE_ERROR = 1; + + // metadata database access error + METADATA_DB_ERROR = 2; +} + +message ApplicationId { + string id = 1; +} + +enum StoreKind { + RECORDING = 0; + BLUEPRINT = 1; +} + +message StoreId { + StoreKind kind = 1; + string id = 2; +} + +// A date-time represented as nanoseconds since unix epoch +message Time { + int64 nanos_since_epoch = 1; +} + +enum EncoderVersion { + V0 = 0; +} diff --git a/crates/store/re_protos/proto/rerun/v0/remote_store.proto b/crates/store/re_protos/proto/rerun/v0/remote_store.proto index 6b156a4b3eda..a544577d20f7 100644 --- a/crates/store/re_protos/proto/rerun/v0/remote_store.proto +++ b/crates/store/re_protos/proto/rerun/v0/remote_store.proto @@ -12,7 +12,8 @@ service StorageNode { // metadata API calls rpc ListRecordings(ListRecordingsRequest) returns (ListRecordingsResponse) {} rpc GetRecordingMetadata(GetRecordingMetadataRequest) returns (GetRecordingMetadataResponse) {} - rpc UpdateRecordingMetadata(UpdateRecordingMetadataRequest) returns (UpdateRecordingMetadataResponse) {} + rpc UpdateRecordingMetadata(UpdateRecordingMetadataRequest) + returns (UpdateRecordingMetadataResponse) {} rpc RegisterRecording(RegisterRecordingRequest) returns (RegisterRecordingResponse) {} } @@ -32,12 +33,12 @@ message RegisterRecordingRequest { // Recording metadata is single row arrow record batch message RecordingMetadata { - EncoderVersion encoder_version = 1; + rerun.common.v0.EncoderVersion encoder_version = 1; bytes payload = 2; } message RegisterRecordingResponse { - RecordingId id = 1; + rerun.common.v0.RecordingId id = 1; // Note / TODO(zehiko): this implies we read the record (for example go through entire .rrd file // chunk by chunk) and extract the metadata. So we might want to 1/ not do this i.e. // only do it as part of explicit GetMetadata request or 2/ do it if Request has "include_metadata=true" @@ -45,26 +46,36 @@ message RegisterRecordingResponse { RecordingMetadata metadata = 2; } +// Server can include details about the error as part of gRPC error (Status) +message RegistrationError { + // error code + rerun.common.v0.ErrorCode code = 1; + // storage url of the recording that failed to register + string storage_url = 2; + // human readable details about the error + string message = 3; +} + // ---------------- GetRecordingMetadata ----------------- message GetRecordingMetadataRequest { - RecordingId recording_id = 1; + rerun.common.v0.RecordingId recording_id = 1; } message GetRecordingMetadataResponse { - RecordingId id = 1; + rerun.common.v0.RecordingId id = 1; RecordingMetadata metadata = 2; } message TimeMetadata { - Timeline timeline = 1; - TimeRange time_range = 2; + rerun.common.v0.Timeline timeline = 1; + rerun.common.v0.TimeRange time_range = 2; } // ---------------- UpdateRecordingMetadata ----------------- message UpdateRecordingMetadataRequest { - RecordingId recording_id = 1; + rerun.common.v0.RecordingId recording_id = 1; RecordingMetadata metadata = 2; } @@ -74,26 +85,20 @@ message UpdateRecordingMetadataResponse {} message QueryRequest { // unique identifier of the recording - RecordingId recording_id = 1; + rerun.common.v0.RecordingId recording_id = 1; // query to execute - Query query = 3; + rerun.common.v0.Query query = 3; } message QueryResponse { // TODO(zehiko) we need to expand this to become something like 'encoder options' // as we will need to specify additional options like compression, including schema // in payload, etc. - EncoderVersion encoder_version = 1; + rerun.common.v0.EncoderVersion encoder_version = 1; // payload is raw bytes that the relevant codec can interpret bytes payload = 2; } - -enum EncoderVersion { - V0 = 0; -} - - // ----------------- ListRecordings ----------------- message ListRecordingsRequest { @@ -118,7 +123,7 @@ enum RecordingType { // ----------------- FetchRecording ----------------- message FetchRecordingRequest { - RecordingId recording_id = 1; + rerun.common.v0.RecordingId recording_id = 1; } // TODO(jleibs): Eventually this becomes either query-mediated in some way, but for now @@ -127,7 +132,7 @@ message FetchRecordingResponse { // TODO(zehiko) we need to expand this to become something like 'encoder options' // as we will need to specify additional options like compression, including schema // in payload, etc. - EncoderVersion encoder_version = 1; + rerun.common.v0.EncoderVersion encoder_version = 1; // payload is raw bytes that the relevant codec can interpret bytes payload = 2; } diff --git a/crates/store/re_protos/src/lib.rs b/crates/store/re_protos/src/lib.rs index 49275482e736..b0bbe04cefb2 100644 --- a/crates/store/re_protos/src/lib.rs +++ b/crates/store/re_protos/src/lib.rs @@ -6,434 +6,235 @@ //! necessary conversion code (in the form of `From` and `TryFrom` traits) in this crate. //! -/// Codec for serializing and deserializing query response (record batch) data -pub mod codec; - -/// Generated types for the remote store gRPC service API v0. -pub mod v0 { - // Ignoring all warnings for the auto-generated code. - #[allow(clippy::doc_markdown)] - #[allow(clippy::derive_partial_eq_without_eq)] - #[allow(clippy::enum_variant_names)] - #[allow(clippy::unwrap_used)] - #[allow(clippy::wildcard_imports)] - #[allow(clippy::manual_is_variant_and)] - #[path = "../v0/rerun.remote_store.v0.rs"] - mod _v0; - - pub use self::_v0::*; - - // ==== below are all necessary transforms from internal rerun types to protobuf types ===== - - use std::{collections::BTreeSet, sync::Arc}; - - #[derive(Debug, thiserror::Error)] - pub enum TypeConversionError { - #[error("missing required field: {0}")] - MissingField(&'static str), - } - - impl From for re_log_types::StoreId { - #[inline] - fn from(value: RecordingId) -> Self { - Self { - kind: re_log_types::StoreKind::Recording, - id: Arc::new(value.id), - } - } - } - - impl From for RecordingId { - #[inline] - fn from(value: re_log_types::StoreId) -> Self { - Self { - id: value.id.to_string(), - } - } - } - - impl From for TimeRange { - fn from(time_range: re_log_types::ResolvedTimeRange) -> Self { - Self { - start: time_range.min().as_i64(), - end: time_range.max().as_i64(), - } - } - } - - impl TryFrom for re_dataframe::QueryExpression { - type Error = TypeConversionError; - - fn try_from(value: Query) -> Result { - let filtered_index = value - .filtered_index - .ok_or(TypeConversionError::MissingField("filtered_index"))? - .try_into()?; - - let selection = value - .column_selection - .map(|cs| { - cs.columns - .into_iter() - .map(re_dataframe::ColumnSelector::try_from) - .collect::, _>>() - }) - .transpose()?; - - let filtered_is_not_null = value - .filtered_is_not_null - .map(re_dataframe::ComponentColumnSelector::try_from) - .transpose()?; - - Ok(Self { - view_contents: value.view_contents.map(|vc| vc.into()), - include_semantically_empty_columns: value.include_semantically_empty_columns, - include_indicator_columns: value.include_indicator_columns, - include_tombstone_columns: value.include_tombstone_columns, - filtered_index: Some(filtered_index), - filtered_index_range: value - .filtered_index_range - .map(|ir| ir.try_into()) - .transpose()?, - filtered_index_values: value - .filtered_index_values - .map(|iv| iv.time_points.into_iter().map(|v| v.into()).collect()), - using_index_values: value - .using_index_values - .map(|uiv| uiv.time_points.into_iter().map(|v| v.into()).collect()), - filtered_is_not_null, - sparse_fill_strategy: re_dataframe::SparseFillStrategy::default(), // TODO(zehiko) implement support for sparse fill strategy - selection, - }) - } - } - - impl From for re_dataframe::ViewContentsSelector { - fn from(value: ViewContents) -> Self { - value - .contents - .into_iter() - .map(|part| { - #[allow(clippy::unwrap_used)] // TODO(zehiko) - let entity_path = Into::::into(part.path.unwrap()); - let column_selector = part.components.map(|cs| { - cs.components - .into_iter() - .map(|c| re_dataframe::external::re_chunk::ComponentName::new(&c.name)) - .collect::>() - }); - (entity_path, column_selector) - }) - .collect::() - } - } - - impl From for re_log_types::EntityPath { - fn from(value: EntityPath) -> Self { - Self::from(value.path) - } - } - - impl TryFrom for re_log_types::Timeline { - type Error = TypeConversionError; - - fn try_from(value: IndexColumnSelector) -> Result { - let timeline_name = value - .timeline - .ok_or(TypeConversionError::MissingField("timeline"))? - .name; - - // TODO(cmc): QueryExpression::filtered_index gotta be a selector - #[allow(clippy::match_same_arms)] - let timeline = match timeline_name.as_str() { - "log_time" => Self::new_temporal(timeline_name), - "log_tick" => Self::new_sequence(timeline_name), - "frame" => Self::new_sequence(timeline_name), - "frame_nr" => Self::new_sequence(timeline_name), - _ => Self::new_temporal(timeline_name), - }; - - Ok(timeline) - } - } - - impl TryFrom for re_dataframe::IndexRange { - type Error = TypeConversionError; +// This extra module is needed, because of how imports from different packages are resolved. +// For example, `rerun.common.v0.EncoderVersion` is resolved to `super::super::common::v0::EncoderVersion`. +// We need an extra module in the path to `common` to make that work. +// Additionally, the `common` module itself has to exist with a `v0` module inside of it, +// which is the reason for the `common`, `log_msg`, `remote_store`, etc. modules below. + +pub mod external { + pub use prost; +} - fn try_from(value: IndexRange) -> Result { - let time_range = value - .time_range - .ok_or(TypeConversionError::MissingField("time_range"))?; +// Note: Be careful with `#[path]` attributes: https://github.com/rust-lang/rust/issues/35016 +mod v0 { + // Note: `allow(clippy::all)` does NOT allow all lints + #![allow(clippy::all, clippy::pedantic, clippy::nursery)] - Ok(Self::new(time_range.start, time_range.end)) - } - } + #[path = "./rerun.common.v0.rs"] + pub mod rerun_common_v0; - impl From for re_log_types::TimeInt { - fn from(value: TimeInt) -> Self { - Self::new_temporal(value.time) - } - } + #[path = "./rerun.log_msg.v0.rs"] + pub mod rerun_log_msg_v0; - impl TryFrom for re_dataframe::ComponentColumnSelector { - type Error = TypeConversionError; - - fn try_from(value: ComponentColumnSelector) -> Result { - let entity_path = value - .entity_path - .ok_or(TypeConversionError::MissingField("entity_path"))? - .into(); - - let component_name = value - .component - .ok_or(TypeConversionError::MissingField("component"))? - .name; + #[path = "./rerun.remote_store.v0.rs"] + pub mod rerun_remote_store_v0; +} - Ok(Self { - entity_path, - component_name, - }) - } +pub mod common { + pub mod v0 { + pub use crate::v0::rerun_common_v0::*; } +} - impl TryFrom for re_dataframe::TimeColumnSelector { - type Error = TypeConversionError; - - fn try_from(value: TimeColumnSelector) -> Result { - let timeline = value - .timeline - .ok_or(TypeConversionError::MissingField("timeline"))?; - - Ok(Self { - timeline: timeline.name.into(), - }) - } +pub mod log_msg { + pub mod v0 { + pub use crate::v0::rerun_log_msg_v0::*; } +} - impl TryFrom for re_dataframe::ColumnSelector { - type Error = TypeConversionError; - - fn try_from(value: ColumnSelector) -> Result { - match value - .selector_type - .ok_or(TypeConversionError::MissingField("selector_type"))? - { - column_selector::SelectorType::ComponentColumn(component_column_selector) => { - let selector: re_dataframe::ComponentColumnSelector = - component_column_selector.try_into()?; - Ok(selector.into()) - } - column_selector::SelectorType::TimeColumn(time_column_selector) => { - let selector: re_dataframe::TimeColumnSelector = - time_column_selector.try_into()?; - - Ok(selector.into()) - } - } - } +/// Generated types for the remote store gRPC service API v0. +pub mod remote_store { + pub mod v0 { + pub use crate::v0::rerun_remote_store_v0::*; } +} - // ---- conversion from rerun's QueryExpression into protobuf Query ---- +// // ==== below are all necessary transforms from internal rerun types to protobuf types ===== - impl From for Query { - fn from(value: re_dataframe::QueryExpression) -> Self { - let view_contents = value - .view_contents - .map(|vc| { - vc.into_iter() - .map(|(path, components)| ViewContentsPart { - path: Some(path.into()), - components: components.map(|cs| ComponentsSet { - components: cs - .into_iter() - .map(|c| Component { - name: c.to_string(), - }) - .collect(), - }), - }) - .collect::>() - }) - .map(|cs| ViewContents { contents: cs }); +// use std::{collections::BTreeSet, sync::Arc}; - Self { - view_contents, - include_semantically_empty_columns: value.include_semantically_empty_columns, - include_indicator_columns: value.include_indicator_columns, - include_tombstone_columns: value.include_tombstone_columns, - filtered_index: value.filtered_index.map(|timeline| IndexColumnSelector { - timeline: Some(Timeline { - name: timeline.name().to_string(), - }), - }), - filtered_index_range: value.filtered_index_range.map(|ir| IndexRange { - time_range: Some(ir.into()), - }), - filtered_index_values: value.filtered_index_values.map(|iv| IndexValues { - time_points: iv - .into_iter() - // TODO(zehiko) is this desired behavior for TimeInt::STATIC? - .map(|v| TimeInt { time: v.as_i64() }) - .collect(), - }), - using_index_values: value.using_index_values.map(|uiv| IndexValues { - time_points: uiv - .into_iter() - .map(|v| TimeInt { time: v.as_i64() }) - .collect(), - }), - filtered_is_not_null: value.filtered_is_not_null.map(|cs| { - ComponentColumnSelector { - entity_path: Some(cs.entity_path.into()), - component: Some(Component { - name: cs.component_name, - }), - } - }), - column_selection: value.selection.map(|cs| ColumnSelection { - columns: cs.into_iter().map(|c| c.into()).collect(), - }), - sparse_fill_strategy: SparseFillStrategy::None.into(), // TODO(zehiko) implement - } - } - } +#[derive(Debug, thiserror::Error)] +pub enum TypeConversionError { + #[error("missing required field: {type_name}.{field_name}")] + MissingField { + type_name: &'static str, + field_name: &'static str, + }, - impl From for EntityPath { - fn from(value: re_dataframe::EntityPath) -> Self { - Self { - path: value.to_string(), - } - } - } + #[error("failed to decode: {0}")] + DecodeError(#[from] prost::DecodeError), - impl From for ColumnSelector { - fn from(value: re_dataframe::ColumnSelector) -> Self { - match value { - re_dataframe::ColumnSelector::Component(ccs) => Self { - selector_type: Some(column_selector::SelectorType::ComponentColumn( - ComponentColumnSelector { - entity_path: Some(ccs.entity_path.into()), - component: Some(Component { - name: ccs.component_name, - }), - }, - )), - }, - re_dataframe::ColumnSelector::Time(tcs) => Self { - selector_type: Some(column_selector::SelectorType::TimeColumn( - TimeColumnSelector { - timeline: Some(Timeline { - name: tcs.timeline.to_string(), - }), - }, - )), - }, - } - } - } + #[error("failed to encode: {0}")] + EncodeError(#[from] prost::EncodeError), - // ------- Application level errors ------- - impl std::error::Error for RemoteStoreError {} + #[error("{0}")] + UnknownEnumValue(#[from] prost::UnknownEnumValue), +} - impl std::fmt::Display for RemoteStoreError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_fmt(format_args!( - "Remote store error. Request identifier: {}, error msg: {}, error code: {}", - self.id, self.message, self.code - )) +impl TypeConversionError { + pub fn missing_field(type_name: &'static str, field_name: &'static str) -> Self { + Self::MissingField { + type_name, + field_name, } } } -#[cfg(test)] -mod tests { - - use crate::v0::{ - column_selector::SelectorType, ColumnSelection, ColumnSelector, Component, - ComponentColumnSelector, ComponentsSet, EntityPath, IndexColumnSelector, IndexRange, - IndexValues, Query, RecordingId, SparseFillStrategy, TimeInt, TimeRange, Timeline, - ViewContents, ViewContentsPart, - }; - - #[test] - pub fn test_query_conversion() { - let grpc_query_before = Query { - view_contents: Some(ViewContents { - contents: vec![ViewContentsPart { - path: Some(EntityPath { - path: "/somepath".to_owned(), - }), - components: Some(ComponentsSet { - components: vec![Component { - name: "component".to_owned(), - }], - }), - }], - }), - include_indicator_columns: false, - include_semantically_empty_columns: true, - include_tombstone_columns: true, - filtered_index: Some(IndexColumnSelector { - timeline: Some(Timeline { - name: "log_time".to_owned(), - }), - }), - filtered_index_range: Some(IndexRange { - time_range: Some(TimeRange { start: 0, end: 100 }), - }), - filtered_index_values: Some(IndexValues { - time_points: vec![ - TimeInt { time: 0 }, - TimeInt { time: 1 }, - TimeInt { time: 2 }, - ], - }), - using_index_values: Some(IndexValues { - time_points: vec![ - TimeInt { time: 3 }, - TimeInt { time: 4 }, - TimeInt { time: 5 }, - ], - }), - filtered_is_not_null: Some(ComponentColumnSelector { - entity_path: Some(EntityPath { - path: "/somepath/c".to_owned(), - }), - component: Some(Component { - name: "component".to_owned(), - }), - }), - column_selection: Some(ColumnSelection { - columns: vec![ColumnSelector { - selector_type: Some(SelectorType::ComponentColumn(ComponentColumnSelector { - entity_path: Some(EntityPath { - path: "/somepath/c".to_owned(), - }), - component: Some(Component { - name: "component".to_owned(), - }), - })), - }], - }), - sparse_fill_strategy: SparseFillStrategy::None.into(), - }; - - let query_expression_native: re_dataframe::QueryExpression = - grpc_query_before.clone().try_into().unwrap(); - let grpc_query_after = query_expression_native.into(); - - assert_eq!(grpc_query_before, grpc_query_after); - } - - #[test] - fn test_recording_id_conversion() { - let recording_id = RecordingId { - id: "recording_id".to_owned(), - }; - - let store_id: re_log_types::StoreId = recording_id.clone().into(); - let recording_id_after: RecordingId = store_id.into(); - - assert_eq!(recording_id, recording_id_after); - } -} +// // ---- conversion from rerun's QueryExpression into protobuf Query ---- + +// impl From for Query { +// fn from(value: re_dataframe::QueryExpression) -> Self { +// let view_contents = value +// .view_contents +// .map(|vc| { +// vc.into_iter() +// .map(|(path, components)| ViewContentsPart { +// path: Some(path.into()), +// components: components.map(|cs| ComponentsSet { +// components: cs +// .into_iter() +// .map(|c| Component { +// name: c.to_string(), +// }) +// .collect(), +// }), +// }) +// .collect::>() +// }) +// .map(|cs| ViewContents { contents: cs }); + +// Self { +// view_contents, +// include_semantically_empty_columns: value.include_semantically_empty_columns, +// include_indicator_columns: value.include_indicator_columns, +// include_tombstone_columns: value.include_tombstone_columns, +// filtered_index: value.filtered_index.map(|timeline| IndexColumnSelector { +// timeline: Some(Timeline { +// name: timeline.name().to_string(), +// }), +// }), +// filtered_index_range: value.filtered_index_range.map(|ir| IndexRange { +// time_range: Some(ir.into()), +// }), +// filtered_index_values: value.filtered_index_values.map(|iv| IndexValues { +// time_points: iv +// .into_iter() +// // TODO(zehiko) is this desired behavior for TimeInt::STATIC? +// .map(|v| TimeInt { time: v.as_i64() }) +// .collect(), +// }), +// using_index_values: value.using_index_values.map(|uiv| IndexValues { +// time_points: uiv +// .into_iter() +// .map(|v| TimeInt { time: v.as_i64() }) +// .collect(), +// }), +// filtered_is_not_null: value +// .filtered_is_not_null +// .map(|cs| ComponentColumnSelector { +// entity_path: Some(cs.entity_path.into()), +// component: Some(Component { +// name: cs.component_name, +// }), +// }), +// column_selection: value.selection.map(|cs| ColumnSelection { +// columns: cs.into_iter().map(|c| c.into()).collect(), +// }), +// sparse_fill_strategy: SparseFillStrategy::None.into(), // TODO(zehiko) implement +// } +// } +// } + +// // ------- Application level errors ------- +// impl std::error::Error for RegistrationError {} + +// impl std::fmt::Display for RegistrationError { +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// f.write_fmt(format_args!( +// "Failed to register recording: {}, error code: {}, error message: {}", +// self.storage_url, self.code, self.message +// )) +// } +// } + +// #[cfg(test)] +// mod tests { + +// use crate::v0::{ +// column_selector::SelectorType, ColumnSelection, ColumnSelector, Component, +// ComponentColumnSelector, ComponentsSet, EntityPath, IndexColumnSelector, IndexRange, +// IndexValues, Query, RecordingId, SparseFillStrategy, TimeInt, TimeRange, Timeline, +// ViewContents, ViewContentsPart, +// }; + +// #[test] +// pub fn test_query_conversion() { +// let grpc_query_before = Query { +// view_contents: Some(ViewContents { +// contents: vec![ViewContentsPart { +// path: Some(EntityPath { +// path: "/somepath".to_owned(), +// }), +// components: Some(ComponentsSet { +// components: vec![Component { +// name: "component".to_owned(), +// }], +// }), +// }], +// }), +// include_indicator_columns: false, +// include_semantically_empty_columns: true, +// include_tombstone_columns: true, +// filtered_index: Some(IndexColumnSelector { +// timeline: Some(Timeline { +// name: "log_time".to_owned(), +// }), +// }), +// filtered_index_range: Some(IndexRange { +// time_range: Some(TimeRange { start: 0, end: 100 }), +// }), +// filtered_index_values: Some(IndexValues { +// time_points: vec![ +// TimeInt { time: 0 }, +// TimeInt { time: 1 }, +// TimeInt { time: 2 }, +// ], +// }), +// using_index_values: Some(IndexValues { +// time_points: vec![ +// TimeInt { time: 3 }, +// TimeInt { time: 4 }, +// TimeInt { time: 5 }, +// ], +// }), +// filtered_is_not_null: Some(ComponentColumnSelector { +// entity_path: Some(EntityPath { +// path: "/somepath/c".to_owned(), +// }), +// component: Some(Component { +// name: "component".to_owned(), +// }), +// }), +// column_selection: Some(ColumnSelection { +// columns: vec![ColumnSelector { +// selector_type: Some(SelectorType::ComponentColumn(ComponentColumnSelector { +// entity_path: Some(EntityPath { +// path: "/somepath/c".to_owned(), +// }), +// component: Some(Component { +// name: "component".to_owned(), +// }), +// })), +// }], +// }), +// sparse_fill_strategy: SparseFillStrategy::None.into(), +// }; + +// let query_expression_native: re_dataframe::QueryExpression = +// grpc_query_before.clone().try_into().unwrap(); +// let grpc_query_after = query_expression_native.into(); + +// assert_eq!(grpc_query_before, grpc_query_after); +// } +// } diff --git a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs index bb0c4bd235f3..0666d35d2cd0 100644 --- a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs @@ -1,214 +1,4 @@ // This file is @generated by prost-build. -/// unique recording identifier. At this point in time it is the same id as the ChunkStore's StoreId -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct RecordingId { - #[prost(string, tag = "1")] - pub id: ::prost::alloc::string::String, -} -/// A recording can have multiple timelines, each is identified by a name, for example `log_tick`, `log_time`, etc. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Timeline { - #[prost(string, tag = "1")] - pub name: ::prost::alloc::string::String, -} -/// A time range between start and end time points. Each 64 bit number can represent different time point data -/// depending on the timeline it is associated with. Time range is inclusive for both start and end time points. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct TimeRange { - #[prost(int64, tag = "1")] - pub start: i64, - #[prost(int64, tag = "2")] - pub end: i64, -} -/// arrow IPC serialized schema -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Schema { - #[prost(bytes = "vec", tag = "1")] - pub arrow_schema: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Query { - /// The subset of the database that the query will run on: a set of EntityPath(s) and their - /// associated Component(s) - #[prost(message, optional, tag = "1")] - pub view_contents: ::core::option::Option, - /// Whether the view_contents should ignore semantically empty columns - /// A semantically empty column is a column that either contains no data at all, or where all - /// values are either nulls or empty arrays (\[\]). - #[prost(bool, tag = "2")] - pub include_semantically_empty_columns: bool, - /// Whether the view_contents should ignore columns corresponding to indicator components - /// Indicator components are marker components, generally automatically inserted by Rerun, that - /// helps keep track of the original context in which a piece of data was logged/sent. - #[prost(bool, tag = "3")] - pub include_indicator_columns: bool, - /// Whether the view_contents should ignore columns corresponding to Clear-related components - #[prost(bool, tag = "4")] - pub include_tombstone_columns: bool, - /// The index used to filter out _rows_ from the view contents. - /// Only rows where at least 1 column contains non-null data at that index will be kept in the - /// final dataset. If left unspecified, the results will only contain static data. - #[prost(message, optional, tag = "5")] - pub filtered_index: ::core::option::Option, - /// The range of index values used to filter out _rows_ from the view contents - /// Only rows where at least 1 of the view-contents contains non-null data within that range will be kept in - /// the final dataset. - /// This has no effect if filtered_index isn't set. - /// This has no effect if using_index_values is set. - #[prost(message, optional, tag = "6")] - pub filtered_index_range: ::core::option::Option, - /// The specific index values used to filter out _rows_ from the view contents. - /// Only rows where at least 1 column contains non-null data at these specific values will be kept - /// in the final dataset. - /// This has no effect if filtered_index isn't set. - /// This has no effect if using_index_values is set. - #[prost(message, optional, tag = "7")] - pub filtered_index_values: ::core::option::Option, - /// The specific index values used to sample _rows_ from the view contents. - /// The final dataset will contain one row per sampled index value, regardless of whether data - /// existed for that index value in the view contents. - /// The semantics of the query are consistent with all other settings: the results will be - /// sorted on the filtered_index, and only contain unique index values. - /// - /// This has no effect if filtered_index isn't set. - /// If set, this overrides both filtered_index_range and filtered_index_values. - #[prost(message, optional, tag = "8")] - pub using_index_values: ::core::option::Option, - /// The component column used to filter out _rows_ from the view contents. - /// Only rows where this column contains non-null data be kept in the final dataset. - #[prost(message, optional, tag = "9")] - pub filtered_is_not_null: ::core::option::Option, - /// / The specific _columns_ to sample from the final view contents. - /// / The order of the samples will be respected in the final result. - /// / - /// / If unspecified, it means - everything. - #[prost(message, optional, tag = "10")] - pub column_selection: ::core::option::Option, - /// Specifies how null values should be filled in the returned dataframe. - #[prost(enumeration = "SparseFillStrategy", tag = "11")] - pub sparse_fill_strategy: i32, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ColumnSelection { - #[prost(message, repeated, tag = "1")] - pub columns: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ColumnSelector { - #[prost(oneof = "column_selector::SelectorType", tags = "2, 3")] - pub selector_type: ::core::option::Option, -} -/// Nested message and enum types in `ColumnSelector`. -pub mod column_selector { - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum SelectorType { - #[prost(message, tag = "2")] - ComponentColumn(super::ComponentColumnSelector), - #[prost(message, tag = "3")] - TimeColumn(super::TimeColumnSelector), - } -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexColumnSelector { - /// TODO(zehiko) we need to add support for other types of index selectors - #[prost(message, optional, tag = "1")] - pub timeline: ::core::option::Option, -} -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct IndexRange { - /// TODO(zehiko) support for other ranges for other index selectors - #[prost(message, optional, tag = "1")] - pub time_range: ::core::option::Option, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexValues { - /// TODO(zehiko) we need to add support for other types of index selectors - #[prost(message, repeated, tag = "1")] - pub time_points: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SampledIndexValues { - #[prost(message, repeated, tag = "1")] - pub sample_points: ::prost::alloc::vec::Vec, -} -/// A 64-bit number describing either nanoseconds, sequence numbers or fully static data. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct TimeInt { - #[prost(int64, tag = "1")] - pub time: i64, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ViewContents { - #[prost(message, repeated, tag = "1")] - pub contents: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ViewContentsPart { - #[prost(message, optional, tag = "1")] - pub path: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub components: ::core::option::Option, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ComponentsSet { - #[prost(message, repeated, tag = "1")] - pub components: ::prost::alloc::vec::Vec, -} -/// The unique identifier of an entity, e.g. `camera/3/points` -/// See <> for more on entity paths. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EntityPath { - #[prost(string, tag = "1")] - pub path: ::prost::alloc::string::String, -} -/// Component describes semantic data that can be used by any number of rerun's archetypes. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Component { - /// component name needs to be a string as user can define their own component - #[prost(string, tag = "1")] - pub name: ::prost::alloc::string::String, -} -/// Used to telect a time column. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TimeColumnSelector { - #[prost(message, optional, tag = "1")] - pub timeline: ::core::option::Option, -} -/// Used to select a component based on its EntityPath and Component name. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ComponentColumnSelector { - #[prost(message, optional, tag = "1")] - pub entity_path: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub component: ::core::option::Option, -} -/// Specifies how null values should be filled in the returned dataframe. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SparseFillStrategy { - None = 0, - LatestAtGlobal = 1, -} -impl SparseFillStrategy { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::None => "NONE", - Self::LatestAtGlobal => "LATEST_AT_GLOBAL", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "NONE" => Some(Self::None), - "LATEST_AT_GLOBAL" => Some(Self::LatestAtGlobal), - _ => None, - } - } -} #[derive(Clone, PartialEq, ::prost::Message)] pub struct RegisterRecordingRequest { /// human readable description of the recording @@ -228,7 +18,7 @@ pub struct RegisterRecordingRequest { /// Recording metadata is single row arrow record batch #[derive(Clone, PartialEq, ::prost::Message)] pub struct RecordingMetadata { - #[prost(enumeration = "EncoderVersion", tag = "1")] + #[prost(enumeration = "super::super::common::v0::EncoderVersion", tag = "1")] pub encoder_version: i32, #[prost(bytes = "vec", tag = "2")] pub payload: ::prost::alloc::vec::Vec, @@ -236,7 +26,7 @@ pub struct RecordingMetadata { #[derive(Clone, PartialEq, ::prost::Message)] pub struct RegisterRecordingResponse { #[prost(message, optional, tag = "1")] - pub id: ::core::option::Option, + pub id: ::core::option::Option, /// Note / TODO(zehiko): this implies we read the record (for example go through entire .rrd file /// chunk by chunk) and extract the metadata. So we might want to 1/ not do this i.e. /// only do it as part of explicit GetMetadata request or 2/ do it if Request has "include_metadata=true" @@ -244,29 +34,42 @@ pub struct RegisterRecordingResponse { #[prost(message, optional, tag = "2")] pub metadata: ::core::option::Option, } +/// Server can include details about the error as part of gRPC error (Status) +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegistrationError { + /// error code + #[prost(enumeration = "super::super::common::v0::ErrorCode", tag = "1")] + pub code: i32, + /// storage url of the recording that failed to register + #[prost(string, tag = "2")] + pub storage_url: ::prost::alloc::string::String, + /// human readable details about the error + #[prost(string, tag = "3")] + pub message: ::prost::alloc::string::String, +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetRecordingMetadataRequest { #[prost(message, optional, tag = "1")] - pub recording_id: ::core::option::Option, + pub recording_id: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct GetRecordingMetadataResponse { #[prost(message, optional, tag = "1")] - pub id: ::core::option::Option, + pub id: ::core::option::Option, #[prost(message, optional, tag = "2")] pub metadata: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct TimeMetadata { #[prost(message, optional, tag = "1")] - pub timeline: ::core::option::Option, + pub timeline: ::core::option::Option, #[prost(message, optional, tag = "2")] - pub time_range: ::core::option::Option, + pub time_range: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct UpdateRecordingMetadataRequest { #[prost(message, optional, tag = "1")] - pub recording_id: ::core::option::Option, + pub recording_id: ::core::option::Option, #[prost(message, optional, tag = "2")] pub metadata: ::core::option::Option, } @@ -276,17 +79,17 @@ pub struct UpdateRecordingMetadataResponse {} pub struct QueryRequest { /// unique identifier of the recording #[prost(message, optional, tag = "1")] - pub recording_id: ::core::option::Option, + pub recording_id: ::core::option::Option, /// query to execute #[prost(message, optional, tag = "3")] - pub query: ::core::option::Option, + pub query: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct QueryResponse { /// TODO(zehiko) we need to expand this to become something like 'encoder options' /// as we will need to specify additional options like compression, including schema /// in payload, etc. - #[prost(enumeration = "EncoderVersion", tag = "1")] + #[prost(enumeration = "super::super::common::v0::EncoderVersion", tag = "1")] pub encoder_version: i32, /// payload is raw bytes that the relevant codec can interpret #[prost(bytes = "vec", tag = "2")] @@ -313,7 +116,7 @@ pub struct ListRecordingsResponse { #[derive(Clone, PartialEq, ::prost::Message)] pub struct FetchRecordingRequest { #[prost(message, optional, tag = "1")] - pub recording_id: ::core::option::Option, + pub recording_id: ::core::option::Option, } /// TODO(jleibs): Eventually this becomes either query-mediated in some way, but for now /// it's useful to be able to just get back the whole RRD somehow. @@ -322,48 +125,12 @@ pub struct FetchRecordingResponse { /// TODO(zehiko) we need to expand this to become something like 'encoder options' /// as we will need to specify additional options like compression, including schema /// in payload, etc. - #[prost(enumeration = "EncoderVersion", tag = "1")] + #[prost(enumeration = "super::super::common::v0::EncoderVersion", tag = "1")] pub encoder_version: i32, /// payload is raw bytes that the relevant codec can interpret #[prost(bytes = "vec", tag = "2")] pub payload: ::prost::alloc::vec::Vec, } -/// Application level error - use as `details` in the `google.rpc.Status` message -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct RemoteStoreError { - /// error code - #[prost(enumeration = "ErrorCode", tag = "1")] - pub code: i32, - /// unique identifier associated with the request (e.g. recording id, recording storage url) - #[prost(string, tag = "2")] - pub id: ::prost::alloc::string::String, - /// human readable details about the error - #[prost(string, tag = "3")] - pub message: ::prost::alloc::string::String, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum EncoderVersion { - V0 = 0, -} -impl EncoderVersion { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::V0 => "V0", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "V0" => Some(Self::V0), - _ => None, - } - } -} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum RecordingType { @@ -387,43 +154,6 @@ impl RecordingType { } } } -/// Error codes for application level errors -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum ErrorCode { - /// unused - Unused = 0, - /// object store access error - ObjectStoreError = 1, - /// metadata database access error - MetadataDbError = 2, - /// Encoding / decoding error - CodecError = 3, -} -impl ErrorCode { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::Unused => "_UNUSED", - Self::ObjectStoreError => "OBJECT_STORE_ERROR", - Self::MetadataDbError => "METADATA_DB_ERROR", - Self::CodecError => "CODEC_ERROR", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "_UNUSED" => Some(Self::Unused), - "OBJECT_STORE_ERROR" => Some(Self::ObjectStoreError), - "METADATA_DB_ERROR" => Some(Self::MetadataDbError), - "CODEC_ERROR" => Some(Self::CodecError), - _ => None, - } - } -} /// Generated client implementations. pub mod storage_node_client { #![allow( @@ -431,10 +161,10 @@ pub mod storage_node_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value + clippy::let_unit_value, )] - use tonic::codegen::http::Uri; use tonic::codegen::*; + use tonic::codegen::http::Uri; #[derive(Debug, Clone)] pub struct StorageNodeClient { inner: tonic::client::Grpc, @@ -467,8 +197,9 @@ pub mod storage_node_client { >::ResponseBody, >, >, - >>::Error: - Into + std::marker::Send + std::marker::Sync, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, { StorageNodeClient::new(InterceptedService::new(inner, interceptor)) } @@ -511,17 +242,21 @@ pub mod storage_node_client { tonic::Response>, tonic::Status, > { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = - http::uri::PathAndQuery::from_static("/rerun.remote_store.v0.StorageNode/Query"); + let path = http::uri::PathAndQuery::from_static( + "/rerun.remote_store.v0.StorageNode/Query", + ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "Query", - )); + req.extensions_mut() + .insert(GrpcMethod::new("rerun.remote_store.v0.StorageNode", "Query")); self.inner.server_streaming(req, path, codec).await } pub async fn fetch_recording( @@ -531,57 +266,85 @@ pub mod storage_node_client { tonic::Response>, tonic::Status, > { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/FetchRecording", ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "FetchRecording", - )); + req.extensions_mut() + .insert( + GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "FetchRecording", + ), + ); self.inner.server_streaming(req, path, codec).await } /// metadata API calls pub async fn list_recordings( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> - { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/ListRecordings", ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "ListRecordings", - )); + req.extensions_mut() + .insert( + GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "ListRecordings", + ), + ); self.inner.unary(req, path, codec).await } pub async fn get_recording_metadata( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> - { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/GetRecordingMetadata", ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "GetRecordingMetadata", - )); + req.extensions_mut() + .insert( + GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "GetRecordingMetadata", + ), + ); self.inner.unary(req, path, codec).await } pub async fn update_recording_metadata( @@ -591,37 +354,55 @@ pub mod storage_node_client { tonic::Response, tonic::Status, > { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/UpdateRecordingMetadata", ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "UpdateRecordingMetadata", - )); + req.extensions_mut() + .insert( + GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "UpdateRecordingMetadata", + ), + ); self.inner.unary(req, path, codec).await } pub async fn register_recording( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> - { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/RegisterRecording", ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "RegisterRecording", - )); + req.extensions_mut() + .insert( + GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "RegisterRecording", + ), + ); self.inner.unary(req, path, codec).await } } @@ -633,7 +414,7 @@ pub mod storage_node_server { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value + clippy::let_unit_value, )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with StorageNodeServer. @@ -642,7 +423,8 @@ pub mod storage_node_server { /// Server streaming response type for the Query method. type QueryStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, - > + std::marker::Send + > + + std::marker::Send + 'static; /// data API calls async fn query( @@ -652,21 +434,31 @@ pub mod storage_node_server { /// Server streaming response type for the FetchRecording method. type FetchRecordingStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, - > + std::marker::Send + > + + std::marker::Send + 'static; async fn fetch_recording( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; /// metadata API calls async fn list_recordings( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn get_recording_metadata( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn update_recording_metadata( &self, request: tonic::Request, @@ -677,7 +469,10 @@ pub mod storage_node_server { async fn register_recording( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } #[derive(Debug)] pub struct StorageNodeServer { @@ -700,7 +495,10 @@ pub mod storage_node_server { max_encoding_message_size: None, } } - pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService where F: tonic::service::Interceptor, { @@ -755,18 +553,24 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/Query" => { #[allow(non_camel_case_types)] struct QuerySvc(pub Arc); - impl tonic::server::ServerStreamingService for QuerySvc { + impl< + T: StorageNode, + > tonic::server::ServerStreamingService + for QuerySvc { type Response = super::QueryResponse; type ResponseStream = T::QueryStream; - type Future = - BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = - async move { ::query(&inner, request).await }; + let fut = async move { + ::query(&inner, request).await + }; Box::pin(fut) } } @@ -795,14 +599,16 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/FetchRecording" => { #[allow(non_camel_case_types)] struct FetchRecordingSvc(pub Arc); - impl - tonic::server::ServerStreamingService - for FetchRecordingSvc - { + impl< + T: StorageNode, + > tonic::server::ServerStreamingService + for FetchRecordingSvc { type Response = super::FetchRecordingResponse; type ResponseStream = T::FetchRecordingStream; - type Future = - BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, request: tonic::Request, @@ -839,11 +645,15 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/ListRecordings" => { #[allow(non_camel_case_types)] struct ListRecordingsSvc(pub Arc); - impl tonic::server::UnaryService - for ListRecordingsSvc - { + impl< + T: StorageNode, + > tonic::server::UnaryService + for ListRecordingsSvc { type Response = super::ListRecordingsResponse; - type Future = BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, request: tonic::Request, @@ -880,19 +690,23 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/GetRecordingMetadata" => { #[allow(non_camel_case_types)] struct GetRecordingMetadataSvc(pub Arc); - impl - tonic::server::UnaryService - for GetRecordingMetadataSvc - { + impl< + T: StorageNode, + > tonic::server::UnaryService + for GetRecordingMetadataSvc { type Response = super::GetRecordingMetadataResponse; - type Future = BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { - ::get_recording_metadata(&inner, request).await + ::get_recording_metadata(&inner, request) + .await }; Box::pin(fut) } @@ -922,19 +736,28 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/UpdateRecordingMetadata" => { #[allow(non_camel_case_types)] struct UpdateRecordingMetadataSvc(pub Arc); - impl - tonic::server::UnaryService - for UpdateRecordingMetadataSvc - { + impl< + T: StorageNode, + > tonic::server::UnaryService + for UpdateRecordingMetadataSvc { type Response = super::UpdateRecordingMetadataResponse; - type Future = BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, - request: tonic::Request, + request: tonic::Request< + super::UpdateRecordingMetadataRequest, + >, ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { - ::update_recording_metadata(&inner, request).await + ::update_recording_metadata( + &inner, + request, + ) + .await }; Box::pin(fut) } @@ -964,19 +787,23 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/RegisterRecording" => { #[allow(non_camel_case_types)] struct RegisterRecordingSvc(pub Arc); - impl - tonic::server::UnaryService - for RegisterRecordingSvc - { + impl< + T: StorageNode, + > tonic::server::UnaryService + for RegisterRecordingSvc { type Response = super::RegisterRecordingResponse; - type Future = BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { - ::register_recording(&inner, request).await + ::register_recording(&inner, request) + .await }; Box::pin(fut) } @@ -1003,19 +830,23 @@ pub mod storage_node_server { }; Box::pin(fut) } - _ => Box::pin(async move { - let mut response = http::Response::new(empty_body()); - let headers = response.headers_mut(); - headers.insert( - tonic::Status::GRPC_STATUS, - (tonic::Code::Unimplemented as i32).into(), - ); - headers.insert( - http::header::CONTENT_TYPE, - tonic::metadata::GRPC_CONTENT_TYPE, - ); - Ok(response) - }), + _ => { + Box::pin(async move { + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } } } } diff --git a/crates/top/rerun/src/lib.rs b/crates/top/rerun/src/lib.rs index fb1b3953db98..0a42e8232d63 100644 --- a/crates/top/rerun/src/lib.rs +++ b/crates/top/rerun/src/lib.rs @@ -145,8 +145,9 @@ pub mod dataframe { /// Everything needed to build custom `ChunkStoreSubscriber`s. pub use re_entity_db::external::re_chunk_store::{ ChunkStore, ChunkStoreConfig, ChunkStoreDiff, ChunkStoreDiffKind, ChunkStoreEvent, - ChunkStoreGeneration, ChunkStoreHandle, ChunkStoreSubscriber, VersionPolicy, + ChunkStoreGeneration, ChunkStoreHandle, ChunkStoreSubscriber, }; +pub use re_log_encoding::VersionPolicy; pub use re_log_types::StoreKind; /// To register a new external data loader, simply add an executable in your $PATH whose name diff --git a/pixi.toml b/pixi.toml index 93d423976d6e..6b3064c11f58 100644 --- a/pixi.toml +++ b/pixi.toml @@ -125,10 +125,8 @@ check-env = "python scripts/check_env.py" # Run the codegen. Optionally pass `--profile` argument if you want. codegen = "cargo --quiet run --package re_types_builder -- " -# Deprecated: use `codegen-proto` instead. -codegen-rstore = "echo '⚠️ Deprecated: use `codegen-proto` instead ⚠️' ; pixi run codegen-proto" -# Run the codegen for our Protobuf/gRPC definitions. -codegen-proto = "cargo --quiet run --package re_protos_builder && pixi run -e cpp format" +# Run the codegen for protobuf types. +codegen-protos = "cargo --quiet run --package re_protos_builder" # Generate the Rerun CLI manual. diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index 397434ddc986..b9c5db4e9840 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -62,6 +62,7 @@ re_chunk = { workspace = true, features = ["arrow"] } re_chunk_store.workspace = true re_dataframe.workspace = true re_log = { workspace = true, features = ["setup"] } +re_log_encoding = { workspace = true } re_log_types.workspace = true re_memory.workspace = true re_sdk = { workspace = true, features = ["data_loaders"] } diff --git a/rerun_py/src/dataframe.rs b/rerun_py/src/dataframe.rs index bae2938ed73f..4b8931e53c12 100644 --- a/rerun_py/src/dataframe.rs +++ b/rerun_py/src/dataframe.rs @@ -21,9 +21,10 @@ use pyo3::{ use re_chunk_store::{ ChunkStore, ChunkStoreConfig, ChunkStoreHandle, ColumnDescriptor, ColumnSelector, ComponentColumnDescriptor, ComponentColumnSelector, QueryExpression, SparseFillStrategy, - TimeColumnDescriptor, TimeColumnSelector, VersionPolicy, ViewContentsSelector, + TimeColumnDescriptor, TimeColumnSelector, ViewContentsSelector, }; use re_dataframe::{QueryEngine, StorageEngine}; +use re_log_encoding::VersionPolicy; use re_log_types::{EntityPathFilter, ResolvedTimeRange, TimeType}; use re_sdk::{ComponentName, EntityPath, StoreId, StoreKind}; From 9b4a9a3eba181c882cccebee52e4bf53e4d38580 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 18:33:31 +0100 Subject: [PATCH 03/47] wip --- Cargo.lock | 2 + crates/store/re_chunk_store/src/dataframe.rs | 16 + .../src/protobuf_conversions.rs | 260 +++++++++ crates/store/re_dataframe/examples/query.rs | 3 +- crates/store/re_grpc_client/Cargo.toml | 1 + crates/store/re_grpc_client/src/lib.rs | 4 +- crates/store/re_log_encoding/src/codec/mod.rs | 38 ++ .../src/codec/wire.rs} | 239 ++++---- .../store/re_log_encoding/src/decoder/mod.rs | 227 +++++--- crates/store/re_log_encoding/src/encoder.rs | 100 ++-- crates/store/re_log_encoding/src/lib.rs | 6 +- crates/store/re_log_encoding/src/protobuf.rs | 252 +++++++++ crates/store/re_log_types/src/lib.rs | 5 + .../re_log_types/src/protobuf_conversions.rs | 530 ++++++++++++++++++ .../re_protos/proto/rerun/v0/common.proto | 9 + .../re_protos/proto/rerun/v0/log_msg.proto | 187 ++++++ crates/store/re_protos/src/lib.rs | 7 + .../store/re_protos/src/v0/rerun.common.v0.rs | 321 +++++++++++ .../re_protos/src/v0/rerun.log_msg.v0.rs | 216 +++++++ .../re_protos/src/v0/rerun.remote_store.v0.rs | 50 ++ crates/utils/re_tuid/Cargo.toml | 2 + crates/utils/re_tuid/src/lib.rs | 2 + .../utils/re_tuid/src/protobuf_conversions.rs | 17 + rerun_py/src/remote.rs | 19 +- scripts/lint.py | 2 +- 25 files changed, 2248 insertions(+), 267 deletions(-) create mode 100644 crates/store/re_chunk_store/src/protobuf_conversions.rs create mode 100644 crates/store/re_log_encoding/src/codec/mod.rs rename crates/store/{re_protos/src/codec.rs => re_log_encoding/src/codec/wire.rs} (65%) create mode 100644 crates/store/re_log_encoding/src/protobuf.rs create mode 100644 crates/store/re_log_types/src/protobuf_conversions.rs create mode 100644 crates/store/re_protos/proto/rerun/v0/log_msg.proto create mode 100644 crates/store/re_protos/src/v0/rerun.common.v0.rs create mode 100644 crates/store/re_protos/src/v0/rerun.log_msg.v0.rs create mode 100644 crates/utils/re_tuid/src/protobuf_conversions.rs diff --git a/Cargo.lock b/Cargo.lock index d7dc5fe19468..b24d23fad3fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5780,6 +5780,7 @@ dependencies = [ name = "re_grpc_client" version = "0.21.0-alpha.1+dev" dependencies = [ + "re_arrow2", "re_chunk", "re_error", "re_log", @@ -6467,6 +6468,7 @@ dependencies = [ "document-features", "getrandom", "once_cell", + "re_protos", "serde", "web-time", ] diff --git a/crates/store/re_chunk_store/src/dataframe.rs b/crates/store/re_chunk_store/src/dataframe.rs index 051ec277e528..ac67eeb776e8 100644 --- a/crates/store/re_chunk_store/src/dataframe.rs +++ b/crates/store/re_chunk_store/src/dataframe.rs @@ -477,20 +477,36 @@ impl std::fmt::Display for SparseFillStrategy { #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] pub struct ViewContentsSelector(pub BTreeMap>>); +impl ViewContentsSelector { + pub fn into_inner(self) -> BTreeMap>> { + self.0 + } +} + impl Deref for ViewContentsSelector { type Target = BTreeMap>>; + #[inline] fn deref(&self) -> &Self::Target { &self.0 } } impl DerefMut for ViewContentsSelector { + #[inline] fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } } +impl FromIterator<(EntityPath, Option>)> for ViewContentsSelector { + fn from_iter>)>>( + iter: T, + ) -> Self { + Self(iter.into_iter().collect()) + } +} + // TODO(cmc): Ultimately, this shouldn't be hardcoded to `Timeline`, but to a generic `I: Index`. // `Index` in this case should also be implemented on tuples (`(I1, I2, ...)`). pub type Index = Timeline; diff --git a/crates/store/re_chunk_store/src/protobuf_conversions.rs b/crates/store/re_chunk_store/src/protobuf_conversions.rs new file mode 100644 index 000000000000..4f9025d32c85 --- /dev/null +++ b/crates/store/re_chunk_store/src/protobuf_conversions.rs @@ -0,0 +1,260 @@ +use std::collections::BTreeMap; +use std::collections::BTreeSet; + +use re_protos::TypeConversionError; + +impl TryFrom for crate::ComponentColumnSelector { + type Error = TypeConversionError; + + fn try_from( + value: re_protos::common::v0::ComponentColumnSelector, + ) -> Result { + let entity_path = value + .entity_path + .ok_or(TypeConversionError::missing_field( + "rerun.common.v0.ComponentColumnSelector", + "entity_path", + ))? + .try_into()?; + + let component_name = value + .component + .ok_or(TypeConversionError::missing_field( + "rerun.common.v0.ComponentColumnSelector", + "component", + ))? + .name; + + Ok(Self { + entity_path, + component_name, + }) + } +} + +impl TryFrom for crate::TimeColumnSelector { + type Error = TypeConversionError; + + fn try_from(value: re_protos::common::v0::TimeColumnSelector) -> Result { + let timeline = value.timeline.ok_or(TypeConversionError::missing_field( + "rerun.common.v0.TimeColumnSelector", + "timeline", + ))?; + + Ok(Self { + timeline: timeline.name.into(), + }) + } +} + +impl TryFrom for crate::ColumnSelector { + type Error = TypeConversionError; + + fn try_from(value: re_protos::common::v0::ColumnSelector) -> Result { + match value + .selector_type + .ok_or(TypeConversionError::missing_field( + "rerun.common.v0.ColumnSelector", + "selector_type", + ))? { + re_protos::common::v0::column_selector::SelectorType::ComponentColumn( + component_column_selector, + ) => { + let selector: crate::ComponentColumnSelector = + component_column_selector.try_into()?; + Ok(selector.into()) + } + re_protos::common::v0::column_selector::SelectorType::TimeColumn( + time_column_selector, + ) => { + let selector: crate::TimeColumnSelector = time_column_selector.try_into()?; + + Ok(selector.into()) + } + } + } +} + +impl From for re_protos::common::v0::ColumnSelector { + fn from(value: crate::ColumnSelector) -> Self { + match value { + crate::ColumnSelector::Component(ccs) => Self { + selector_type: Some( + re_protos::common::v0::column_selector::SelectorType::ComponentColumn( + re_protos::common::v0::ComponentColumnSelector { + entity_path: Some(ccs.entity_path.into()), + component: Some(re_protos::common::v0::Component { + name: ccs.component_name, + }), + }, + ), + ), + }, + crate::ColumnSelector::Time(tcs) => Self { + selector_type: Some( + re_protos::common::v0::column_selector::SelectorType::TimeColumn( + re_protos::common::v0::TimeColumnSelector { + timeline: Some(re_protos::common::v0::Timeline { + name: tcs.timeline.to_string(), + }), + }, + ), + ), + }, + } + } +} + +impl TryFrom for crate::ViewContentsSelector { + type Error = TypeConversionError; + + fn try_from(value: re_protos::common::v0::ViewContents) -> Result { + value + .contents + .into_iter() + .map(|part| { + let entity_path: re_log_types::EntityPath = part + .path + .ok_or(TypeConversionError::missing_field( + "rerun.common.v0.ViewContentsPart", + "path", + ))? + .try_into()?; + let column_selector = part.components.map(|cs| { + cs.components + .into_iter() + .map(|c| re_chunk::ComponentName::new(&c.name)) + .collect::>() + }); + Ok((entity_path, column_selector)) + }) + .collect::, Self::Error>>() + .map(crate::ViewContentsSelector) + } +} + +impl TryFrom for crate::QueryExpression { + type Error = TypeConversionError; + + fn try_from(value: re_protos::common::v0::Query) -> Result { + let filtered_index = value + .filtered_index + .ok_or(TypeConversionError::missing_field( + "rerun.common.v0.Query", + "filtered_index", + ))? + .try_into()?; + + let selection = value + .column_selection + .map(|cs| { + cs.columns + .into_iter() + .map(crate::ColumnSelector::try_from) + .collect::, _>>() + }) + .transpose()?; + + let filtered_is_not_null = value + .filtered_is_not_null + .map(crate::ComponentColumnSelector::try_from) + .transpose()?; + + Ok(Self { + view_contents: value.view_contents.map(|vc| vc.try_into()).transpose()?, + include_semantically_empty_columns: value.include_semantically_empty_columns, + include_indicator_columns: value.include_indicator_columns, + include_tombstone_columns: value.include_tombstone_columns, + filtered_index: Some(filtered_index), + filtered_index_range: value + .filtered_index_range + .map(|ir| ir.try_into()) + .transpose()?, + filtered_index_values: value + .filtered_index_values + .map(|iv| iv.time_points.into_iter().map(|v| v.into()).collect()), + using_index_values: value + .using_index_values + .map(|uiv| uiv.time_points.into_iter().map(|v| v.into()).collect()), + filtered_is_not_null, + sparse_fill_strategy: crate::SparseFillStrategy::default(), // TODO(zehiko) implement support for sparse fill strategy + selection, + }) + } +} + +impl From for re_protos::common::v0::Query { + fn from(value: crate::QueryExpression) -> Self { + Self { + view_contents: value + .view_contents + .map(|vc| { + vc.into_inner() + .into_iter() + .map( + |(path, components)| re_protos::common::v0::ViewContentsPart { + path: Some(path.into()), + components: components.map(|cs| { + re_protos::common::v0::ComponentsSet { + components: cs + .into_iter() + .map(|c| re_protos::common::v0::Component { + name: c.to_string(), + }) + .collect(), + } + }), + }, + ) + .collect::>() + }) + .map(|cs| re_protos::common::v0::ViewContents { contents: cs }), + include_semantically_empty_columns: value.include_semantically_empty_columns, + include_indicator_columns: value.include_indicator_columns, + include_tombstone_columns: value.include_tombstone_columns, + filtered_index: value.filtered_index.map(|timeline| { + re_protos::common::v0::IndexColumnSelector { + timeline: Some(re_protos::common::v0::Timeline { + name: timeline.name().to_string(), + }), + } + }), + filtered_index_range: value.filtered_index_range.map(|ir| { + re_protos::common::v0::IndexRange { + time_range: Some(ir.into()), + } + }), + filtered_index_values: value.filtered_index_values.map(|iv| { + re_protos::common::v0::IndexValues { + time_points: iv + .into_iter() + // TODO(zehiko) is this desired behavior for TimeInt::STATIC? + .map(|v| re_protos::common::v0::TimeInt { time: v.as_i64() }) + .collect(), + } + }), + using_index_values: value.using_index_values.map(|uiv| { + re_protos::common::v0::IndexValues { + time_points: uiv + .into_iter() + .map(|v| re_protos::common::v0::TimeInt { time: v.as_i64() }) + .collect(), + } + }), + filtered_is_not_null: value.filtered_is_not_null.map(|cs| { + re_protos::common::v0::ComponentColumnSelector { + entity_path: Some(cs.entity_path.into()), + component: Some(re_protos::common::v0::Component { + name: cs.component_name, + }), + } + }), + column_selection: value + .selection + .map(|cs| re_protos::common::v0::ColumnSelection { + columns: cs.into_iter().map(|c| c.into()).collect(), + }), + sparse_fill_strategy: re_protos::common::v0::SparseFillStrategy::None.into(), // TODO(zehiko) implement + } + } +} diff --git a/crates/store/re_dataframe/examples/query.rs b/crates/store/re_dataframe/examples/query.rs index 3cc999177439..f4ffc1af746d 100644 --- a/crates/store/re_dataframe/examples/query.rs +++ b/crates/store/re_dataframe/examples/query.rs @@ -4,8 +4,9 @@ use itertools::Itertools; use re_dataframe::{ ChunkStoreConfig, EntityPathFilter, QueryEngine, QueryExpression, ResolvedTimeRange, - SparseFillStrategy, StoreKind, TimeInt, Timeline, VersionPolicy, + SparseFillStrategy, StoreKind, TimeInt, Timeline, }; +use re_log_encoding::VersionPolicy; fn main() -> anyhow::Result<()> { let args = std::env::args().collect_vec(); diff --git a/crates/store/re_grpc_client/Cargo.toml b/crates/store/re_grpc_client/Cargo.toml index 58fea586e418..efad14e9615d 100644 --- a/crates/store/re_grpc_client/Cargo.toml +++ b/crates/store/re_grpc_client/Cargo.toml @@ -28,6 +28,7 @@ re_log_types.workspace = true re_protos.workspace = true re_smart_channel.workspace = true +arrow2.workspace = true thiserror.workspace = true tokio-stream.workspace = true diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs index 5576f15a4bd2..7d313995c8e4 100644 --- a/crates/store/re_grpc_client/src/lib.rs +++ b/crates/store/re_grpc_client/src/lib.rs @@ -9,7 +9,7 @@ pub use address::{Address, InvalidAddressError}; use std::{error::Error, str::FromStr}; use re_chunk::Chunk; -use re_log_encoding::codec::{decode, CodecError}; +use re_log_encoding::codec::{self, CodecError}; use re_log_types::{ ApplicationId, LogMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; @@ -193,7 +193,7 @@ async fn stream_recording_async( re_log::info!("Starting to read..."); while let Some(result) = resp.next().await { let response = result.map_err(TonicStatusError)?; - let tc = decode(EncoderVersion::V0, &response.payload)?; + let tc = codec::wire::decode(EncoderVersion::V0, &response.payload)?; let Some(tc) = tc else { return Err(StreamError::MissingTransportChunk); diff --git a/crates/store/re_log_encoding/src/codec/mod.rs b/crates/store/re_log_encoding/src/codec/mod.rs new file mode 100644 index 000000000000..aab99001deed --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/mod.rs @@ -0,0 +1,38 @@ +// pub mod file; +pub mod wire; + +#[derive(Debug, thiserror::Error)] +pub enum CodecError { + #[error("Arrow serialization error: {0}")] + ArrowSerialization(arrow2::error::Error), + + #[error("Failed to decode message header {0}")] + HeaderDecoding(std::io::Error), + + #[error("Failed to encode message header {0}")] + HeaderEncoding(std::io::Error), + + #[error("Missing record batch")] + MissingRecordBatch, + + #[error("Unexpected stream state")] + UnexpectedStreamState, + + #[error("Unsupported encoding, expected Arrow IPC")] + UnsupportedEncoding, + + #[error("Invalid file header")] + InvalidFileHeader, + + #[error("Unknown message header")] + UnknownMessageHeader, + + #[error("Invalid message header")] + InvalidMessageHeader, + + #[error("Unknown message kind {0}")] + UnknownMessageKind(u8), + + #[error("Invalid argument: {0}")] + InvalidArgument(String), +} diff --git a/crates/store/re_protos/src/codec.rs b/crates/store/re_log_encoding/src/codec/wire.rs similarity index 65% rename from crates/store/re_protos/src/codec.rs rename to crates/store/re_log_encoding/src/codec/wire.rs index 12a4b0a08382..f9b3815af529 100644 --- a/crates/store/re_protos/src/codec.rs +++ b/crates/store/re_log_encoding/src/codec/wire.rs @@ -1,35 +1,10 @@ -use arrow2::array::Array as Arrow2Array; use arrow2::chunk::Chunk as Arrow2Chunk; use arrow2::datatypes::Schema as Arrow2Schema; -use arrow2::error::Error as Arrow2Error; -use arrow2::io::ipc::{read, write}; -use re_dataframe::TransportChunk; +use arrow2::io::ipc; +use re_chunk::Arrow2Array; +use re_chunk::TransportChunk; -use crate::v0::{EncoderVersion, RecordingMetadata}; - -#[derive(Debug, thiserror::Error)] -pub enum CodecError { - #[error("Arrow serialization error: {0}")] - ArrowSerialization(Arrow2Error), - - #[error("Failed to decode message header {0}")] - HeaderDecoding(std::io::Error), - - #[error("Failed to encode message header {0}")] - HeaderEncoding(std::io::Error), - - #[error("Missing record batch")] - MissingRecordBatch, - - #[error("Unexpected stream state")] - UnexpectedStreamState, - - #[error("Unknown message header")] - UnknownMessageHeader, - - #[error("Invalid argument: {0}")] - InvalidArgument(String), -} +use super::CodecError; #[derive(Clone, Copy, PartialEq, Eq, Hash, Default)] pub struct MessageHeader(pub u8); @@ -111,115 +86,125 @@ impl TransportMessageV0 { // of sending schema in each transport message for the same stream of batches. This will require codec // to become stateful and keep track if schema was sent / received. /// Encode a transport chunk into a byte stream. -pub fn encode(version: EncoderVersion, chunk: TransportChunk) -> Result, CodecError> { +pub fn encode( + version: re_protos::common::v0::EncoderVersion, + chunk: TransportChunk, +) -> Result, CodecError> { match version { - EncoderVersion::V0 => TransportMessageV0::RecordBatch(chunk).to_bytes(), + re_protos::common::v0::EncoderVersion::V0 => { + TransportMessageV0::RecordBatch(chunk).to_bytes() + } } } -/// Encode a `NoData` message into a byte stream. This can be used by the remote store -/// (i.e. data producer) to signal back to the client that there's no data available. -pub fn no_data(version: EncoderVersion) -> Result, CodecError> { - match version { - EncoderVersion::V0 => TransportMessageV0::NoData.to_bytes(), - } -} +/// Create `RecordingMetadata` from `TransportChunk`. We rely on `TransportChunk` until +/// we migrate from arrow2 to arrow. +pub fn chunk_to_recording_metadata( + version: re_protos::common::v0::EncoderVersion, + metadata: &TransportChunk, +) -> Result { + if metadata.data.len() != 1 { + return Err(CodecError::InvalidArgument(format!( + "metadata record batch can only have a single row, batch with {} rows given", + metadata.data.len() + ))); + }; -/// Decode transport data from a byte stream - if there's a record batch present, return it, otherwise return `None`. -pub fn decode(version: EncoderVersion, data: &[u8]) -> Result, CodecError> { match version { - EncoderVersion::V0 => { - let msg = TransportMessageV0::from_bytes(data)?; - match msg { - TransportMessageV0::RecordBatch(chunk) => Ok(Some(chunk)), - TransportMessageV0::NoData => Ok(None), - } + re_protos::common::v0::EncoderVersion::V0 => { + let mut data: Vec = Vec::new(); + write_arrow_to_bytes(&mut data, &metadata.schema, &metadata.data)?; + + Ok(re_protos::remote_store::v0::RecordingMetadata { + encoder_version: version as i32, + payload: data, + }) } } } -impl RecordingMetadata { - /// Create `RecordingMetadata` from `TransportChunk`. We rely on `TransportChunk` until - /// we migrate from arrow2 to arrow. - pub fn try_from( - version: EncoderVersion, - metadata: &TransportChunk, - ) -> Result { - if metadata.data.len() != 1 { - return Err(CodecError::InvalidArgument(format!( - "metadata record batch can only have a single row, batch with {} rows given", - metadata.data.len() - ))); - }; +/// Get metadata as arrow data +pub fn recording_metadata_to_chunk( + metadata: &re_protos::remote_store::v0::RecordingMetadata, +) -> Result { + let mut reader = std::io::Cursor::new(metadata.payload.clone()); - match version { - EncoderVersion::V0 => { - let mut data: Vec = Vec::new(); - write_arrow_to_bytes(&mut data, &metadata.schema, &metadata.data)?; + let encoder_version = re_protos::common::v0::EncoderVersion::try_from(metadata.encoder_version) + .map_err(|err| CodecError::InvalidArgument(err.to_string()))?; - Ok(Self { - encoder_version: version as i32, - payload: data, - }) - } + match encoder_version { + re_protos::common::v0::EncoderVersion::V0 => { + let (schema, data) = read_arrow_from_bytes(&mut reader)?; + Ok(TransportChunk { schema, data }) } } +} - /// Get metadata as arrow data - pub fn data(&self) -> Result { - let mut reader = std::io::Cursor::new(self.payload.clone()); +/// Returns unique id of the recording +pub fn recording_id( + metadata: &re_protos::remote_store::v0::RecordingMetadata, +) -> Result { + let chunk = recording_metadata_to_chunk(metadata)?; + let id_pos = chunk + .schema + .fields + .iter() + // TODO(zehiko) we need to figure out where mandatory fields live + .position(|field| field.name == "id") + .ok_or_else(|| CodecError::InvalidArgument("missing id field in schema".to_owned()))?; - let encoder_version = EncoderVersion::try_from(self.encoder_version) - .map_err(|err| CodecError::InvalidArgument(err.to_string()))?; + use arrow2::array::Utf8Array as Arrow2Utf8Array; - match encoder_version { - EncoderVersion::V0 => { - let (schema, data) = read_arrow_from_bytes(&mut reader)?; - Ok(TransportChunk { schema, data }) - } - } + let id = chunk.data.columns()[id_pos] + .as_any() + .downcast_ref::>() + .ok_or_else(|| { + CodecError::InvalidArgument(format!( + "unexpected type for id with position {id_pos} in schema: {:?}", + chunk.schema + )) + })? + .value(0); + + Ok(re_log_types::StoreId::from_string( + re_log_types::StoreKind::Recording, + id.to_owned(), + )) +} + +/// Encode a `NoData` message into a byte stream. This can be used by the remote store +/// (i.e. data producer) to signal back to the client that there's no data available. +pub fn no_data(version: re_protos::common::v0::EncoderVersion) -> Result, CodecError> { + match version { + re_protos::common::v0::EncoderVersion::V0 => TransportMessageV0::NoData.to_bytes(), } +} - /// Returns unique id of the recording - pub fn id(&self) -> Result { - let metadata = self.data()?; - let id_pos = metadata - .schema - .fields - .iter() - // TODO(zehiko) we need to figure out where mandatory fields live - .position(|field| field.name == "id") - .ok_or_else(|| CodecError::InvalidArgument("missing id field in schema".to_owned()))?; - - use arrow2::array::Utf8Array as Arrow2Utf8Array; - - let id = metadata.data.columns()[id_pos] - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - CodecError::InvalidArgument(format!( - "Unexpected type for id with position {id_pos} in schema: {:?}", - metadata.schema - )) - })? - .value(0); - - Ok(re_log_types::StoreId::from_string( - re_log_types::StoreKind::Recording, - id.to_owned(), - )) +/// Decode transport data from a byte stream - if there's a record batch present, return it, otherwise return `None`. +pub fn decode( + version: re_protos::common::v0::EncoderVersion, + data: &[u8], +) -> Result, CodecError> { + match version { + re_protos::common::v0::EncoderVersion::V0 => { + let msg = TransportMessageV0::from_bytes(data)?; + match msg { + TransportMessageV0::RecordBatch(chunk) => Ok(Some(chunk)), + TransportMessageV0::NoData => Ok(None), + } + } } } /// Helper function that serializes given arrow schema and record batch into bytes /// using Arrow IPC format. -fn write_arrow_to_bytes( +pub fn write_arrow_to_bytes( writer: &mut W, schema: &Arrow2Schema, data: &Arrow2Chunk>, ) -> Result<(), CodecError> { - let options = write::WriteOptions { compression: None }; - let mut sw = write::StreamWriter::new(writer, options); + let options = ipc::write::WriteOptions { compression: None }; + let mut sw = ipc::write::StreamWriter::new(writer, options); sw.start(schema, None) .map_err(CodecError::ArrowSerialization)?; @@ -232,11 +217,12 @@ fn write_arrow_to_bytes( /// Helper function that deserializes raw bytes into arrow schema and record batch /// using Arrow IPC format. -fn read_arrow_from_bytes( +pub fn read_arrow_from_bytes( reader: &mut R, ) -> Result<(Arrow2Schema, Arrow2Chunk>), CodecError> { - let metadata = read::read_stream_metadata(reader).map_err(CodecError::ArrowSerialization)?; - let mut stream = read::StreamReader::new(reader, metadata, None); + let metadata = + ipc::read::read_stream_metadata(reader).map_err(CodecError::ArrowSerialization)?; + let mut stream = ipc::read::StreamReader::new(reader, metadata, None); let schema = stream.schema().clone(); // there should be at least one record batch in the stream @@ -246,8 +232,8 @@ fn read_arrow_from_bytes( .map_err(CodecError::ArrowSerialization)?; match stream_state { - read::StreamState::Waiting => Err(CodecError::UnexpectedStreamState), - read::StreamState::Some(chunk) => Ok((schema, chunk)), + ipc::read::StreamState::Waiting => Err(CodecError::UnexpectedStreamState), + ipc::read::StreamState::Some(chunk) => Ok((schema, chunk)), } } @@ -259,16 +245,17 @@ mod tests { array::Int32Array as Arrow2Int32Array, datatypes::Field as Arrow2Field, datatypes::Schema as Arrow2Schema, }; - use re_dataframe::external::re_chunk::{Chunk, RowId}; - use re_dataframe::TransportChunk; + use re_chunk::TransportChunk; + use re_chunk::{Chunk, RowId}; use re_log_types::StoreId; use re_log_types::{example_components::MyPoint, Timeline}; - use crate::v0::RecordingMetadata; - use crate::{ - codec::{decode, encode, CodecError, TransportMessageV0}, - v0::EncoderVersion, - }; + use crate::codec::wire::chunk_to_recording_metadata; + use crate::codec::wire::recording_id; + use crate::codec::wire::recording_metadata_to_chunk; + use crate::codec::wire::{decode, encode, TransportMessageV0}; + use crate::codec::CodecError; + use re_protos::common::v0::EncoderVersion; fn get_test_chunk() -> Chunk { let row_id1 = RowId::new(); @@ -372,13 +359,13 @@ mod tests { data: expected_chunk.clone(), }; - let metadata = RecordingMetadata::try_from(EncoderVersion::V0, &metadata_tc).unwrap(); + let metadata = chunk_to_recording_metadata(EncoderVersion::V0, &metadata_tc).unwrap(); assert_eq!( StoreId::from_string(re_log_types::StoreKind::Recording, "some_id".to_owned()), - metadata.id().unwrap() + recording_id(&metadata).unwrap() ); - let tc = metadata.data().unwrap(); + let tc = recording_metadata_to_chunk(&metadata).unwrap(); assert_eq!(expected_schema, tc.schema); assert_eq!(expected_chunk, tc.data); @@ -400,7 +387,7 @@ mod tests { data: expected_chunk, }; - let metadata = RecordingMetadata::try_from(EncoderVersion::V0, &metadata_tc); + let metadata = chunk_to_recording_metadata(EncoderVersion::V0, &metadata_tc); assert!(matches!( metadata.err().unwrap(), diff --git a/crates/store/re_log_encoding/src/decoder/mod.rs b/crates/store/re_log_encoding/src/decoder/mod.rs index 9dd4150c38c3..617b2cc0a6f4 100644 --- a/crates/store/re_log_encoding/src/decoder/mod.rs +++ b/crates/store/re_log_encoding/src/decoder/mod.rs @@ -83,10 +83,22 @@ pub enum DecodeError { Options(#[from] crate::OptionsError), #[error("Failed to read: {0}")] - Read(std::io::Error), + Read(#[from] std::io::Error), #[error("lz4 error: {0}")] - Lz4(lz4_flex::block::DecompressError), + Lz4(#[from] lz4_flex::block::DecompressError), + + #[error("Protobuf error: {0}")] + Protobuf(#[from] re_protos::external::prost::DecodeError), + + #[error("Could not convert type from protobuf: {0}")] + TypeConversion(#[from] re_protos::TypeConversionError), + + #[error("Failed to read chunk: {0}")] + Chunk(#[from] re_chunk::ChunkError), + + #[error("Arrow error: {0}")] + Arrow(#[from] arrow2::error::Error), #[error("MsgPack error: {0}")] MsgPack(#[from] rmp_serde::decode::Error), @@ -121,7 +133,7 @@ pub fn read_options( let FileHeader { magic, version, - mut options, + options, } = FileHeader::decode(&mut read)?; if OLD_RRD_HEADERS.contains(&magic) { @@ -133,10 +145,7 @@ pub fn read_options( warn_on_version_mismatch(version_policy, version)?; match options.serializer { - Serializer::MsgPack => {} - Serializer::Protobuf => { - options.compression = Compression::Off; - } + Serializer::MsgPack | Serializer::Protobuf => {} } Ok((CrateVersion::from_bytes(version), options)) @@ -240,6 +249,7 @@ impl Decoder { self.version } + // TODO(jan): stop returning number of read bytes, use cursors wrapping readers instead. /// Returns the size in bytes of the data that has been decoded up to now. #[inline] pub fn size_bytes(&self) -> u64 { @@ -295,94 +305,110 @@ impl Iterator for Decoder { self.size_bytes += FileHeader::SIZE as u64; } - let header = match MessageHeader::decode(&mut self.read) { - Ok(header) => header, - Err(err) => match err { - DecodeError::Read(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { - return None; - } - other => return Some(Err(other)), - }, - }; - self.size_bytes += MessageHeader::SIZE as u64; - - let (uncompressed_len, compressed_len) = match header { - MessageHeader::Data { - compressed_len, - uncompressed_len, - } => (uncompressed_len as usize, compressed_len as usize), - MessageHeader::EndOfStream => { - // we might have a concatenated stream, so we peek beyond end of file marker to see - if self.peek_file_header() { - re_log::debug!("Reached end of stream, but it seems we have a concatenated file, continuing"); - return self.next(); - } - - re_log::debug!("Reached end of stream, iterator complete"); - return None; - } - }; - - self.uncompressed - .resize(self.uncompressed.len().max(uncompressed_len), 0); - - match self.options.compression { - Compression::Off => { - re_tracing::profile_scope!("read uncompressed"); - if let Err(err) = self - .read - .read_exact(&mut self.uncompressed[..uncompressed_len]) - { - return Some(Err(DecodeError::Read(err))); + let msg = match self.options.serializer { + Serializer::Protobuf => { + match crate::protobuf::decode(&mut self.read, self.options.compression) { + Ok((read_bytes, msg)) => { + self.size_bytes += read_bytes; + msg + } + Err(err) => return Some(Err(err)), } - self.size_bytes += uncompressed_len as u64; } - - Compression::LZ4 => { - self.compressed - .resize(self.compressed.len().max(compressed_len), 0); - - { - re_tracing::profile_scope!("read compressed"); - if let Err(err) = self.read.read_exact(&mut self.compressed[..compressed_len]) { - return Some(Err(DecodeError::Read(err))); + Serializer::MsgPack => { + let header = match MessageHeader::decode(&mut self.read) { + Ok(header) => header, + Err(err) => match err { + DecodeError::Read(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { + return None; + } + other => return Some(Err(other)), + }, + }; + self.size_bytes += MessageHeader::SIZE as u64; + + match header { + MessageHeader::Data { + compressed_len, + uncompressed_len, + } => { + let uncompressed_len = uncompressed_len as usize; + let compressed_len = compressed_len as usize; + + self.uncompressed + .resize(self.uncompressed.len().max(uncompressed_len), 0); + + match self.options.compression { + Compression::Off => { + re_tracing::profile_scope!("read uncompressed"); + if let Err(err) = self + .read + .read_exact(&mut self.uncompressed[..uncompressed_len]) + { + return Some(Err(DecodeError::Read(err))); + } + self.size_bytes += uncompressed_len as u64; + } + + Compression::LZ4 => { + self.compressed + .resize(self.compressed.len().max(compressed_len), 0); + + { + re_tracing::profile_scope!("read compressed"); + if let Err(err) = + self.read.read_exact(&mut self.compressed[..compressed_len]) + { + return Some(Err(DecodeError::Read(err))); + } + } + + re_tracing::profile_scope!("lz4"); + if let Err(err) = lz4_flex::block::decompress_into( + &self.compressed[..compressed_len], + &mut self.uncompressed[..uncompressed_len], + ) { + return Some(Err(DecodeError::Lz4(err))); + } + + self.size_bytes += compressed_len as u64; + } + } + + let data = &self.uncompressed[..uncompressed_len]; + { + re_tracing::profile_scope!("MsgPack deser"); + match rmp_serde::from_slice::(data) { + Ok(msg) => Some(msg), + Err(err) => return Some(Err(err.into())), + } + } } + MessageHeader::EndOfStream => None, } - - re_tracing::profile_scope!("lz4"); - if let Err(err) = lz4_flex::block::decompress_into( - &self.compressed[..compressed_len], - &mut self.uncompressed[..uncompressed_len], - ) { - return Some(Err(DecodeError::Lz4(err))); - } - - self.size_bytes += compressed_len as u64; } - } + }; - let data = &self.uncompressed[..uncompressed_len]; - let result = match self.options.serializer { - Serializer::MsgPack => { - re_tracing::profile_scope!("MsgPack deser"); - rmp_serde::from_slice(data).map_err(|e| e.into()) - } - Serializer::Protobuf => { - re_tracing::profile_scope!("Protobuf deser"); - codec::decode_log_msg(data).map_err(|e| e.into()) + let Some(mut msg) = msg else { + // we might have a concatenated stream, so we peek beyond end of file marker to see + if self.peek_file_header() { + re_log::debug!( + "Reached end of stream, but it seems we have a concatenated file, continuing" + ); + return self.next(); } + + re_log::debug!("Reached end of stream, iterator complete"); + return None; }; - match result { - Ok(re_log_types::LogMsg::SetStoreInfo(mut msg)) => { - // Propagate the protocol version from the header into the `StoreInfo` so that all - // parts of the app can easily access it. - msg.info.store_version = Some(self.version()); - Some(Ok(re_log_types::LogMsg::SetStoreInfo(msg))) - } - Ok(msg) => Some(Ok(msg)), - Err(err) => Some(Err(err)), + if let LogMsg::SetStoreInfo(msg) = &mut msg { + // Propagate the protocol version from the header into the `StoreInfo` so that all + // parts of the app can easily access it. + msg.info.store_version = Some(self.version()); } + + Some(Ok(msg)) } } @@ -397,6 +423,21 @@ mod tests { ApplicationId, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; + // useful for debugging reads in the absence of a functional debugger + /* + struct PrintReader { + inner: R, + } + + impl std::io::Read for PrintReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let read = self.inner.read(buf)?; + println!("read {} bytes: {:?}", read, &buf[..read]); + Ok(read) + } + } + */ + fn fake_log_message() -> LogMsg { LogMsg::SetStoreInfo(SetStoreInfo { row_id: *RowId::new(), @@ -430,6 +471,14 @@ mod tests { compression: Compression::LZ4, serializer: Serializer::MsgPack, }, + EncodingOptions { + compression: Compression::Off, + serializer: Serializer::Protobuf, + }, + EncodingOptions { + compression: Compression::LZ4, + serializer: Serializer::Protobuf, + }, ]; for options in options { @@ -457,6 +506,14 @@ mod tests { compression: Compression::LZ4, serializer: Serializer::MsgPack, }, + EncodingOptions { + compression: Compression::Off, + serializer: Serializer::Protobuf, + }, + EncodingOptions { + compression: Compression::LZ4, + serializer: Serializer::Protobuf, + }, ]; for options in options { diff --git a/crates/store/re_log_encoding/src/encoder.rs b/crates/store/re_log_encoding/src/encoder.rs index 73127fc2c1a0..b43943a8d378 100644 --- a/crates/store/re_log_encoding/src/encoder.rs +++ b/crates/store/re_log_encoding/src/encoder.rs @@ -1,14 +1,13 @@ //! Encoding of [`LogMsg`]es as a binary stream, e.g. to store in an `.rrd` file, or send over network. -use re_build_info::CrateVersion; -use re_chunk::{ChunkError, ChunkResult}; -use re_log_types::LogMsg; - use crate::codec; use crate::FileHeader; use crate::MessageHeader; use crate::Serializer; use crate::{Compression, EncodingOptions}; +use re_build_info::CrateVersion; +use re_chunk::{ChunkError, ChunkResult}; +use re_log_types::LogMsg; // ---------------------------------------------------------------------------- @@ -16,14 +15,20 @@ use crate::{Compression, EncodingOptions}; #[derive(thiserror::Error, Debug)] pub enum EncodeError { #[error("Failed to write: {0}")] - Write(std::io::Error), + Write(#[from] std::io::Error), #[error("lz4 error: {0}")] - Lz4(lz4_flex::block::CompressError), + Lz4(#[from] lz4_flex::block::CompressError), #[error("MsgPack error: {0}")] MsgPack(#[from] rmp_serde::encode::Error), + #[error("Protobuf error: {0}")] + Protobuf(#[from] re_protos::external::prost::EncodeError), + + #[error("Arrow error: {0}")] + Arrow(#[from] arrow2::error::Error), + #[error("{0}")] Codec(#[from] codec::CodecError), @@ -134,15 +139,9 @@ impl Encoder { } .encode(&mut write)?; - let serializer = options.serializer; - let compression = match serializer { - Serializer::MsgPack => options.compression, - Serializer::Protobuf => Compression::Off, - }; - Ok(Self { - serializer, - compression, + serializer: options.serializer, + compression: options.compression, write, uncompressed: Vec::new(), compressed: Vec::new(), @@ -155,42 +154,50 @@ impl Encoder { self.uncompressed.clear(); match self.serializer { - Serializer::MsgPack => { - rmp_serde::encode::write_named(&mut self.uncompressed, message)?; - } Serializer::Protobuf => { - crate::codec::encode_log_msg(message, &mut self.uncompressed)?; - } - } + crate::protobuf::encode(&mut self.uncompressed, message, self.compression)?; - match self.compression { - Compression::Off => { - MessageHeader::Data { - uncompressed_len: self.uncompressed.len() as u32, - compressed_len: self.uncompressed.len() as u32, - } - .encode(&mut self.write)?; self.write .write_all(&self.uncompressed) .map(|_| self.uncompressed.len() as _) .map_err(EncodeError::Write) } + Serializer::MsgPack => { + rmp_serde::encode::write_named(&mut self.uncompressed, message)?; - Compression::LZ4 => { - let max_len = lz4_flex::block::get_maximum_output_size(self.uncompressed.len()); - self.compressed.resize(max_len, 0); - let compressed_len = - lz4_flex::block::compress_into(&self.uncompressed, &mut self.compressed) + match self.compression { + Compression::Off => { + MessageHeader::Data { + uncompressed_len: self.uncompressed.len() as u32, + compressed_len: self.uncompressed.len() as u32, + } + .encode(&mut self.write)?; + self.write + .write_all(&self.uncompressed) + .map(|_| self.uncompressed.len() as _) + .map_err(EncodeError::Write) + } + + Compression::LZ4 => { + let max_len = + lz4_flex::block::get_maximum_output_size(self.uncompressed.len()); + self.compressed.resize(max_len, 0); + let compressed_len = lz4_flex::block::compress_into( + &self.uncompressed, + &mut self.compressed, + ) .map_err(EncodeError::Lz4)?; - MessageHeader::Data { - uncompressed_len: self.uncompressed.len() as u32, - compressed_len: compressed_len as u32, + MessageHeader::Data { + uncompressed_len: self.uncompressed.len() as u32, + compressed_len: compressed_len as u32, + } + .encode(&mut self.write)?; + self.write + .write_all(&self.compressed[..compressed_len]) + .map(|_| compressed_len as _) + .map_err(EncodeError::Write) + } } - .encode(&mut self.write)?; - self.write - .write_all(&self.compressed[..compressed_len]) - .map(|_| compressed_len as _) - .map_err(EncodeError::Write) } } } @@ -199,7 +206,18 @@ impl Encoder { // does a partial move. #[inline] pub fn finish(&mut self) -> Result<(), EncodeError> { - MessageHeader::EndOfStream.encode(&mut self.write)?; + match self.serializer { + Serializer::MsgPack => { + MessageHeader::EndOfStream.encode(&mut self.write)?; + } + Serializer::Protobuf => { + crate::protobuf::MessageHeader { + kind: crate::protobuf::MessageKind::End, + len: 0, + } + .encode(&mut self.write)?; + } + } Ok(()) } diff --git a/crates/store/re_log_encoding/src/lib.rs b/crates/store/re_log_encoding/src/lib.rs index 064ad9ad575f..54bb4396744f 100644 --- a/crates/store/re_log_encoding/src/lib.rs +++ b/crates/store/re_log_encoding/src/lib.rs @@ -8,6 +8,8 @@ pub use decoder::VersionPolicy; #[cfg(feature = "encoder")] pub mod encoder; +mod protobuf; + pub mod codec; #[cfg(feature = "encoder")] @@ -26,7 +28,7 @@ pub use file_sink::{FileSink, FileSinkError}; // ---------------------------------------------------------------------------- #[cfg(any(feature = "encoder", feature = "decoder"))] -const RRD_HEADER: &[u8; 4] = b"RRF3"; +const RRD_HEADER: &[u8; 4] = b"RRIO"; #[cfg(feature = "decoder")] const OLD_RRD_HEADERS: &[[u8; 4]] = &[*b"RRF0", *b"RRF1", *b"RRF2"]; @@ -67,7 +69,7 @@ impl EncodingOptions { serializer: Serializer::MsgPack, }; pub const PROTOBUF: Self = Self { - compression: Compression::Off, + compression: Compression::LZ4, serializer: Serializer::Protobuf, }; diff --git a/crates/store/re_log_encoding/src/protobuf.rs b/crates/store/re_log_encoding/src/protobuf.rs new file mode 100644 index 000000000000..b24846b4940b --- /dev/null +++ b/crates/store/re_log_encoding/src/protobuf.rs @@ -0,0 +1,252 @@ +use re_log_types::LogMsg; +use re_protos::TypeConversionError; + +use crate::codec::CodecError; +use crate::decoder::DecodeError; +use crate::encoder::EncodeError; +use crate::Compression; + +#[derive(Debug, Clone, Copy)] +#[repr(u32)] +pub(crate) enum MessageKind { + SetStoreInfo = 1, + ArrowMsg = 2, + BlueprintActivationCommand = 3, + End = 255, +} + +impl MessageKind { + pub(crate) fn encode(&self, buf: &mut impl std::io::Write) -> Result<(), EncodeError> { + let kind: u32 = *self as u32; + buf.write_all(&kind.to_le_bytes())?; + Ok(()) + } + + pub(crate) fn decode(data: &mut impl std::io::Read) -> Result { + let mut buf = [0; 4]; + data.read_exact(&mut buf)?; + + match u32::from_le_bytes(buf) { + 1 => Ok(Self::SetStoreInfo), + 2 => Ok(Self::ArrowMsg), + 3 => Ok(Self::BlueprintActivationCommand), + 255 => Ok(Self::End), + _ => Err(DecodeError::Codec(CodecError::UnknownMessageHeader)), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct MessageHeader { + pub(crate) kind: MessageKind, + pub(crate) len: u32, +} + +impl MessageHeader { + pub(crate) fn encode(&self, buf: &mut impl std::io::Write) -> Result<(), EncodeError> { + self.kind.encode(buf)?; + buf.write_all(&self.len.to_le_bytes())?; + Ok(()) + } + + pub(crate) fn decode(data: &mut impl std::io::Read) -> Result { + let kind = MessageKind::decode(data)?; + let mut buf = [0; 4]; + data.read_exact(&mut buf)?; + let len = u32::from_le_bytes(buf); + + Ok(Self { kind, len }) + } +} + +pub(crate) fn encode( + buf: &mut Vec, + message: &LogMsg, + compression: Compression, +) -> Result<(), EncodeError> { + use re_protos::external::prost::Message; + use re_protos::log_msg::v0::{ + self as proto, ArrowMsg, BlueprintActivationCommand, Encoding, SetStoreInfo, + }; + + match message { + LogMsg::SetStoreInfo(set_store_info) => { + let set_store_info: SetStoreInfo = set_store_info.clone().into(); + let header = MessageHeader { + kind: MessageKind::SetStoreInfo, + len: set_store_info.encoded_len() as u32, + }; + header.encode(buf)?; + set_store_info.encode(buf)?; + } + LogMsg::ArrowMsg(store_id, arrow_msg) => { + let arrow_msg = ArrowMsg { + store_id: Some(store_id.clone().into()), + compression: match compression { + Compression::Off => proto::Compression::None as i32, + Compression::LZ4 => proto::Compression::Lz4 as i32, + }, + encoding: Encoding::ArrowIpc as i32, + payload: encode_arrow(&arrow_msg.schema, &arrow_msg.chunk, compression)?, + }; + let header = MessageHeader { + kind: MessageKind::ArrowMsg, + len: arrow_msg.encoded_len() as u32, + }; + header.encode(buf)?; + arrow_msg.encode(buf)?; + } + LogMsg::BlueprintActivationCommand(blueprint_activation_command) => { + let blueprint_activation_command: BlueprintActivationCommand = + blueprint_activation_command.clone().into(); + let header = MessageHeader { + kind: MessageKind::BlueprintActivationCommand, + len: blueprint_activation_command.encoded_len() as u32, + }; + header.encode(buf)?; + blueprint_activation_command.encode(buf)?; + } + } + + Ok(()) +} + +pub(crate) fn decode( + data: &mut impl std::io::Read, + compression: Compression, +) -> Result<(u64, Option), DecodeError> { + use re_protos::external::prost::Message; + use re_protos::log_msg::v0::{ArrowMsg, BlueprintActivationCommand, Encoding, SetStoreInfo}; + + let mut read_bytes = 0u64; + let header = MessageHeader::decode(data)?; + read_bytes += std::mem::size_of::() as u64 + header.len as u64; + + let mut buf = vec![0; header.len as usize]; + data.read_exact(&mut buf[..])?; + + let msg = match header.kind { + MessageKind::SetStoreInfo => { + let set_store_info = SetStoreInfo::decode(&buf[..])?; + Some(LogMsg::SetStoreInfo(set_store_info.try_into()?)) + } + MessageKind::ArrowMsg => { + let arrow_msg = ArrowMsg::decode(&buf[..])?; + if arrow_msg.encoding() != Encoding::ArrowIpc { + return Err(DecodeError::Codec(CodecError::UnsupportedEncoding)); + } + + let (schema, chunk) = decode_arrow(&arrow_msg.payload, compression)?; + + let store_id: re_log_types::StoreId = arrow_msg + .store_id + .ok_or_else(|| { + TypeConversionError::missing_field("rerun.log_msg.v0.ArrowMsg", "store_id") + })? + .into(); + + let chunk = re_chunk::Chunk::from_transport(&re_chunk::TransportChunk { + schema, + data: chunk, + })?; + + Some(LogMsg::ArrowMsg(store_id, chunk.to_arrow_msg()?)) + } + MessageKind::BlueprintActivationCommand => { + let blueprint_activation_command = BlueprintActivationCommand::decode(&buf[..])?; + Some(LogMsg::BlueprintActivationCommand( + blueprint_activation_command.try_into()?, + )) + } + MessageKind::End => None, + }; + + Ok((read_bytes, msg)) +} + +fn encode_arrow( + schema: &arrow2::datatypes::Schema, + chunk: &arrow2::chunk::Chunk>, + compression: crate::Compression, +) -> Result, EncodeError> { + let mut uncompressed = Vec::new(); + write_arrow_to_bytes(&mut uncompressed, schema, chunk)?; + + match compression { + crate::Compression::Off => Ok(uncompressed), + crate::Compression::LZ4 => Ok(lz4_flex::block::compress(&uncompressed)), + } +} + +fn decode_arrow( + data: &[u8], + compression: crate::Compression, +) -> Result< + ( + arrow2::datatypes::Schema, + arrow2::chunk::Chunk>, + ), + DecodeError, +> { + let mut uncompressed = Vec::new(); + let data = match compression { + crate::Compression::Off => data, + crate::Compression::LZ4 => { + lz4_flex::block::decompress_into(data, &mut uncompressed)?; + uncompressed.as_slice() + } + }; + + Ok(read_arrow_from_bytes(&mut &data[..])?) +} + +/// Helper function that serializes given arrow schema and record batch into bytes +/// using Arrow IPC format. +fn write_arrow_to_bytes( + writer: &mut W, + schema: &arrow2::datatypes::Schema, + data: &arrow2::chunk::Chunk>, +) -> Result<(), CodecError> { + use arrow2::io::ipc; + + let options = ipc::write::WriteOptions { compression: None }; + let mut sw = ipc::write::StreamWriter::new(writer, options); + + sw.start(schema, None) + .map_err(CodecError::ArrowSerialization)?; + sw.write(data, None) + .map_err(CodecError::ArrowSerialization)?; + sw.finish().map_err(CodecError::ArrowSerialization)?; + + Ok(()) +} + +/// Helper function that deserializes raw bytes into arrow schema and record batch +/// using Arrow IPC format. +fn read_arrow_from_bytes( + reader: &mut R, +) -> Result< + ( + arrow2::datatypes::Schema, + arrow2::chunk::Chunk>, + ), + CodecError, +> { + use arrow2::io::ipc; + + let metadata = + ipc::read::read_stream_metadata(reader).map_err(CodecError::ArrowSerialization)?; + let mut stream = ipc::read::StreamReader::new(reader, metadata, None); + + let schema = stream.schema().clone(); + // there should be at least one record batch in the stream + let stream_state = stream + .next() + .ok_or(CodecError::MissingRecordBatch)? + .map_err(CodecError::ArrowSerialization)?; + + match stream_state { + ipc::read::StreamState::Waiting => Err(CodecError::UnexpectedStreamState), + ipc::read::StreamState::Some(chunk) => Ok((schema, chunk)), + } +} diff --git a/crates/store/re_log_types/src/lib.rs b/crates/store/re_log_types/src/lib.rs index 50d1fa212efb..36a9ef977ba3 100644 --- a/crates/store/re_log_types/src/lib.rs +++ b/crates/store/re_log_types/src/lib.rs @@ -450,14 +450,19 @@ impl std::str::FromStr for PythonVersion { pub enum PythonVersionParseError { #[error("missing major version")] MissingMajor, + #[error("missing minor version")] MissingMinor, + #[error("missing patch version")] MissingPatch, + #[error("invalid major version")] InvalidMajor, + #[error("invalid minor version")] InvalidMinor, + #[error("invalid patch version")] InvalidPatch, } diff --git a/crates/store/re_log_types/src/protobuf_conversions.rs b/crates/store/re_log_types/src/protobuf_conversions.rs new file mode 100644 index 000000000000..6eafa0b019f7 --- /dev/null +++ b/crates/store/re_log_types/src/protobuf_conversions.rs @@ -0,0 +1,530 @@ +use std::sync::Arc; + +use re_protos::TypeConversionError; + +impl From for re_protos::common::v0::EntityPath { + fn from(value: crate::EntityPath) -> Self { + Self { + path: value.to_string(), + } + } +} + +impl TryFrom for crate::EntityPath { + type Error = TypeConversionError; + + fn try_from(value: re_protos::common::v0::EntityPath) -> Result { + Self::parse_strict(&value.path).map_err(|err| TypeConversionError::InvalidField { + type_name: "rerun.common.v0.EntityPath", + field_name: "path", + reason: err.to_string(), + }) + } +} + +impl From for crate::Time { + fn from(value: re_protos::common::v0::Time) -> Self { + Self::from_ns_since_epoch(value.nanos_since_epoch) + } +} + +impl From for re_protos::common::v0::Time { + fn from(value: crate::Time) -> Self { + Self { + nanos_since_epoch: value.nanos_since_epoch(), + } + } +} + +impl From for re_protos::common::v0::TimeInt { + fn from(value: crate::TimeInt) -> Self { + Self { + time: value.as_i64(), + } + } +} + +impl From for crate::TimeInt { + fn from(value: re_protos::common::v0::TimeInt) -> Self { + Self::new_temporal(value.time) + } +} + +impl From for re_protos::common::v0::TimeRange { + fn from(value: crate::ResolvedTimeRange) -> Self { + Self { + start: value.min().as_i64(), + end: value.max().as_i64(), + } + } +} + +impl From for crate::ResolvedTimeRange { + fn from(value: re_protos::common::v0::TimeRange) -> Self { + Self::new( + crate::TimeInt::new_temporal(value.start), + crate::TimeInt::new_temporal(value.end), + ) + } +} + +impl From for re_protos::common::v0::IndexRange { + fn from(value: crate::ResolvedTimeRange) -> Self { + Self { + time_range: Some(value.into()), + } + } +} + +impl TryFrom for crate::ResolvedTimeRange { + type Error = TypeConversionError; + + fn try_from(value: re_protos::common::v0::IndexRange) -> Result { + value + .time_range + .ok_or(TypeConversionError::missing_field( + "rerun.common.v0.IndexRange", + "time_range", + )) + .map(|time_range| Self::new(time_range.start, time_range.end)) + } +} + +impl TryFrom for crate::Timeline { + type Error = TypeConversionError; + + fn try_from(value: re_protos::common::v0::IndexColumnSelector) -> Result { + let timeline_name = value + .timeline + .ok_or(TypeConversionError::missing_field( + "rerun.common.v0.IndexColumnSelector", + "timeline", + ))? + .name; + + // TODO(cmc): QueryExpression::filtered_index gotta be a selector + #[allow(clippy::match_same_arms)] + let timeline = match timeline_name.as_str() { + "log_time" => Self::new_temporal(timeline_name), + "log_tick" => Self::new_sequence(timeline_name), + "frame" => Self::new_sequence(timeline_name), + "frame_nr" => Self::new_sequence(timeline_name), + _ => Self::new_temporal(timeline_name), + }; + + Ok(timeline) + } +} + +impl From for crate::ApplicationId { + #[inline] + fn from(value: re_protos::common::v0::ApplicationId) -> Self { + Self(value.id) + } +} + +impl From for re_protos::common::v0::ApplicationId { + #[inline] + fn from(value: crate::ApplicationId) -> Self { + Self { id: value.0 } + } +} + +impl From for crate::StoreKind { + #[inline] + fn from(value: re_protos::common::v0::StoreKind) -> Self { + match value { + re_protos::common::v0::StoreKind::Recording => Self::Recording, + re_protos::common::v0::StoreKind::Blueprint => Self::Blueprint, + } + } +} + +impl From for re_protos::common::v0::StoreKind { + #[inline] + fn from(value: crate::StoreKind) -> Self { + match value { + crate::StoreKind::Recording => Self::Recording, + crate::StoreKind::Blueprint => Self::Blueprint, + } + } +} + +impl From for crate::StoreId { + #[inline] + fn from(value: re_protos::common::v0::StoreId) -> Self { + Self { + kind: crate::StoreKind::Recording, + id: Arc::new(value.id), + } + } +} + +impl From for re_protos::common::v0::StoreId { + #[inline] + fn from(value: crate::StoreId) -> Self { + let kind: re_protos::common::v0::StoreKind = value.kind.into(); + Self { + kind: kind as i32, + id: String::clone(&*value.id), + } + } +} + +impl From for crate::StoreId { + #[inline] + fn from(value: re_protos::common::v0::RecordingId) -> Self { + Self { + kind: crate::StoreKind::Recording, + id: Arc::new(value.id), + } + } +} + +impl From for re_protos::common::v0::RecordingId { + #[inline] + fn from(value: crate::StoreId) -> Self { + Self { + id: String::clone(&*value.id), + } + } +} + +impl From for re_protos::log_msg::v0::StoreSource { + #[inline] + fn from(value: crate::StoreSource) -> Self { + use re_protos::external::prost::Message as _; + + let (kind, payload) = match value { + crate::StoreSource::Unknown => ( + re_protos::log_msg::v0::StoreSourceKind::UnknownKind as i32, + Vec::new(), + ), + crate::StoreSource::CSdk => ( + re_protos::log_msg::v0::StoreSourceKind::CSdk as i32, + Vec::new(), + ), + crate::StoreSource::PythonSdk(python_version) => ( + re_protos::log_msg::v0::StoreSourceKind::PythonSdk as i32, + re_protos::log_msg::v0::PythonVersion::from(python_version).encode_to_vec(), + ), + crate::StoreSource::RustSdk { + rustc_version, + llvm_version, + } => ( + re_protos::log_msg::v0::StoreSourceKind::RustSdk as i32, + re_protos::log_msg::v0::CrateInfo { + rustc_version, + llvm_version, + } + .encode_to_vec(), + ), + crate::StoreSource::File { file_source } => ( + re_protos::log_msg::v0::StoreSourceKind::File as i32, + re_protos::log_msg::v0::FileSource::from(file_source).encode_to_vec(), + ), + crate::StoreSource::Viewer => ( + re_protos::log_msg::v0::StoreSourceKind::Viewer as i32, + Vec::new(), + ), + crate::StoreSource::Other(description) => ( + re_protos::log_msg::v0::StoreSourceKind::Other as i32, + description.into_bytes(), + ), + }; + + Self { + kind, + extra: Some(re_protos::log_msg::v0::StoreSourceExtra { payload }), + } + } +} + +impl TryFrom for crate::StoreSource { + type Error = TypeConversionError; + + #[inline] + fn try_from(value: re_protos::log_msg::v0::StoreSource) -> Result { + use re_protos::external::prost::Message as _; + use re_protos::log_msg::v0::StoreSourceKind; + + match value.kind() { + StoreSourceKind::UnknownKind => Ok(Self::Unknown), + StoreSourceKind::CSdk => Ok(Self::CSdk), + StoreSourceKind::PythonSdk => { + let extra = value.extra.ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.StoreSource", + "extra", + ))?; + let python_version = + re_protos::log_msg::v0::PythonVersion::decode(&mut &extra.payload[..])?; + Ok(Self::PythonSdk(crate::PythonVersion::try_from( + python_version, + )?)) + } + StoreSourceKind::RustSdk => { + let extra = value.extra.ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.StoreSource", + "extra", + ))?; + let crate_info = + re_protos::log_msg::v0::CrateInfo::decode(&mut &extra.payload[..])?; + Ok(Self::RustSdk { + rustc_version: crate_info.rustc_version, + llvm_version: crate_info.llvm_version, + }) + } + StoreSourceKind::File => { + let extra = value.extra.ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.StoreSource", + "extra", + ))?; + let file_source = + re_protos::log_msg::v0::FileSource::decode(&mut &extra.payload[..])?; + Ok(Self::File { + file_source: crate::FileSource::try_from(file_source)?, + }) + } + StoreSourceKind::Viewer => Ok(Self::Viewer), + StoreSourceKind::Other => { + let description = value.extra.ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.StoreSource", + "extra", + ))?; + let description = String::from_utf8(description.payload).map_err(|err| { + TypeConversionError::InvalidField { + type_name: "rerun.log_msg.v0.StoreSource", + field_name: "extra", + reason: err.to_string(), + } + })?; + Ok(Self::Other(description)) + } + } + } +} + +impl From for re_protos::log_msg::v0::PythonVersion { + #[inline] + fn from(value: crate::PythonVersion) -> Self { + let mut version = Vec::new(); + version.push(value.major); + version.push(value.minor); + version.push(value.patch); + version.extend_from_slice(value.suffix.as_bytes()); + + Self { version } + } +} + +impl TryFrom for crate::PythonVersion { + type Error = TypeConversionError; + + #[inline] + fn try_from(value: re_protos::log_msg::v0::PythonVersion) -> Result { + if value.version.len() < 3 { + return Err(TypeConversionError::InvalidField { + type_name: "rerun.log_msg.v0.PythonVersion", + field_name: "version", + reason: "expected at least 3 bytes".to_owned(), + }); + } + + let major = value.version[0]; + let minor = value.version[1]; + let patch = value.version[2]; + let suffix = std::str::from_utf8(&value.version[3..]) + .map_err(|err| TypeConversionError::InvalidField { + type_name: "rerun.log_msg.v0.PythonVersion", + field_name: "version", + reason: err.to_string(), + })? + .to_owned(); + + Ok(Self { + major, + minor, + patch, + suffix, + }) + } +} + +impl From for re_protos::log_msg::v0::FileSource { + #[inline] + fn from(value: crate::FileSource) -> Self { + let kind = match value { + crate::FileSource::Cli => re_protos::log_msg::v0::FileSourceKind::Cli as i32, + crate::FileSource::Uri => re_protos::log_msg::v0::FileSourceKind::Uri as i32, + crate::FileSource::DragAndDrop { .. } => { + re_protos::log_msg::v0::FileSourceKind::DragAndDrop as i32 + } + crate::FileSource::FileDialog { .. } => { + re_protos::log_msg::v0::FileSourceKind::FileDialog as i32 + } + crate::FileSource::Sdk => re_protos::log_msg::v0::FileSourceKind::Sdk as i32, + }; + + Self { kind } + } +} + +impl TryFrom for crate::FileSource { + type Error = TypeConversionError; + + #[inline] + fn try_from(value: re_protos::log_msg::v0::FileSource) -> Result { + use re_protos::log_msg::v0::FileSourceKind; + + match value.kind() { + FileSourceKind::Cli => Ok(Self::Cli), + FileSourceKind::Uri => Ok(Self::Uri), + FileSourceKind::DragAndDrop => Ok(Self::DragAndDrop { + recommended_application_id: None, + recommended_recording_id: None, + force_store_info: false, + }), + FileSourceKind::FileDialog => Ok(Self::FileDialog { + recommended_application_id: None, + recommended_recording_id: None, + force_store_info: false, + }), + FileSourceKind::Sdk => Ok(Self::Sdk), + FileSourceKind::UnknownSource => Err(TypeConversionError::InvalidField { + type_name: "rerun.log_msg.v0.FileSource", + field_name: "kind", + reason: "unknown kind".to_owned(), + }), + } + } +} + +impl From for re_protos::log_msg::v0::StoreInfo { + #[inline] + fn from(value: crate::StoreInfo) -> Self { + Self { + application_id: Some(value.application_id.into()), + store_id: Some(value.store_id.into()), + is_official_example: value.is_official_example, + started: Some(value.started.into()), + store_source: Some(value.store_source.into()), + } + } +} + +impl TryFrom for crate::StoreInfo { + type Error = TypeConversionError; + + #[inline] + fn try_from(value: re_protos::log_msg::v0::StoreInfo) -> Result { + let application_id: crate::ApplicationId = value + .application_id + .ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.StoreInfo", + "application_id", + ))? + .into(); + let store_id: crate::StoreId = value + .store_id + .ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.StoreInfo", + "store_id", + ))? + .into(); + let is_official_example = value.is_official_example; + let started: crate::Time = value + .started + .ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.StoreInfo", + "started", + ))? + .into(); + let store_source: crate::StoreSource = value + .store_source + .ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.StoreInfo", + "store_source", + ))? + .try_into()?; + + Ok(Self { + application_id, + store_id, + cloned_from: None, + is_official_example, + started, + store_source, + store_version: Some(re_build_info::CrateVersion::LOCAL), + }) + } +} + +impl From for re_protos::log_msg::v0::SetStoreInfo { + #[inline] + fn from(value: crate::SetStoreInfo) -> Self { + Self { + row_id: Some(value.row_id.into()), + info: Some(value.info.into()), + } + } +} + +impl TryFrom for crate::SetStoreInfo { + type Error = TypeConversionError; + + #[inline] + fn try_from(value: re_protos::log_msg::v0::SetStoreInfo) -> Result { + Ok(Self { + row_id: value + .row_id + .ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.SetStoreInfo", + "row_id", + ))? + .into(), + info: value + .info + .ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.SetStoreInfo", + "info", + ))? + .try_into()?, + }) + } +} + +impl From + for re_protos::log_msg::v0::BlueprintActivationCommand +{ + #[inline] + fn from(value: crate::BlueprintActivationCommand) -> Self { + Self { + blueprint_id: Some(value.blueprint_id.into()), + make_active: value.make_active, + make_default: value.make_default, + } + } +} + +impl TryFrom + for crate::BlueprintActivationCommand +{ + type Error = TypeConversionError; + + #[inline] + fn try_from( + value: re_protos::log_msg::v0::BlueprintActivationCommand, + ) -> Result { + Ok(Self { + blueprint_id: value + .blueprint_id + .ok_or(TypeConversionError::missing_field( + "rerun.log_msg.v0.BlueprintActivationCommand", + "blueprint_id", + ))? + .into(), + make_active: value.make_active, + make_default: value.make_default, + }) + } +} diff --git a/crates/store/re_protos/proto/rerun/v0/common.proto b/crates/store/re_protos/proto/rerun/v0/common.proto index bf4c8d43a33f..9370873d4989 100644 --- a/crates/store/re_protos/proto/rerun/v0/common.proto +++ b/crates/store/re_protos/proto/rerun/v0/common.proto @@ -196,3 +196,12 @@ message Time { enum EncoderVersion { V0 = 0; } + +message Tuid { + // Approximate nanoseconds since epoch. + fixed64 time_ns = 1; + + // Initialized to something random on each thread, + // then incremented for each new `Tuid` being allocated. + fixed64 inc = 2; +} diff --git a/crates/store/re_protos/proto/rerun/v0/log_msg.proto b/crates/store/re_protos/proto/rerun/v0/log_msg.proto new file mode 100644 index 000000000000..b3c42f0d6580 --- /dev/null +++ b/crates/store/re_protos/proto/rerun/v0/log_msg.proto @@ -0,0 +1,187 @@ +syntax = "proto3"; + +package rerun.log_msg.v0; + +import "rerun/v0/common.proto"; + +/* +// not actually protobuf +message MessageHeader { + enum MessageKind { + UNKNOWN_KIND = 0; + ARROW_MSG = 1; + SET_STORE_INFO = 2; + BLUEPRINT_ACTIVATION_COMMAND = 3; + } + + MessageKind kind = 1; + + // len-prefixed + bytes payload = 1000; +} +*/ + +message SetStoreInfo { + rerun.common.v0.Tuid row_id = 1; + + StoreInfo info = 2; +} + +enum Compression { + NONE = 0; + LZ4 = 1; +} + +enum Encoding { + UNKNOWN = 0; + ARROW_IPC = 1; +} + +message ArrowMsg { + rerun.common.v0.StoreId store_id = 1; + Compression compression = 2; + Encoding encoding = 3; + + // Arrow-IPC encoded schema and chunk, + // compressed according to `compression` + bytes payload = 1000; +} + +message BlueprintActivationCommand { + rerun.common.v0.StoreId blueprint_id = 1; + bool make_active = 2; + bool make_default = 3; +} + +message StoreInfo { + // User-chosen name of the application doing the logging. + rerun.common.v0.ApplicationId application_id = 1; + + // Unique ID of the recording. + rerun.common.v0.StoreId store_id = 2; + + /// True if the recording is one of the official Rerun examples. + bool is_official_example = 3; + + // When the recording started. + rerun.common.v0.Time started = 4; + + StoreSource store_source = 5; +} + +message StoreSource { + // Determines what is encoded in `extra`. + StoreSourceKind kind = 1; + StoreSourceExtra extra = 2; +} + +enum StoreSourceKind { + // `extra` is unused. + UNKNOWN_KIND = 0; + // `extra` is unused. + C_SDK = 1; + // `extra` is `PythonVersion`. + PYTHON_SDK = 2; + // `extra` is `CrateInfo`. + RUST_SDK = 3; + // `extra` is `FileSource`. + FILE = 4; + // `extra` is unused. + VIEWER = 5; + // `extra` is a string. + OTHER = 6; +} + +message StoreSourceExtra { + bytes payload = 1; +} + +message PythonVersion { + // [u8; major, minor, patch, ..suffix] + bytes version = 1; +} + +message CrateInfo { + string rustc_version = 1; + string llvm_version = 2; +} + +message FileSource { + FileSourceKind kind = 1; +} + +enum FileSourceKind { + UNKNOWN_SOURCE = 0; + CLI = 1; + URI = 2; + DRAG_AND_DROP = 3; + FILE_DIALOG = 4; + SDK = 5; +} + +/* +pub struct StoreInfo { + /// The user-chosen name of the application doing the logging. + pub application_id: ApplicationId, + + /// Should be unique for each recording. + pub store_id: StoreId, + + /// If this store is the result of a clone, which store was it cloned from? + /// + /// A cloned store always gets a new unique ID. + /// + /// We currently only clone stores for blueprints: + /// when we receive a _default_ blueprints on the wire (e.g. from a recording), + /// we clone it and make the clone the _active_ blueprint. + /// This means all active blueprints are clones. + pub cloned_from: Option, + + /// True if the recording is one of the official Rerun examples. + pub is_official_example: bool, + + /// When the recording started. + /// + /// Should be an absolute time, i.e. relative to Unix Epoch. + pub started: Time, + + pub store_source: StoreSource, + + /// The Rerun version used to encoded the RRD data. + /// + // NOTE: The version comes directly from the decoded RRD stream's header, duplicating it here + // would probably only lead to more issues down the line. + #[cfg_attr(feature = "serde", serde(skip))] + pub store_version: Option, +} + +pub enum StoreSource { + Unknown, + + /// The official Rerun C Logging SDK + CSdk, + + /// The official Rerun Python Logging SDK + PythonSdk(PythonVersion), + + /// The official Rerun Rust Logging SDK + RustSdk { + /// Rust version of the code compiling the Rust SDK + rustc_version: String, + + /// LLVM version of the code compiling the Rust SDK + llvm_version: String, + }, + + /// Loading a file via CLI, drag-and-drop, a file-dialog, etc. + File { + file_source: FileSource, + }, + + /// Generated from the viewer itself. + Viewer, + + /// Perhaps from some manual data ingestion? + Other(String), +} +*/ diff --git a/crates/store/re_protos/src/lib.rs b/crates/store/re_protos/src/lib.rs index b0bbe04cefb2..841f2a95a0c5 100644 --- a/crates/store/re_protos/src/lib.rs +++ b/crates/store/re_protos/src/lib.rs @@ -62,6 +62,13 @@ pub enum TypeConversionError { field_name: &'static str, }, + #[error("invalid value for field {type_name}.{field_name}: {reason}")] + InvalidField { + type_name: &'static str, + field_name: &'static str, + reason: String, + }, + #[error("failed to decode: {0}")] DecodeError(#[from] prost::DecodeError), diff --git a/crates/store/re_protos/src/v0/rerun.common.v0.rs b/crates/store/re_protos/src/v0/rerun.common.v0.rs new file mode 100644 index 000000000000..171e0633a9a8 --- /dev/null +++ b/crates/store/re_protos/src/v0/rerun.common.v0.rs @@ -0,0 +1,321 @@ +// This file is @generated by prost-build. +/// unique recording identifier. At this point in time it is the same id as the ChunkStore's StoreId +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RecordingId { + #[prost(string, tag = "1")] + pub id: ::prost::alloc::string::String, +} +/// A recording can have multiple timelines, each is identified by a name, for example `log_tick`, `log_time`, etc. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Timeline { + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, +} +/// A time range between start and end time points. Each 64 bit number can represent different time point data +/// depending on the timeline it is associated with. Time range is inclusive for both start and end time points. +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct TimeRange { + #[prost(int64, tag = "1")] + pub start: i64, + #[prost(int64, tag = "2")] + pub end: i64, +} +/// arrow IPC serialized schema +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Schema { + #[prost(bytes = "vec", tag = "1")] + pub arrow_schema: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Query { + /// The subset of the database that the query will run on: a set of EntityPath(s) and their + /// associated Component(s) + #[prost(message, optional, tag = "1")] + pub view_contents: ::core::option::Option, + /// Whether the view_contents should ignore semantically empty columns + /// A semantically empty column is a column that either contains no data at all, or where all + /// values are either nulls or empty arrays (\[\]). + #[prost(bool, tag = "2")] + pub include_semantically_empty_columns: bool, + /// Whether the view_contents should ignore columns corresponding to indicator components + /// Indicator components are marker components, generally automatically inserted by Rerun, that + /// helps keep track of the original context in which a piece of data was logged/sent. + #[prost(bool, tag = "3")] + pub include_indicator_columns: bool, + /// Whether the view_contents should ignore columns corresponding to Clear-related components + #[prost(bool, tag = "4")] + pub include_tombstone_columns: bool, + /// The index used to filter out _rows_ from the view contents. + /// Only rows where at least 1 column contains non-null data at that index will be kept in the + /// final dataset. If left unspecified, the results will only contain static data. + #[prost(message, optional, tag = "5")] + pub filtered_index: ::core::option::Option, + /// The range of index values used to filter out _rows_ from the view contents + /// Only rows where at least 1 of the view-contents contains non-null data within that range will be kept in + /// the final dataset. + /// This has no effect if filtered_index isn't set. + /// This has no effect if using_index_values is set. + #[prost(message, optional, tag = "6")] + pub filtered_index_range: ::core::option::Option, + /// The specific index values used to filter out _rows_ from the view contents. + /// Only rows where at least 1 column contains non-null data at these specific values will be kept + /// in the final dataset. + /// This has no effect if filtered_index isn't set. + /// This has no effect if using_index_values is set. + #[prost(message, optional, tag = "7")] + pub filtered_index_values: ::core::option::Option, + /// The specific index values used to sample _rows_ from the view contents. + /// The final dataset will contain one row per sampled index value, regardless of whether data + /// existed for that index value in the view contents. + /// The semantics of the query are consistent with all other settings: the results will be + /// sorted on the filtered_index, and only contain unique index values. + /// + /// This has no effect if filtered_index isn't set. + /// If set, this overrides both filtered_index_range and filtered_index_values. + #[prost(message, optional, tag = "8")] + pub using_index_values: ::core::option::Option, + /// The component column used to filter out _rows_ from the view contents. + /// Only rows where this column contains non-null data be kept in the final dataset. + #[prost(message, optional, tag = "9")] + pub filtered_is_not_null: ::core::option::Option, + /// / The specific _columns_ to sample from the final view contents. + /// / The order of the samples will be respected in the final result. + /// / + /// / If unspecified, it means - everything. + #[prost(message, optional, tag = "10")] + pub column_selection: ::core::option::Option, + /// Specifies how null values should be filled in the returned dataframe. + #[prost(enumeration = "SparseFillStrategy", tag = "11")] + pub sparse_fill_strategy: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ColumnSelection { + #[prost(message, repeated, tag = "1")] + pub columns: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ColumnSelector { + #[prost(oneof = "column_selector::SelectorType", tags = "2, 3")] + pub selector_type: ::core::option::Option, +} +/// Nested message and enum types in `ColumnSelector`. +pub mod column_selector { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum SelectorType { + #[prost(message, tag = "2")] + ComponentColumn(super::ComponentColumnSelector), + #[prost(message, tag = "3")] + TimeColumn(super::TimeColumnSelector), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IndexColumnSelector { + /// TODO(zehiko) we need to add support for other types of index selectors + #[prost(message, optional, tag = "1")] + pub timeline: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct IndexRange { + /// TODO(zehiko) support for other ranges for other index selectors + #[prost(message, optional, tag = "1")] + pub time_range: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IndexValues { + /// TODO(zehiko) we need to add support for other types of index selectors + #[prost(message, repeated, tag = "1")] + pub time_points: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SampledIndexValues { + #[prost(message, repeated, tag = "1")] + pub sample_points: ::prost::alloc::vec::Vec, +} +/// A 64-bit number describing either nanoseconds, sequence numbers or fully static data. +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct TimeInt { + #[prost(int64, tag = "1")] + pub time: i64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ViewContents { + #[prost(message, repeated, tag = "1")] + pub contents: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ViewContentsPart { + #[prost(message, optional, tag = "1")] + pub path: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub components: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ComponentsSet { + #[prost(message, repeated, tag = "1")] + pub components: ::prost::alloc::vec::Vec, +} +/// The unique identifier of an entity, e.g. `camera/3/points` +/// See <> for more on entity paths. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EntityPath { + #[prost(string, tag = "1")] + pub path: ::prost::alloc::string::String, +} +/// Component describes semantic data that can be used by any number of rerun's archetypes. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Component { + /// component name needs to be a string as user can define their own component + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, +} +/// Used to telect a time column. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TimeColumnSelector { + #[prost(message, optional, tag = "1")] + pub timeline: ::core::option::Option, +} +/// Used to select a component based on its EntityPath and Component name. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ComponentColumnSelector { + #[prost(message, optional, tag = "1")] + pub entity_path: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub component: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ApplicationId { + #[prost(string, tag = "1")] + pub id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreId { + #[prost(enumeration = "StoreKind", tag = "1")] + pub kind: i32, + #[prost(string, tag = "2")] + pub id: ::prost::alloc::string::String, +} +/// A date-time represented as nanoseconds since unix epoch +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct Time { + #[prost(int64, tag = "1")] + pub nanos_since_epoch: i64, +} +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct Tuid { + /// Approximate nanoseconds since epoch. + #[prost(fixed64, tag = "1")] + pub time_ns: u64, + /// Initialized to something random on each thread, + /// then incremented for each new `Tuid` being allocated. + #[prost(fixed64, tag = "2")] + pub inc: u64, +} +/// Specifies how null values should be filled in the returned dataframe. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum SparseFillStrategy { + None = 0, + LatestAtGlobal = 1, +} +impl SparseFillStrategy { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::None => "NONE", + Self::LatestAtGlobal => "LATEST_AT_GLOBAL", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "NONE" => Some(Self::None), + "LATEST_AT_GLOBAL" => Some(Self::LatestAtGlobal), + _ => None, + } + } +} +/// Error codes for application level errors +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ErrorCode { + /// unused + Unused = 0, + /// object store access error + ObjectStoreError = 1, + /// metadata database access error + MetadataDbError = 2, +} +impl ErrorCode { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unused => "_UNUSED", + Self::ObjectStoreError => "OBJECT_STORE_ERROR", + Self::MetadataDbError => "METADATA_DB_ERROR", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "_UNUSED" => Some(Self::Unused), + "OBJECT_STORE_ERROR" => Some(Self::ObjectStoreError), + "METADATA_DB_ERROR" => Some(Self::MetadataDbError), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum StoreKind { + Recording = 0, + Blueprint = 1, +} +impl StoreKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Recording => "RECORDING", + Self::Blueprint => "BLUEPRINT", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "RECORDING" => Some(Self::Recording), + "BLUEPRINT" => Some(Self::Blueprint), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum EncoderVersion { + V0 = 0, +} +impl EncoderVersion { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::V0 => "V0", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "V0" => Some(Self::V0), + _ => None, + } + } +} diff --git a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs new file mode 100644 index 000000000000..096b2926f720 --- /dev/null +++ b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs @@ -0,0 +1,216 @@ +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetStoreInfo { + #[prost(message, optional, tag = "1")] + pub row_id: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub info: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ArrowMsg { + #[prost(message, optional, tag = "1")] + pub store_id: ::core::option::Option, + #[prost(enumeration = "Compression", tag = "2")] + pub compression: i32, + #[prost(enumeration = "Encoding", tag = "3")] + pub encoding: i32, + /// Arrow-IPC encoded schema and chunk, + /// compressed according to `compression` + #[prost(bytes = "vec", tag = "1000")] + pub payload: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlueprintActivationCommand { + #[prost(message, optional, tag = "1")] + pub blueprint_id: ::core::option::Option, + #[prost(bool, tag = "2")] + pub make_active: bool, + #[prost(bool, tag = "3")] + pub make_default: bool, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreInfo { + /// User-chosen name of the application doing the logging. + #[prost(message, optional, tag = "1")] + pub application_id: ::core::option::Option, + /// Unique ID of the recording. + #[prost(message, optional, tag = "2")] + pub store_id: ::core::option::Option, + /// / True if the recording is one of the official Rerun examples. + #[prost(bool, tag = "3")] + pub is_official_example: bool, + /// When the recording started. + #[prost(message, optional, tag = "4")] + pub started: ::core::option::Option, + #[prost(message, optional, tag = "5")] + pub store_source: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreSource { + /// Determines what is encoded in `extra`. + #[prost(enumeration = "StoreSourceKind", tag = "1")] + pub kind: i32, + #[prost(message, optional, tag = "2")] + pub extra: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreSourceExtra { + #[prost(bytes = "vec", tag = "1")] + pub payload: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PythonVersion { + /// \[u8; major, minor, patch, ..suffix\] + #[prost(bytes = "vec", tag = "1")] + pub version: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CrateInfo { + #[prost(string, tag = "1")] + pub rustc_version: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub llvm_version: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct FileSource { + #[prost(enumeration = "FileSourceKind", tag = "1")] + pub kind: i32, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum Compression { + None = 0, + Lz4 = 1, +} +impl Compression { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::None => "NONE", + Self::Lz4 => "LZ4", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "NONE" => Some(Self::None), + "LZ4" => Some(Self::Lz4), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum Encoding { + Unknown = 0, + ArrowIpc = 1, +} +impl Encoding { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unknown => "UNKNOWN", + Self::ArrowIpc => "ARROW_IPC", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UNKNOWN" => Some(Self::Unknown), + "ARROW_IPC" => Some(Self::ArrowIpc), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum StoreSourceKind { + /// `extra` is unused. + UnknownKind = 0, + /// `extra` is unused. + CSdk = 1, + /// `extra` is `PythonVersion`. + PythonSdk = 2, + /// `extra` is `CrateInfo`. + RustSdk = 3, + /// `extra` is `FileSource`. + File = 4, + /// `extra` is unused. + Viewer = 5, + /// `extra` is a string. + Other = 6, +} +impl StoreSourceKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::UnknownKind => "UNKNOWN_KIND", + Self::CSdk => "C_SDK", + Self::PythonSdk => "PYTHON_SDK", + Self::RustSdk => "RUST_SDK", + Self::File => "FILE", + Self::Viewer => "VIEWER", + Self::Other => "OTHER", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UNKNOWN_KIND" => Some(Self::UnknownKind), + "C_SDK" => Some(Self::CSdk), + "PYTHON_SDK" => Some(Self::PythonSdk), + "RUST_SDK" => Some(Self::RustSdk), + "FILE" => Some(Self::File), + "VIEWER" => Some(Self::Viewer), + "OTHER" => Some(Self::Other), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum FileSourceKind { + UnknownSource = 0, + Cli = 1, + Uri = 2, + DragAndDrop = 3, + FileDialog = 4, + Sdk = 5, +} +impl FileSourceKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::UnknownSource => "UNKNOWN_SOURCE", + Self::Cli => "CLI", + Self::Uri => "URI", + Self::DragAndDrop => "DRAG_AND_DROP", + Self::FileDialog => "FILE_DIALOG", + Self::Sdk => "SDK", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UNKNOWN_SOURCE" => Some(Self::UnknownSource), + "CLI" => Some(Self::Cli), + "URI" => Some(Self::Uri), + "DRAG_AND_DROP" => Some(Self::DragAndDrop), + "FILE_DIALOG" => Some(Self::FileDialog), + "SDK" => Some(Self::Sdk), + _ => None, + } + } +} diff --git a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs index 0666d35d2cd0..9cb8f8dde1c1 100644 --- a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs @@ -131,6 +131,19 @@ pub struct FetchRecordingResponse { #[prost(bytes = "vec", tag = "2")] pub payload: ::prost::alloc::vec::Vec, } +/// Application level error - use as `details` in the `google.rpc.Status` message +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RemoteStoreError { + /// error code + #[prost(enumeration = "ErrorCode", tag = "1")] + pub code: i32, + /// unique identifier associated with the request (e.g. recording id, recording storage url) + #[prost(string, tag = "2")] + pub id: ::prost::alloc::string::String, + /// human readable details about the error + #[prost(string, tag = "3")] + pub message: ::prost::alloc::string::String, +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum RecordingType { @@ -154,6 +167,43 @@ impl RecordingType { } } } +/// Error codes for application level errors +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ErrorCode { + /// unused + Unused = 0, + /// object store access error + ObjectStoreError = 1, + /// metadata database access error + MetadataDbError = 2, + /// Encoding / decoding error + CodecError = 3, +} +impl ErrorCode { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unused => "_UNUSED", + Self::ObjectStoreError => "OBJECT_STORE_ERROR", + Self::MetadataDbError => "METADATA_DB_ERROR", + Self::CodecError => "CODEC_ERROR", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "_UNUSED" => Some(Self::Unused), + "OBJECT_STORE_ERROR" => Some(Self::ObjectStoreError), + "METADATA_DB_ERROR" => Some(Self::MetadataDbError), + "CODEC_ERROR" => Some(Self::CodecError), + _ => None, + } + } +} /// Generated client implementations. pub mod storage_node_client { #![allow( diff --git a/crates/utils/re_tuid/Cargo.toml b/crates/utils/re_tuid/Cargo.toml index 248f40c7b96f..3591c5e5c8ac 100644 --- a/crates/utils/re_tuid/Cargo.toml +++ b/crates/utils/re_tuid/Cargo.toml @@ -27,6 +27,8 @@ serde = ["dep:serde"] [dependencies] +re_protos.workspace = true + document-features.workspace = true getrandom.workspace = true once_cell.workspace = true diff --git a/crates/utils/re_tuid/src/lib.rs b/crates/utils/re_tuid/src/lib.rs index e50ed36c66ef..b95d8bd9fec9 100644 --- a/crates/utils/re_tuid/src/lib.rs +++ b/crates/utils/re_tuid/src/lib.rs @@ -6,6 +6,8 @@ #![doc = document_features::document_features!()] //! +mod protobuf_conversions; + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct Tuid { diff --git a/crates/utils/re_tuid/src/protobuf_conversions.rs b/crates/utils/re_tuid/src/protobuf_conversions.rs new file mode 100644 index 000000000000..2dfd04627d4b --- /dev/null +++ b/crates/utils/re_tuid/src/protobuf_conversions.rs @@ -0,0 +1,17 @@ +impl From for crate::Tuid { + fn from(value: re_protos::common::v0::Tuid) -> Self { + Self { + time_ns: value.time_ns, + inc: value.inc, + } + } +} + +impl From for re_protos::common::v0::Tuid { + fn from(value: crate::Tuid) -> Self { + Self { + time_ns: value.time_ns, + inc: value.inc, + } + } +} diff --git a/rerun_py/src/remote.rs b/rerun_py/src/remote.rs index eeaf24c652a7..89e1949127a0 100644 --- a/rerun_py/src/remote.rs +++ b/rerun_py/src/remote.rs @@ -9,13 +9,14 @@ use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyDict, Bound, PyResul use re_chunk::{Chunk, TransportChunk}; use re_chunk_store::ChunkStore; use re_dataframe::ChunkStoreHandle; +use re_log_encoding::codec; +use re_log_encoding::codec::wire::chunk_to_recording_metadata; use re_log_types::{StoreInfo, StoreSource}; use re_protos::{ - codec::decode, - v0::{ - storage_node_client::StorageNodeClient, EncoderVersion, FetchRecordingRequest, - ListRecordingsRequest, RecordingId, RecordingMetadata, RecordingType, - RegisterRecordingRequest, UpdateRecordingMetadataRequest, + common::v0::{EncoderVersion, RecordingId}, + remote_store::v0::{ + storage_node_client::StorageNodeClient, FetchRecordingRequest, ListRecordingsRequest, + RecordingType, RegisterRecordingRequest, UpdateRecordingMetadataRequest, }, }; use re_sdk::{ApplicationId, StoreId, StoreKind, Time}; @@ -98,7 +99,7 @@ impl PyStorageNodeClient { .into_inner() .recordings .into_iter() - .map(|recording| recording.data()) + .map(|recording| codec::wire::recording_metadata_to_chunk(&recording)) .collect::, _>>() .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; @@ -178,7 +179,7 @@ impl PyStorageNodeClient { data, }; - RecordingMetadata::try_from(EncoderVersion::V0, &metadata_tc) + chunk_to_recording_metadata(EncoderVersion::V0, &metadata_tc) .map_err(|err| PyRuntimeError::new_err(err.to_string())) }) .transpose()?; @@ -244,7 +245,7 @@ impl PyStorageNodeClient { data, }; - let metadata = RecordingMetadata::try_from(EncoderVersion::V0, &metadata_tc) + let metadata = chunk_to_recording_metadata(EncoderVersion::V0, &metadata_tc) .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; let request = UpdateRecordingMetadataRequest { @@ -307,7 +308,7 @@ impl PyStorageNodeClient { while let Some(result) = resp.next().await { let response = result.map_err(|err| PyRuntimeError::new_err(err.to_string()))?; - let tc = decode(EncoderVersion::V0, &response.payload) + let tc = codec::wire::decode(EncoderVersion::V0, &response.payload) .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; let Some(tc) = tc else { diff --git a/scripts/lint.py b/scripts/lint.py index f60358192e93..6f703f647143 100755 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -1223,7 +1223,7 @@ def main() -> None: "./.pytest_cache", "./CODE_STYLE.md", "./crates/build/re_types_builder/src/reflection.rs", # auto-generated - "./crates/store/re_protos/src/v0/rerun.remote_store.v0.rs", # auto-generated + "./crates/store/re_protos/src/v0", # auto-generated "./docs/content/concepts/app-model.md", # this really needs custom letter casing "./docs/content/reference/cli.md", # auto-generated "./docs/snippets/all/tutorials/custom-application-id.cpp", # nuh-uh, I don't want rerun_example_ here From 8209db4a7ce6da5414871eb2ec8a590530f13488 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 18:53:43 +0100 Subject: [PATCH 04/47] fix after merge --- crates/store/re_chunk_store/src/dataframe.rs | 2 +- crates/store/re_grpc_client/src/lib.rs | 6 +- .../store/re_log_encoding/src/codec/wire.rs | 27 +- .../re_protos/proto/rerun/v0/common.proto | 4 - .../proto/rerun/v0/remote_store.proto | 14 +- crates/store/re_protos/src/lib.rs | 2 +- .../store/re_protos/src/v0/rerun.common.v0.rs | 23 - .../re_protos/src/v0/rerun.remote_store.v0.rs | 485 +++++++----------- rerun_py/src/remote.rs | 12 +- 9 files changed, 207 insertions(+), 368 deletions(-) diff --git a/crates/store/re_chunk_store/src/dataframe.rs b/crates/store/re_chunk_store/src/dataframe.rs index ac67eeb776e8..e17b58865bd3 100644 --- a/crates/store/re_chunk_store/src/dataframe.rs +++ b/crates/store/re_chunk_store/src/dataframe.rs @@ -12,7 +12,7 @@ use itertools::Itertools; use re_chunk::TimelineName; use re_log_types::{ComponentPath, EntityPath, ResolvedTimeRange, TimeInt, Timeline}; -use re_types_core::{ArchetypeName, ComponentName, ComponentNameSet}; +use re_types_core::{ArchetypeName, ComponentName}; use crate::{ChunkStore, ColumnMetadata}; diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs index 12a4ca5da1b3..24af0b83ed26 100644 --- a/crates/store/re_grpc_client/src/lib.rs +++ b/crates/store/re_grpc_client/src/lib.rs @@ -10,14 +10,14 @@ use url::Url; use std::error::Error; use re_chunk::Chunk; +use re_log_encoding::codec::{wire::decode, CodecError}; use re_log_types::{ ApplicationId, LogMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; use re_protos::{ - codec::{decode, CodecError}, - v0::{ + common::v0::RecordingId, + remote_store::v0::{ storage_node_client::StorageNodeClient, FetchRecordingRequest, QueryCatalogRequest, - RecordingId, }, }; diff --git a/crates/store/re_log_encoding/src/codec/wire.rs b/crates/store/re_log_encoding/src/codec/wire.rs index f9b3815af529..a2eb0fee7513 100644 --- a/crates/store/re_log_encoding/src/codec/wire.rs +++ b/crates/store/re_log_encoding/src/codec/wire.rs @@ -87,11 +87,11 @@ impl TransportMessageV0 { // to become stateful and keep track if schema was sent / received. /// Encode a transport chunk into a byte stream. pub fn encode( - version: re_protos::common::v0::EncoderVersion, + version: re_protos::remote_store::v0::EncoderVersion, chunk: TransportChunk, ) -> Result, CodecError> { match version { - re_protos::common::v0::EncoderVersion::V0 => { + re_protos::remote_store::v0::EncoderVersion::V0 => { TransportMessageV0::RecordBatch(chunk).to_bytes() } } @@ -100,7 +100,7 @@ pub fn encode( /// Create `RecordingMetadata` from `TransportChunk`. We rely on `TransportChunk` until /// we migrate from arrow2 to arrow. pub fn chunk_to_recording_metadata( - version: re_protos::common::v0::EncoderVersion, + version: re_protos::remote_store::v0::EncoderVersion, metadata: &TransportChunk, ) -> Result { if metadata.data.len() != 1 { @@ -111,7 +111,7 @@ pub fn chunk_to_recording_metadata( }; match version { - re_protos::common::v0::EncoderVersion::V0 => { + re_protos::remote_store::v0::EncoderVersion::V0 => { let mut data: Vec = Vec::new(); write_arrow_to_bytes(&mut data, &metadata.schema, &metadata.data)?; @@ -129,11 +129,12 @@ pub fn recording_metadata_to_chunk( ) -> Result { let mut reader = std::io::Cursor::new(metadata.payload.clone()); - let encoder_version = re_protos::common::v0::EncoderVersion::try_from(metadata.encoder_version) - .map_err(|err| CodecError::InvalidArgument(err.to_string()))?; + let encoder_version = + re_protos::remote_store::v0::EncoderVersion::try_from(metadata.encoder_version) + .map_err(|err| CodecError::InvalidArgument(err.to_string()))?; match encoder_version { - re_protos::common::v0::EncoderVersion::V0 => { + re_protos::remote_store::v0::EncoderVersion::V0 => { let (schema, data) = read_arrow_from_bytes(&mut reader)?; Ok(TransportChunk { schema, data }) } @@ -174,19 +175,21 @@ pub fn recording_id( /// Encode a `NoData` message into a byte stream. This can be used by the remote store /// (i.e. data producer) to signal back to the client that there's no data available. -pub fn no_data(version: re_protos::common::v0::EncoderVersion) -> Result, CodecError> { +pub fn no_data( + version: re_protos::remote_store::v0::EncoderVersion, +) -> Result, CodecError> { match version { - re_protos::common::v0::EncoderVersion::V0 => TransportMessageV0::NoData.to_bytes(), + re_protos::remote_store::v0::EncoderVersion::V0 => TransportMessageV0::NoData.to_bytes(), } } /// Decode transport data from a byte stream - if there's a record batch present, return it, otherwise return `None`. pub fn decode( - version: re_protos::common::v0::EncoderVersion, + version: re_protos::remote_store::v0::EncoderVersion, data: &[u8], ) -> Result, CodecError> { match version { - re_protos::common::v0::EncoderVersion::V0 => { + re_protos::remote_store::v0::EncoderVersion::V0 => { let msg = TransportMessageV0::from_bytes(data)?; match msg { TransportMessageV0::RecordBatch(chunk) => Ok(Some(chunk)), @@ -255,7 +258,7 @@ mod tests { use crate::codec::wire::recording_metadata_to_chunk; use crate::codec::wire::{decode, encode, TransportMessageV0}; use crate::codec::CodecError; - use re_protos::common::v0::EncoderVersion; + use re_protos::remote_store::v0::EncoderVersion; fn get_test_chunk() -> Chunk { let row_id1 = RowId::new(); diff --git a/crates/store/re_protos/proto/rerun/v0/common.proto b/crates/store/re_protos/proto/rerun/v0/common.proto index 9370873d4989..46a34a2a505b 100644 --- a/crates/store/re_protos/proto/rerun/v0/common.proto +++ b/crates/store/re_protos/proto/rerun/v0/common.proto @@ -193,10 +193,6 @@ message Time { int64 nanos_since_epoch = 1; } -enum EncoderVersion { - V0 = 0; -} - message Tuid { // Approximate nanoseconds since epoch. fixed64 time_ns = 1; diff --git a/crates/store/re_protos/proto/rerun/v0/remote_store.proto b/crates/store/re_protos/proto/rerun/v0/remote_store.proto index e98a40958d0d..49d8e5a64936 100644 --- a/crates/store/re_protos/proto/rerun/v0/remote_store.proto +++ b/crates/store/re_protos/proto/rerun/v0/remote_store.proto @@ -36,7 +36,7 @@ message RecordingMetadata { } message RegisterRecordingResponse { - RecordingId id = 1; + rerun.common.v0.RecordingId id = 1; // Note / TODO(zehiko): this implies we read the record (for example go through entire .rrd file // chunk by chunk) and extract the metadata. So we might want to 1/ not do this i.e. // only do it as part of explicit GetMetadata request or 2/ do it if Request has "include_metadata=true" @@ -47,7 +47,7 @@ message RegisterRecordingResponse { // ---------------- UpdateCatalog ----------------- message UpdateCatalogRequest { - RecordingId recording_id = 1; + rerun.common.v0.RecordingId recording_id = 1; RecordingMetadata metadata = 2; } @@ -57,9 +57,9 @@ message UpdateCatalogResponse {} message QueryRequest { // unique identifier of the recording - RecordingId recording_id = 1; + rerun.common.v0.RecordingId recording_id = 1; // query to execute - Query query = 3; + rerun.common.v0.Query query = 3; } message QueryResponse { @@ -71,12 +71,10 @@ message QueryResponse { bytes payload = 2; } - enum EncoderVersion { V0 = 0; } - // ----------------- QueryCatalog ----------------- message QueryCatalogRequest { @@ -94,7 +92,7 @@ message ColumnProjection { message CatalogFilter { // Filtering is very simple right now, we can only select // recordings by their ids. - repeated RecordingId recording_ids = 1; + repeated rerun.common.v0.RecordingId recording_ids = 1; } message QueryCatalogResponse { @@ -110,7 +108,7 @@ enum RecordingType { // ----------------- FetchRecording ----------------- message FetchRecordingRequest { - RecordingId recording_id = 1; + rerun.common.v0.RecordingId recording_id = 1; } // TODO(jleibs): Eventually this becomes either query-mediated in some way, but for now diff --git a/crates/store/re_protos/src/lib.rs b/crates/store/re_protos/src/lib.rs index 841f2a95a0c5..c3e917a871db 100644 --- a/crates/store/re_protos/src/lib.rs +++ b/crates/store/re_protos/src/lib.rs @@ -7,7 +7,7 @@ //! // This extra module is needed, because of how imports from different packages are resolved. -// For example, `rerun.common.v0.EncoderVersion` is resolved to `super::super::common::v0::EncoderVersion`. +// For example, `rerun.remote_store.v0.EncoderVersion` is resolved to `super::super::remote_store::v0::EncoderVersion`. // We need an extra module in the path to `common` to make that work. // Additionally, the `common` module itself has to exist with a `v0` module inside of it, // which is the reason for the `common`, `log_msg`, `remote_store`, etc. modules below. diff --git a/crates/store/re_protos/src/v0/rerun.common.v0.rs b/crates/store/re_protos/src/v0/rerun.common.v0.rs index 171e0633a9a8..34811ca634c4 100644 --- a/crates/store/re_protos/src/v0/rerun.common.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.common.v0.rs @@ -296,26 +296,3 @@ impl StoreKind { } } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum EncoderVersion { - V0 = 0, -} -impl EncoderVersion { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::V0 => "V0", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "V0" => Some(Self::V0), - _ => None, - } - } -} diff --git a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs index e956a1082d00..c458f491f9f3 100644 --- a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs @@ -1,214 +1,4 @@ // This file is @generated by prost-build. -/// unique recording identifier. At this point in time it is the same id as the ChunkStore's StoreId -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct RecordingId { - #[prost(string, tag = "1")] - pub id: ::prost::alloc::string::String, -} -/// A recording can have multiple timelines, each is identified by a name, for example `log_tick`, `log_time`, etc. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Timeline { - #[prost(string, tag = "1")] - pub name: ::prost::alloc::string::String, -} -/// A time range between start and end time points. Each 64 bit number can represent different time point data -/// depending on the timeline it is associated with. Time range is inclusive for both start and end time points. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct TimeRange { - #[prost(int64, tag = "1")] - pub start: i64, - #[prost(int64, tag = "2")] - pub end: i64, -} -/// arrow IPC serialized schema -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Schema { - #[prost(bytes = "vec", tag = "1")] - pub arrow_schema: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Query { - /// The subset of the database that the query will run on: a set of EntityPath(s) and their - /// associated Component(s) - #[prost(message, optional, tag = "1")] - pub view_contents: ::core::option::Option, - /// Whether the view_contents should ignore semantically empty columns - /// A semantically empty column is a column that either contains no data at all, or where all - /// values are either nulls or empty arrays (\[\]). - #[prost(bool, tag = "2")] - pub include_semantically_empty_columns: bool, - /// Whether the view_contents should ignore columns corresponding to indicator components - /// Indicator components are marker components, generally automatically inserted by Rerun, that - /// helps keep track of the original context in which a piece of data was logged/sent. - #[prost(bool, tag = "3")] - pub include_indicator_columns: bool, - /// Whether the view_contents should ignore columns corresponding to Clear-related components - #[prost(bool, tag = "4")] - pub include_tombstone_columns: bool, - /// The index used to filter out _rows_ from the view contents. - /// Only rows where at least 1 column contains non-null data at that index will be kept in the - /// final dataset. If left unspecified, the results will only contain static data. - #[prost(message, optional, tag = "5")] - pub filtered_index: ::core::option::Option, - /// The range of index values used to filter out _rows_ from the view contents - /// Only rows where at least 1 of the view-contents contains non-null data within that range will be kept in - /// the final dataset. - /// This has no effect if filtered_index isn't set. - /// This has no effect if using_index_values is set. - #[prost(message, optional, tag = "6")] - pub filtered_index_range: ::core::option::Option, - /// The specific index values used to filter out _rows_ from the view contents. - /// Only rows where at least 1 column contains non-null data at these specific values will be kept - /// in the final dataset. - /// This has no effect if filtered_index isn't set. - /// This has no effect if using_index_values is set. - #[prost(message, optional, tag = "7")] - pub filtered_index_values: ::core::option::Option, - /// The specific index values used to sample _rows_ from the view contents. - /// The final dataset will contain one row per sampled index value, regardless of whether data - /// existed for that index value in the view contents. - /// The semantics of the query are consistent with all other settings: the results will be - /// sorted on the filtered_index, and only contain unique index values. - /// - /// This has no effect if filtered_index isn't set. - /// If set, this overrides both filtered_index_range and filtered_index_values. - #[prost(message, optional, tag = "8")] - pub using_index_values: ::core::option::Option, - /// The component column used to filter out _rows_ from the view contents. - /// Only rows where this column contains non-null data be kept in the final dataset. - #[prost(message, optional, tag = "9")] - pub filtered_is_not_null: ::core::option::Option, - /// / The specific _columns_ to sample from the final view contents. - /// / The order of the samples will be respected in the final result. - /// / - /// / If unspecified, it means - everything. - #[prost(message, optional, tag = "10")] - pub column_selection: ::core::option::Option, - /// Specifies how null values should be filled in the returned dataframe. - #[prost(enumeration = "SparseFillStrategy", tag = "11")] - pub sparse_fill_strategy: i32, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ColumnSelection { - #[prost(message, repeated, tag = "1")] - pub columns: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ColumnSelector { - #[prost(oneof = "column_selector::SelectorType", tags = "2, 3")] - pub selector_type: ::core::option::Option, -} -/// Nested message and enum types in `ColumnSelector`. -pub mod column_selector { - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum SelectorType { - #[prost(message, tag = "2")] - ComponentColumn(super::ComponentColumnSelector), - #[prost(message, tag = "3")] - TimeColumn(super::TimeColumnSelector), - } -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexColumnSelector { - /// TODO(zehiko) we need to add support for other types of index selectors - #[prost(message, optional, tag = "1")] - pub timeline: ::core::option::Option, -} -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct IndexRange { - /// TODO(zehiko) support for other ranges for other index selectors - #[prost(message, optional, tag = "1")] - pub time_range: ::core::option::Option, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexValues { - /// TODO(zehiko) we need to add support for other types of index selectors - #[prost(message, repeated, tag = "1")] - pub time_points: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SampledIndexValues { - #[prost(message, repeated, tag = "1")] - pub sample_points: ::prost::alloc::vec::Vec, -} -/// A 64-bit number describing either nanoseconds, sequence numbers or fully static data. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct TimeInt { - #[prost(int64, tag = "1")] - pub time: i64, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ViewContents { - #[prost(message, repeated, tag = "1")] - pub contents: ::prost::alloc::vec::Vec, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ViewContentsPart { - #[prost(message, optional, tag = "1")] - pub path: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub components: ::core::option::Option, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ComponentsSet { - #[prost(message, repeated, tag = "1")] - pub components: ::prost::alloc::vec::Vec, -} -/// The unique identifier of an entity, e.g. `camera/3/points` -/// See <> for more on entity paths. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EntityPath { - #[prost(string, tag = "1")] - pub path: ::prost::alloc::string::String, -} -/// Component describes semantic data that can be used by any number of rerun's archetypes. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Component { - /// component name needs to be a string as user can define their own component - #[prost(string, tag = "1")] - pub name: ::prost::alloc::string::String, -} -/// Used to telect a time column. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TimeColumnSelector { - #[prost(message, optional, tag = "1")] - pub timeline: ::core::option::Option, -} -/// Used to select a component based on its EntityPath and Component name. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ComponentColumnSelector { - #[prost(message, optional, tag = "1")] - pub entity_path: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub component: ::core::option::Option, -} -/// Specifies how null values should be filled in the returned dataframe. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SparseFillStrategy { - None = 0, - LatestAtGlobal = 1, -} -impl SparseFillStrategy { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::None => "NONE", - Self::LatestAtGlobal => "LATEST_AT_GLOBAL", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "NONE" => Some(Self::None), - "LATEST_AT_GLOBAL" => Some(Self::LatestAtGlobal), - _ => None, - } - } -} #[derive(Clone, PartialEq, ::prost::Message)] pub struct RegisterRecordingRequest { /// human readable description of the recording @@ -236,7 +26,7 @@ pub struct RecordingMetadata { #[derive(Clone, PartialEq, ::prost::Message)] pub struct RegisterRecordingResponse { #[prost(message, optional, tag = "1")] - pub id: ::core::option::Option, + pub id: ::core::option::Option, /// Note / TODO(zehiko): this implies we read the record (for example go through entire .rrd file /// chunk by chunk) and extract the metadata. So we might want to 1/ not do this i.e. /// only do it as part of explicit GetMetadata request or 2/ do it if Request has "include_metadata=true" @@ -247,7 +37,7 @@ pub struct RegisterRecordingResponse { #[derive(Clone, PartialEq, ::prost::Message)] pub struct UpdateCatalogRequest { #[prost(message, optional, tag = "1")] - pub recording_id: ::core::option::Option, + pub recording_id: ::core::option::Option, #[prost(message, optional, tag = "2")] pub metadata: ::core::option::Option, } @@ -257,10 +47,10 @@ pub struct UpdateCatalogResponse {} pub struct QueryRequest { /// unique identifier of the recording #[prost(message, optional, tag = "1")] - pub recording_id: ::core::option::Option, + pub recording_id: ::core::option::Option, /// query to execute #[prost(message, optional, tag = "3")] - pub query: ::core::option::Option, + pub query: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct QueryResponse { @@ -293,7 +83,7 @@ pub struct CatalogFilter { /// Filtering is very simple right now, we can only select /// recordings by their ids. #[prost(message, repeated, tag = "1")] - pub recording_ids: ::prost::alloc::vec::Vec, + pub recording_ids: ::prost::alloc::vec::Vec, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct QueryCatalogResponse { @@ -306,7 +96,7 @@ pub struct QueryCatalogResponse { #[derive(Clone, PartialEq, ::prost::Message)] pub struct FetchRecordingRequest { #[prost(message, optional, tag = "1")] - pub recording_id: ::core::option::Option, + pub recording_id: ::core::option::Option, } /// TODO(jleibs): Eventually this becomes either query-mediated in some way, but for now /// it's useful to be able to just get back the whole RRD somehow. @@ -424,10 +214,10 @@ pub mod storage_node_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value + clippy::let_unit_value, )] - use tonic::codegen::http::Uri; use tonic::codegen::*; + use tonic::codegen::http::Uri; #[derive(Debug, Clone)] pub struct StorageNodeClient { inner: tonic::client::Grpc, @@ -460,8 +250,9 @@ pub mod storage_node_client { >::ResponseBody, >, >, - >>::Error: - Into + std::marker::Send + std::marker::Sync, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, { StorageNodeClient::new(InterceptedService::new(inner, interceptor)) } @@ -504,17 +295,21 @@ pub mod storage_node_client { tonic::Response>, tonic::Status, > { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = - http::uri::PathAndQuery::from_static("/rerun.remote_store.v0.StorageNode/Query"); + let path = http::uri::PathAndQuery::from_static( + "/rerun.remote_store.v0.StorageNode/Query", + ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "Query", - )); + req.extensions_mut() + .insert(GrpcMethod::new("rerun.remote_store.v0.StorageNode", "Query")); self.inner.server_streaming(req, path, codec).await } pub async fn fetch_recording( @@ -524,18 +319,26 @@ pub mod storage_node_client { tonic::Response>, tonic::Status, > { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/FetchRecording", ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "FetchRecording", - )); + req.extensions_mut() + .insert( + GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "FetchRecording", + ), + ); self.inner.server_streaming(req, path, codec).await } /// metadata API calls @@ -546,56 +349,78 @@ pub mod storage_node_client { tonic::Response>, tonic::Status, > { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/QueryCatalog", ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "QueryCatalog", - )); + req.extensions_mut() + .insert( + GrpcMethod::new("rerun.remote_store.v0.StorageNode", "QueryCatalog"), + ); self.inner.server_streaming(req, path, codec).await } pub async fn update_catalog( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> - { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/UpdateCatalog", ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "UpdateCatalog", - )); + req.extensions_mut() + .insert( + GrpcMethod::new("rerun.remote_store.v0.StorageNode", "UpdateCatalog"), + ); self.inner.unary(req, path, codec).await } pub async fn register_recording( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> - { - self.inner.ready().await.map_err(|e| { - tonic::Status::unknown(format!("Service was not ready: {}", e.into())) - })?; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/RegisterRecording", ); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "RegisterRecording", - )); + req.extensions_mut() + .insert( + GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "RegisterRecording", + ), + ); self.inner.unary(req, path, codec).await } } @@ -607,7 +432,7 @@ pub mod storage_node_server { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value + clippy::let_unit_value, )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with StorageNodeServer. @@ -616,7 +441,8 @@ pub mod storage_node_server { /// Server streaming response type for the Query method. type QueryStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, - > + std::marker::Send + > + + std::marker::Send + 'static; /// data API calls async fn query( @@ -626,30 +452,44 @@ pub mod storage_node_server { /// Server streaming response type for the FetchRecording method. type FetchRecordingStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, - > + std::marker::Send + > + + std::marker::Send + 'static; async fn fetch_recording( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; /// Server streaming response type for the QueryCatalog method. type QueryCatalogStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, - > + std::marker::Send + > + + std::marker::Send + 'static; /// metadata API calls async fn query_catalog( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn update_catalog( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn register_recording( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } #[derive(Debug)] pub struct StorageNodeServer { @@ -672,7 +512,10 @@ pub mod storage_node_server { max_encoding_message_size: None, } } - pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService where F: tonic::service::Interceptor, { @@ -727,18 +570,24 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/Query" => { #[allow(non_camel_case_types)] struct QuerySvc(pub Arc); - impl tonic::server::ServerStreamingService for QuerySvc { + impl< + T: StorageNode, + > tonic::server::ServerStreamingService + for QuerySvc { type Response = super::QueryResponse; type ResponseStream = T::QueryStream; - type Future = - BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = - async move { ::query(&inner, request).await }; + let fut = async move { + ::query(&inner, request).await + }; Box::pin(fut) } } @@ -767,14 +616,16 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/FetchRecording" => { #[allow(non_camel_case_types)] struct FetchRecordingSvc(pub Arc); - impl - tonic::server::ServerStreamingService - for FetchRecordingSvc - { + impl< + T: StorageNode, + > tonic::server::ServerStreamingService + for FetchRecordingSvc { type Response = super::FetchRecordingResponse; type ResponseStream = T::FetchRecordingStream; - type Future = - BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, request: tonic::Request, @@ -811,14 +662,16 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/QueryCatalog" => { #[allow(non_camel_case_types)] struct QueryCatalogSvc(pub Arc); - impl - tonic::server::ServerStreamingService - for QueryCatalogSvc - { + impl< + T: StorageNode, + > tonic::server::ServerStreamingService + for QueryCatalogSvc { type Response = super::QueryCatalogResponse; type ResponseStream = T::QueryCatalogStream; - type Future = - BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, request: tonic::Request, @@ -855,11 +708,15 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/UpdateCatalog" => { #[allow(non_camel_case_types)] struct UpdateCatalogSvc(pub Arc); - impl tonic::server::UnaryService - for UpdateCatalogSvc - { + impl< + T: StorageNode, + > tonic::server::UnaryService + for UpdateCatalogSvc { type Response = super::UpdateCatalogResponse; - type Future = BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, request: tonic::Request, @@ -896,19 +753,23 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/RegisterRecording" => { #[allow(non_camel_case_types)] struct RegisterRecordingSvc(pub Arc); - impl - tonic::server::UnaryService - for RegisterRecordingSvc - { + impl< + T: StorageNode, + > tonic::server::UnaryService + for RegisterRecordingSvc { type Response = super::RegisterRecordingResponse; - type Future = BoxFuture, tonic::Status>; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; fn call( &mut self, request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { - ::register_recording(&inner, request).await + ::register_recording(&inner, request) + .await }; Box::pin(fut) } @@ -935,19 +796,23 @@ pub mod storage_node_server { }; Box::pin(fut) } - _ => Box::pin(async move { - let mut response = http::Response::new(empty_body()); - let headers = response.headers_mut(); - headers.insert( - tonic::Status::GRPC_STATUS, - (tonic::Code::Unimplemented as i32).into(), - ); - headers.insert( - http::header::CONTENT_TYPE, - tonic::metadata::GRPC_CONTENT_TYPE, - ); - Ok(response) - }), + _ => { + Box::pin(async move { + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } } } } diff --git a/rerun_py/src/remote.rs b/rerun_py/src/remote.rs index abb95fc12e1f..bb4e223e8129 100644 --- a/rerun_py/src/remote.rs +++ b/rerun_py/src/remote.rs @@ -9,13 +9,13 @@ use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyDict, Bound, PyResul use re_chunk::{Chunk, TransportChunk}; use re_chunk_store::ChunkStore; use re_dataframe::ChunkStoreHandle; +use re_log_encoding::codec::wire::{self, decode}; use re_log_types::{StoreInfo, StoreSource}; use re_protos::{ - codec::decode, - v0::{ + common::v0::RecordingId, + remote_store::v0::{ storage_node_client::StorageNodeClient, EncoderVersion, FetchRecordingRequest, - QueryCatalogRequest, RecordingId, RecordingMetadata, RecordingType, - RegisterRecordingRequest, UpdateCatalogRequest, + QueryCatalogRequest, RecordingType, RegisterRecordingRequest, UpdateCatalogRequest, }, }; use re_sdk::{ApplicationId, StoreId, StoreKind, Time}; @@ -183,7 +183,7 @@ impl PyStorageNodeClient { data, }; - RecordingMetadata::try_from(EncoderVersion::V0, &metadata_tc) + wire::chunk_to_recording_metadata(EncoderVersion::V0, &metadata_tc) .map_err(|err| PyRuntimeError::new_err(err.to_string())) }) .transpose()?; @@ -249,7 +249,7 @@ impl PyStorageNodeClient { data, }; - let metadata = RecordingMetadata::try_from(EncoderVersion::V0, &metadata_tc) + let metadata = wire::chunk_to_recording_metadata(EncoderVersion::V0, &metadata_tc) .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; let request = UpdateCatalogRequest { From 34dcd4c011e4ff5e9b58722fc0d771487a6612f7 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 18:57:27 +0100 Subject: [PATCH 05/47] remove unused dep --- Cargo.lock | 1 - crates/store/re_grpc_client/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d4482d682c8..2b3c63241b25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5780,7 +5780,6 @@ dependencies = [ name = "re_grpc_client" version = "0.21.0-alpha.1+dev" dependencies = [ - "re_arrow2", "re_chunk", "re_error", "re_log", diff --git a/crates/store/re_grpc_client/Cargo.toml b/crates/store/re_grpc_client/Cargo.toml index 00f9f3d70efa..665e7fc6faa9 100644 --- a/crates/store/re_grpc_client/Cargo.toml +++ b/crates/store/re_grpc_client/Cargo.toml @@ -28,7 +28,6 @@ re_log_types.workspace = true re_protos.workspace = true re_smart_channel.workspace = true -arrow2.workspace = true thiserror.workspace = true tokio-stream.workspace = true url.workspace = true From ba4cfbb8e753c0aa5db744386e235053c20811cb Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 18:58:47 +0100 Subject: [PATCH 06/47] exclude `re_grpc_client/address` links --- lychee.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/lychee.toml b/lychee.toml index 5b40598bb128..d11974584ef0 100644 --- a/lychee.toml +++ b/lychee.toml @@ -67,6 +67,7 @@ exclude_path = [ "scripts/lint.py", # Contains url-matching regexes that aren't actual urls "scripts/screenshot_compare/assets/templates/", "crates/viewer/re_viewer/src/reflection/mod.rs", # Checker struggles how links from examples are escaped here. They are all checked elsewhere, so not an issue. + "crates/store/re_grpc_client/src/address.rs", # Contains some malformed URLs, but they are not actual links. ] # Exclude URLs and mail addresses from checking (supports regex). From 691fb465cdff3fe351c4e2d9d1b1a2213be69a29 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 19:08:08 +0100 Subject: [PATCH 07/47] cargo fmt --- .../re_protos/src/v0/rerun.remote_store.v0.rs | 263 +++++++----------- 1 file changed, 94 insertions(+), 169 deletions(-) diff --git a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs index c458f491f9f3..84be4fb4acca 100644 --- a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs @@ -214,10 +214,10 @@ pub mod storage_node_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; #[derive(Debug, Clone)] pub struct StorageNodeClient { inner: tonic::client::Grpc, @@ -250,9 +250,8 @@ pub mod storage_node_client { >::ResponseBody, >, >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { StorageNodeClient::new(InterceptedService::new(inner, interceptor)) } @@ -295,21 +294,17 @@ pub mod storage_node_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/rerun.remote_store.v0.StorageNode/Query", - ); + let path = + http::uri::PathAndQuery::from_static("/rerun.remote_store.v0.StorageNode/Query"); let mut req = request.into_request(); - req.extensions_mut() - .insert(GrpcMethod::new("rerun.remote_store.v0.StorageNode", "Query")); + req.extensions_mut().insert(GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "Query", + )); self.inner.server_streaming(req, path, codec).await } pub async fn fetch_recording( @@ -319,26 +314,18 @@ pub mod storage_node_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/FetchRecording", ); let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "FetchRecording", - ), - ); + req.extensions_mut().insert(GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "FetchRecording", + )); self.inner.server_streaming(req, path, codec).await } /// metadata API calls @@ -349,78 +336,56 @@ pub mod storage_node_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/QueryCatalog", ); let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new("rerun.remote_store.v0.StorageNode", "QueryCatalog"), - ); + req.extensions_mut().insert(GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "QueryCatalog", + )); self.inner.server_streaming(req, path, codec).await } pub async fn update_catalog( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/UpdateCatalog", ); let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new("rerun.remote_store.v0.StorageNode", "UpdateCatalog"), - ); + req.extensions_mut().insert(GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "UpdateCatalog", + )); self.inner.unary(req, path, codec).await } pub async fn register_recording( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/rerun.remote_store.v0.StorageNode/RegisterRecording", ); let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new( - "rerun.remote_store.v0.StorageNode", - "RegisterRecording", - ), - ); + req.extensions_mut().insert(GrpcMethod::new( + "rerun.remote_store.v0.StorageNode", + "RegisterRecording", + )); self.inner.unary(req, path, codec).await } } @@ -432,7 +397,7 @@ pub mod storage_node_server { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with StorageNodeServer. @@ -441,8 +406,7 @@ pub mod storage_node_server { /// Server streaming response type for the Query method. type QueryStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, - > - + std::marker::Send + > + std::marker::Send + 'static; /// data API calls async fn query( @@ -452,44 +416,30 @@ pub mod storage_node_server { /// Server streaming response type for the FetchRecording method. type FetchRecordingStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, - > - + std::marker::Send + > + std::marker::Send + 'static; async fn fetch_recording( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; /// Server streaming response type for the QueryCatalog method. type QueryCatalogStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, - > - + std::marker::Send + > + std::marker::Send + 'static; /// metadata API calls async fn query_catalog( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn update_catalog( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn register_recording( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] pub struct StorageNodeServer { @@ -512,10 +462,7 @@ pub mod storage_node_server { max_encoding_message_size: None, } } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> InterceptedService + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService where F: tonic::service::Interceptor, { @@ -570,24 +517,18 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/Query" => { #[allow(non_camel_case_types)] struct QuerySvc(pub Arc); - impl< - T: StorageNode, - > tonic::server::ServerStreamingService - for QuerySvc { + impl tonic::server::ServerStreamingService for QuerySvc { type Response = super::QueryResponse; type ResponseStream = T::QueryStream; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = + BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::query(&inner, request).await - }; + let fut = + async move { ::query(&inner, request).await }; Box::pin(fut) } } @@ -616,16 +557,14 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/FetchRecording" => { #[allow(non_camel_case_types)] struct FetchRecordingSvc(pub Arc); - impl< - T: StorageNode, - > tonic::server::ServerStreamingService - for FetchRecordingSvc { + impl + tonic::server::ServerStreamingService + for FetchRecordingSvc + { type Response = super::FetchRecordingResponse; type ResponseStream = T::FetchRecordingStream; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = + BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request, @@ -662,16 +601,14 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/QueryCatalog" => { #[allow(non_camel_case_types)] struct QueryCatalogSvc(pub Arc); - impl< - T: StorageNode, - > tonic::server::ServerStreamingService - for QueryCatalogSvc { + impl + tonic::server::ServerStreamingService + for QueryCatalogSvc + { type Response = super::QueryCatalogResponse; type ResponseStream = T::QueryCatalogStream; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = + BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request, @@ -708,15 +645,11 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/UpdateCatalog" => { #[allow(non_camel_case_types)] struct UpdateCatalogSvc(pub Arc); - impl< - T: StorageNode, - > tonic::server::UnaryService - for UpdateCatalogSvc { + impl tonic::server::UnaryService + for UpdateCatalogSvc + { type Response = super::UpdateCatalogResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request, @@ -753,23 +686,19 @@ pub mod storage_node_server { "/rerun.remote_store.v0.StorageNode/RegisterRecording" => { #[allow(non_camel_case_types)] struct RegisterRecordingSvc(pub Arc); - impl< - T: StorageNode, - > tonic::server::UnaryService - for RegisterRecordingSvc { + impl + tonic::server::UnaryService + for RegisterRecordingSvc + { type Response = super::RegisterRecordingResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; + type Future = BoxFuture, tonic::Status>; fn call( &mut self, request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { - ::register_recording(&inner, request) - .await + ::register_recording(&inner, request).await }; Box::pin(fut) } @@ -796,23 +725,19 @@ pub mod storage_node_server { }; Box::pin(fut) } - _ => { - Box::pin(async move { - let mut response = http::Response::new(empty_body()); - let headers = response.headers_mut(); - headers - .insert( - tonic::Status::GRPC_STATUS, - (tonic::Code::Unimplemented as i32).into(), - ); - headers - .insert( - http::header::CONTENT_TYPE, - tonic::metadata::GRPC_CONTENT_TYPE, - ); - Ok(response) - }) - } + _ => Box::pin(async move { + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), } } } From a2e58fb1982162b81a13a3f0477d80055445d8b7 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 19:36:50 +0100 Subject: [PATCH 08/47] fix lints --- crates/store/re_log_types/src/lib.rs | 32 +++++++++------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/crates/store/re_log_types/src/lib.rs b/crates/store/re_log_types/src/lib.rs index 36a9ef977ba3..79eda6b77f9b 100644 --- a/crates/store/re_log_types/src/lib.rs +++ b/crates/store/re_log_types/src/lib.rs @@ -431,16 +431,17 @@ impl std::str::FromStr for PythonVersion { if patch.is_empty() { return Err(PythonVersionParseError::MissingPatch); } + Ok(Self { major: major .parse() - .map_err(|_| PythonVersionParseError::InvalidMajor)?, + .map_err(PythonVersionParseError::InvalidMajor)?, minor: minor .parse() - .map_err(|_| PythonVersionParseError::InvalidMinor)?, + .map_err(PythonVersionParseError::InvalidMinor)?, patch: patch .parse() - .map_err(|_| PythonVersionParseError::InvalidPatch)?, + .map_err(PythonVersionParseError::InvalidPatch)?, suffix: suffix.into(), }) } @@ -457,27 +458,14 @@ pub enum PythonVersionParseError { #[error("missing patch version")] MissingPatch, - #[error("invalid major version")] - InvalidMajor, - - #[error("invalid minor version")] - InvalidMinor, + #[error("invalid major version: {0}")] + InvalidMajor(std::num::ParseIntError), - #[error("invalid patch version")] - InvalidPatch, -} + #[error("invalid minor version: {0}")] + InvalidMinor(std::num::ParseIntError), -impl PythonVersionParseError { - pub fn as_str(&self) -> &'static str { - match self { - Self::MissingMajor => "missing major version", - Self::MissingMinor => "missing minor version", - Self::MissingPatch => "missing patch version", - Self::InvalidMajor => "invalid major version", - Self::InvalidMinor => "invalid minor version", - Self::InvalidPatch => "invalid patch version", - } - } + #[error("invalid patch version: {0}")] + InvalidPatch(std::num::ParseIntError), } #[derive(Clone, Debug, PartialEq, Eq, Hash)] From f22cb031492b042e9f54b70263adc4ee88a266b9 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 19:37:53 +0100 Subject: [PATCH 09/47] rm dead comment --- .../re_protos/proto/rerun/v0/log_msg.proto | 67 ------------------- 1 file changed, 67 deletions(-) diff --git a/crates/store/re_protos/proto/rerun/v0/log_msg.proto b/crates/store/re_protos/proto/rerun/v0/log_msg.proto index b3c42f0d6580..e65b7453e032 100644 --- a/crates/store/re_protos/proto/rerun/v0/log_msg.proto +++ b/crates/store/re_protos/proto/rerun/v0/log_msg.proto @@ -118,70 +118,3 @@ enum FileSourceKind { FILE_DIALOG = 4; SDK = 5; } - -/* -pub struct StoreInfo { - /// The user-chosen name of the application doing the logging. - pub application_id: ApplicationId, - - /// Should be unique for each recording. - pub store_id: StoreId, - - /// If this store is the result of a clone, which store was it cloned from? - /// - /// A cloned store always gets a new unique ID. - /// - /// We currently only clone stores for blueprints: - /// when we receive a _default_ blueprints on the wire (e.g. from a recording), - /// we clone it and make the clone the _active_ blueprint. - /// This means all active blueprints are clones. - pub cloned_from: Option, - - /// True if the recording is one of the official Rerun examples. - pub is_official_example: bool, - - /// When the recording started. - /// - /// Should be an absolute time, i.e. relative to Unix Epoch. - pub started: Time, - - pub store_source: StoreSource, - - /// The Rerun version used to encoded the RRD data. - /// - // NOTE: The version comes directly from the decoded RRD stream's header, duplicating it here - // would probably only lead to more issues down the line. - #[cfg_attr(feature = "serde", serde(skip))] - pub store_version: Option, -} - -pub enum StoreSource { - Unknown, - - /// The official Rerun C Logging SDK - CSdk, - - /// The official Rerun Python Logging SDK - PythonSdk(PythonVersion), - - /// The official Rerun Rust Logging SDK - RustSdk { - /// Rust version of the code compiling the Rust SDK - rustc_version: String, - - /// LLVM version of the code compiling the Rust SDK - llvm_version: String, - }, - - /// Loading a file via CLI, drag-and-drop, a file-dialog, etc. - File { - file_source: FileSource, - }, - - /// Generated from the viewer itself. - Viewer, - - /// Perhaps from some manual data ingestion? - Other(String), -} -*/ From b76cf9c3932e0582395bda18e1da0808d0ca5329 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 19:38:32 +0100 Subject: [PATCH 10/47] add todo --- .../re_protos/proto/rerun/v0/log_msg.proto | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/crates/store/re_protos/proto/rerun/v0/log_msg.proto b/crates/store/re_protos/proto/rerun/v0/log_msg.proto index e65b7453e032..1babb3312423 100644 --- a/crates/store/re_protos/proto/rerun/v0/log_msg.proto +++ b/crates/store/re_protos/proto/rerun/v0/log_msg.proto @@ -4,22 +4,7 @@ package rerun.log_msg.v0; import "rerun/v0/common.proto"; -/* -// not actually protobuf -message MessageHeader { - enum MessageKind { - UNKNOWN_KIND = 0; - ARROW_MSG = 1; - SET_STORE_INFO = 2; - BLUEPRINT_ACTIVATION_COMMAND = 3; - } - - MessageKind kind = 1; - - // len-prefixed - bytes payload = 1000; -} -*/ +// TODO(jan): documentation here message SetStoreInfo { rerun.common.v0.Tuid row_id = 1; From f91737b84d2033ec443e4483c212d52a66bc59c9 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 19:51:15 +0100 Subject: [PATCH 11/47] gate behind feature --- .../store/re_log_encoding/src/decoder/mod.rs | 2 +- crates/store/re_log_encoding/src/encoder.rs | 6 +- crates/store/re_log_encoding/src/protobuf.rs | 234 +----------------- .../re_log_encoding/src/protobuf/decoder.rs | 133 ++++++++++ .../re_log_encoding/src/protobuf/encoder.rs | 105 ++++++++ 5 files changed, 248 insertions(+), 232 deletions(-) create mode 100644 crates/store/re_log_encoding/src/protobuf/decoder.rs create mode 100644 crates/store/re_log_encoding/src/protobuf/encoder.rs diff --git a/crates/store/re_log_encoding/src/decoder/mod.rs b/crates/store/re_log_encoding/src/decoder/mod.rs index 617b2cc0a6f4..f6cd36578d12 100644 --- a/crates/store/re_log_encoding/src/decoder/mod.rs +++ b/crates/store/re_log_encoding/src/decoder/mod.rs @@ -307,7 +307,7 @@ impl Iterator for Decoder { let msg = match self.options.serializer { Serializer::Protobuf => { - match crate::protobuf::decode(&mut self.read, self.options.compression) { + match crate::protobuf::decoder::decode(&mut self.read, self.options.compression) { Ok((read_bytes, msg)) => { self.size_bytes += read_bytes; msg diff --git a/crates/store/re_log_encoding/src/encoder.rs b/crates/store/re_log_encoding/src/encoder.rs index b43943a8d378..247a3b97d2bd 100644 --- a/crates/store/re_log_encoding/src/encoder.rs +++ b/crates/store/re_log_encoding/src/encoder.rs @@ -155,7 +155,11 @@ impl Encoder { self.uncompressed.clear(); match self.serializer { Serializer::Protobuf => { - crate::protobuf::encode(&mut self.uncompressed, message, self.compression)?; + crate::protobuf::encoder::encode( + &mut self.uncompressed, + message, + self.compression, + )?; self.write .write_all(&self.uncompressed) diff --git a/crates/store/re_log_encoding/src/protobuf.rs b/crates/store/re_log_encoding/src/protobuf.rs index b24846b4940b..cd1e2990b670 100644 --- a/crates/store/re_log_encoding/src/protobuf.rs +++ b/crates/store/re_log_encoding/src/protobuf.rs @@ -2,8 +2,6 @@ use re_log_types::LogMsg; use re_protos::TypeConversionError; use crate::codec::CodecError; -use crate::decoder::DecodeError; -use crate::encoder::EncodeError; use crate::Compression; #[derive(Debug, Clone, Copy)] @@ -15,238 +13,14 @@ pub(crate) enum MessageKind { End = 255, } -impl MessageKind { - pub(crate) fn encode(&self, buf: &mut impl std::io::Write) -> Result<(), EncodeError> { - let kind: u32 = *self as u32; - buf.write_all(&kind.to_le_bytes())?; - Ok(()) - } - - pub(crate) fn decode(data: &mut impl std::io::Read) -> Result { - let mut buf = [0; 4]; - data.read_exact(&mut buf)?; - - match u32::from_le_bytes(buf) { - 1 => Ok(Self::SetStoreInfo), - 2 => Ok(Self::ArrowMsg), - 3 => Ok(Self::BlueprintActivationCommand), - 255 => Ok(Self::End), - _ => Err(DecodeError::Codec(CodecError::UnknownMessageHeader)), - } - } -} - #[derive(Debug, Clone, Copy)] pub(crate) struct MessageHeader { pub(crate) kind: MessageKind, pub(crate) len: u32, } -impl MessageHeader { - pub(crate) fn encode(&self, buf: &mut impl std::io::Write) -> Result<(), EncodeError> { - self.kind.encode(buf)?; - buf.write_all(&self.len.to_le_bytes())?; - Ok(()) - } - - pub(crate) fn decode(data: &mut impl std::io::Read) -> Result { - let kind = MessageKind::decode(data)?; - let mut buf = [0; 4]; - data.read_exact(&mut buf)?; - let len = u32::from_le_bytes(buf); - - Ok(Self { kind, len }) - } -} - -pub(crate) fn encode( - buf: &mut Vec, - message: &LogMsg, - compression: Compression, -) -> Result<(), EncodeError> { - use re_protos::external::prost::Message; - use re_protos::log_msg::v0::{ - self as proto, ArrowMsg, BlueprintActivationCommand, Encoding, SetStoreInfo, - }; - - match message { - LogMsg::SetStoreInfo(set_store_info) => { - let set_store_info: SetStoreInfo = set_store_info.clone().into(); - let header = MessageHeader { - kind: MessageKind::SetStoreInfo, - len: set_store_info.encoded_len() as u32, - }; - header.encode(buf)?; - set_store_info.encode(buf)?; - } - LogMsg::ArrowMsg(store_id, arrow_msg) => { - let arrow_msg = ArrowMsg { - store_id: Some(store_id.clone().into()), - compression: match compression { - Compression::Off => proto::Compression::None as i32, - Compression::LZ4 => proto::Compression::Lz4 as i32, - }, - encoding: Encoding::ArrowIpc as i32, - payload: encode_arrow(&arrow_msg.schema, &arrow_msg.chunk, compression)?, - }; - let header = MessageHeader { - kind: MessageKind::ArrowMsg, - len: arrow_msg.encoded_len() as u32, - }; - header.encode(buf)?; - arrow_msg.encode(buf)?; - } - LogMsg::BlueprintActivationCommand(blueprint_activation_command) => { - let blueprint_activation_command: BlueprintActivationCommand = - blueprint_activation_command.clone().into(); - let header = MessageHeader { - kind: MessageKind::BlueprintActivationCommand, - len: blueprint_activation_command.encoded_len() as u32, - }; - header.encode(buf)?; - blueprint_activation_command.encode(buf)?; - } - } - - Ok(()) -} - -pub(crate) fn decode( - data: &mut impl std::io::Read, - compression: Compression, -) -> Result<(u64, Option), DecodeError> { - use re_protos::external::prost::Message; - use re_protos::log_msg::v0::{ArrowMsg, BlueprintActivationCommand, Encoding, SetStoreInfo}; - - let mut read_bytes = 0u64; - let header = MessageHeader::decode(data)?; - read_bytes += std::mem::size_of::() as u64 + header.len as u64; - - let mut buf = vec![0; header.len as usize]; - data.read_exact(&mut buf[..])?; - - let msg = match header.kind { - MessageKind::SetStoreInfo => { - let set_store_info = SetStoreInfo::decode(&buf[..])?; - Some(LogMsg::SetStoreInfo(set_store_info.try_into()?)) - } - MessageKind::ArrowMsg => { - let arrow_msg = ArrowMsg::decode(&buf[..])?; - if arrow_msg.encoding() != Encoding::ArrowIpc { - return Err(DecodeError::Codec(CodecError::UnsupportedEncoding)); - } - - let (schema, chunk) = decode_arrow(&arrow_msg.payload, compression)?; - - let store_id: re_log_types::StoreId = arrow_msg - .store_id - .ok_or_else(|| { - TypeConversionError::missing_field("rerun.log_msg.v0.ArrowMsg", "store_id") - })? - .into(); - - let chunk = re_chunk::Chunk::from_transport(&re_chunk::TransportChunk { - schema, - data: chunk, - })?; - - Some(LogMsg::ArrowMsg(store_id, chunk.to_arrow_msg()?)) - } - MessageKind::BlueprintActivationCommand => { - let blueprint_activation_command = BlueprintActivationCommand::decode(&buf[..])?; - Some(LogMsg::BlueprintActivationCommand( - blueprint_activation_command.try_into()?, - )) - } - MessageKind::End => None, - }; - - Ok((read_bytes, msg)) -} - -fn encode_arrow( - schema: &arrow2::datatypes::Schema, - chunk: &arrow2::chunk::Chunk>, - compression: crate::Compression, -) -> Result, EncodeError> { - let mut uncompressed = Vec::new(); - write_arrow_to_bytes(&mut uncompressed, schema, chunk)?; - - match compression { - crate::Compression::Off => Ok(uncompressed), - crate::Compression::LZ4 => Ok(lz4_flex::block::compress(&uncompressed)), - } -} - -fn decode_arrow( - data: &[u8], - compression: crate::Compression, -) -> Result< - ( - arrow2::datatypes::Schema, - arrow2::chunk::Chunk>, - ), - DecodeError, -> { - let mut uncompressed = Vec::new(); - let data = match compression { - crate::Compression::Off => data, - crate::Compression::LZ4 => { - lz4_flex::block::decompress_into(data, &mut uncompressed)?; - uncompressed.as_slice() - } - }; - - Ok(read_arrow_from_bytes(&mut &data[..])?) -} - -/// Helper function that serializes given arrow schema and record batch into bytes -/// using Arrow IPC format. -fn write_arrow_to_bytes( - writer: &mut W, - schema: &arrow2::datatypes::Schema, - data: &arrow2::chunk::Chunk>, -) -> Result<(), CodecError> { - use arrow2::io::ipc; - - let options = ipc::write::WriteOptions { compression: None }; - let mut sw = ipc::write::StreamWriter::new(writer, options); - - sw.start(schema, None) - .map_err(CodecError::ArrowSerialization)?; - sw.write(data, None) - .map_err(CodecError::ArrowSerialization)?; - sw.finish().map_err(CodecError::ArrowSerialization)?; - - Ok(()) -} +#[cfg(feature = "encoder")] +pub(crate) mod encoder; -/// Helper function that deserializes raw bytes into arrow schema and record batch -/// using Arrow IPC format. -fn read_arrow_from_bytes( - reader: &mut R, -) -> Result< - ( - arrow2::datatypes::Schema, - arrow2::chunk::Chunk>, - ), - CodecError, -> { - use arrow2::io::ipc; - - let metadata = - ipc::read::read_stream_metadata(reader).map_err(CodecError::ArrowSerialization)?; - let mut stream = ipc::read::StreamReader::new(reader, metadata, None); - - let schema = stream.schema().clone(); - // there should be at least one record batch in the stream - let stream_state = stream - .next() - .ok_or(CodecError::MissingRecordBatch)? - .map_err(CodecError::ArrowSerialization)?; - - match stream_state { - ipc::read::StreamState::Waiting => Err(CodecError::UnexpectedStreamState), - ipc::read::StreamState::Some(chunk) => Ok((schema, chunk)), - } -} +#[cfg(feature = "decoder")] +pub(crate) mod decoder; diff --git a/crates/store/re_log_encoding/src/protobuf/decoder.rs b/crates/store/re_log_encoding/src/protobuf/decoder.rs new file mode 100644 index 000000000000..30d2a7b13015 --- /dev/null +++ b/crates/store/re_log_encoding/src/protobuf/decoder.rs @@ -0,0 +1,133 @@ +use super::*; +use crate::decoder::DecodeError; + +impl MessageKind { + pub(crate) fn decode(data: &mut impl std::io::Read) -> Result { + let mut buf = [0; 4]; + data.read_exact(&mut buf)?; + + match u32::from_le_bytes(buf) { + 1 => Ok(Self::SetStoreInfo), + 2 => Ok(Self::ArrowMsg), + 3 => Ok(Self::BlueprintActivationCommand), + 255 => Ok(Self::End), + _ => Err(DecodeError::Codec(CodecError::UnknownMessageHeader)), + } + } +} + +impl MessageHeader { + pub(crate) fn decode(data: &mut impl std::io::Read) -> Result { + let kind = MessageKind::decode(data)?; + let mut buf = [0; 4]; + data.read_exact(&mut buf)?; + let len = u32::from_le_bytes(buf); + + Ok(Self { kind, len }) + } +} + +pub(crate) fn decode( + data: &mut impl std::io::Read, + compression: Compression, +) -> Result<(u64, Option), DecodeError> { + use re_protos::external::prost::Message; + use re_protos::log_msg::v0::{ArrowMsg, BlueprintActivationCommand, Encoding, SetStoreInfo}; + + let mut read_bytes = 0u64; + let header = MessageHeader::decode(data)?; + read_bytes += std::mem::size_of::() as u64 + header.len as u64; + + let mut buf = vec![0; header.len as usize]; + data.read_exact(&mut buf[..])?; + + let msg = match header.kind { + MessageKind::SetStoreInfo => { + let set_store_info = SetStoreInfo::decode(&buf[..])?; + Some(LogMsg::SetStoreInfo(set_store_info.try_into()?)) + } + MessageKind::ArrowMsg => { + let arrow_msg = ArrowMsg::decode(&buf[..])?; + if arrow_msg.encoding() != Encoding::ArrowIpc { + return Err(DecodeError::Codec(CodecError::UnsupportedEncoding)); + } + + let (schema, chunk) = decode_arrow(&arrow_msg.payload, compression)?; + + let store_id: re_log_types::StoreId = arrow_msg + .store_id + .ok_or_else(|| { + TypeConversionError::missing_field("rerun.log_msg.v0.ArrowMsg", "store_id") + })? + .into(); + + let chunk = re_chunk::Chunk::from_transport(&re_chunk::TransportChunk { + schema, + data: chunk, + })?; + + Some(LogMsg::ArrowMsg(store_id, chunk.to_arrow_msg()?)) + } + MessageKind::BlueprintActivationCommand => { + let blueprint_activation_command = BlueprintActivationCommand::decode(&buf[..])?; + Some(LogMsg::BlueprintActivationCommand( + blueprint_activation_command.try_into()?, + )) + } + MessageKind::End => None, + }; + + Ok((read_bytes, msg)) +} + +fn decode_arrow( + data: &[u8], + compression: crate::Compression, +) -> Result< + ( + arrow2::datatypes::Schema, + arrow2::chunk::Chunk>, + ), + DecodeError, +> { + let mut uncompressed = Vec::new(); + let data = match compression { + crate::Compression::Off => data, + crate::Compression::LZ4 => { + lz4_flex::block::decompress_into(data, &mut uncompressed)?; + uncompressed.as_slice() + } + }; + + Ok(read_arrow_from_bytes(&mut &data[..])?) +} + +/// Helper function that deserializes raw bytes into arrow schema and record batch +/// using Arrow IPC format. +fn read_arrow_from_bytes( + reader: &mut R, +) -> Result< + ( + arrow2::datatypes::Schema, + arrow2::chunk::Chunk>, + ), + CodecError, +> { + use arrow2::io::ipc; + + let metadata = + ipc::read::read_stream_metadata(reader).map_err(CodecError::ArrowSerialization)?; + let mut stream = ipc::read::StreamReader::new(reader, metadata, None); + + let schema = stream.schema().clone(); + // there should be at least one record batch in the stream + let stream_state = stream + .next() + .ok_or(CodecError::MissingRecordBatch)? + .map_err(CodecError::ArrowSerialization)?; + + match stream_state { + ipc::read::StreamState::Waiting => Err(CodecError::UnexpectedStreamState), + ipc::read::StreamState::Some(chunk) => Ok((schema, chunk)), + } +} diff --git a/crates/store/re_log_encoding/src/protobuf/encoder.rs b/crates/store/re_log_encoding/src/protobuf/encoder.rs new file mode 100644 index 000000000000..340949e7f6c2 --- /dev/null +++ b/crates/store/re_log_encoding/src/protobuf/encoder.rs @@ -0,0 +1,105 @@ +use super::*; +use crate::encoder::EncodeError; + +impl MessageKind { + pub(crate) fn encode(&self, buf: &mut impl std::io::Write) -> Result<(), EncodeError> { + let kind: u32 = *self as u32; + buf.write_all(&kind.to_le_bytes())?; + Ok(()) + } +} + +impl MessageHeader { + pub(crate) fn encode(&self, buf: &mut impl std::io::Write) -> Result<(), EncodeError> { + self.kind.encode(buf)?; + buf.write_all(&self.len.to_le_bytes())?; + Ok(()) + } +} + +pub(crate) fn encode( + buf: &mut Vec, + message: &LogMsg, + compression: Compression, +) -> Result<(), EncodeError> { + use re_protos::external::prost::Message; + use re_protos::log_msg::v0::{ + self as proto, ArrowMsg, BlueprintActivationCommand, Encoding, SetStoreInfo, + }; + + match message { + LogMsg::SetStoreInfo(set_store_info) => { + let set_store_info: SetStoreInfo = set_store_info.clone().into(); + let header = MessageHeader { + kind: MessageKind::SetStoreInfo, + len: set_store_info.encoded_len() as u32, + }; + header.encode(buf)?; + set_store_info.encode(buf)?; + } + LogMsg::ArrowMsg(store_id, arrow_msg) => { + let arrow_msg = ArrowMsg { + store_id: Some(store_id.clone().into()), + compression: match compression { + Compression::Off => proto::Compression::None as i32, + Compression::LZ4 => proto::Compression::Lz4 as i32, + }, + encoding: Encoding::ArrowIpc as i32, + payload: encode_arrow(&arrow_msg.schema, &arrow_msg.chunk, compression)?, + }; + let header = MessageHeader { + kind: MessageKind::ArrowMsg, + len: arrow_msg.encoded_len() as u32, + }; + header.encode(buf)?; + arrow_msg.encode(buf)?; + } + LogMsg::BlueprintActivationCommand(blueprint_activation_command) => { + let blueprint_activation_command: BlueprintActivationCommand = + blueprint_activation_command.clone().into(); + let header = MessageHeader { + kind: MessageKind::BlueprintActivationCommand, + len: blueprint_activation_command.encoded_len() as u32, + }; + header.encode(buf)?; + blueprint_activation_command.encode(buf)?; + } + } + + Ok(()) +} + +fn encode_arrow( + schema: &arrow2::datatypes::Schema, + chunk: &arrow2::chunk::Chunk>, + compression: crate::Compression, +) -> Result, EncodeError> { + let mut uncompressed = Vec::new(); + write_arrow_to_bytes(&mut uncompressed, schema, chunk)?; + + match compression { + crate::Compression::Off => Ok(uncompressed), + crate::Compression::LZ4 => Ok(lz4_flex::block::compress(&uncompressed)), + } +} + +/// Helper function that serializes given arrow schema and record batch into bytes +/// using Arrow IPC format. +fn write_arrow_to_bytes( + writer: &mut W, + schema: &arrow2::datatypes::Schema, + data: &arrow2::chunk::Chunk>, +) -> Result<(), CodecError> { + use arrow2::io::ipc; + + let options = ipc::write::WriteOptions { compression: None }; + let mut sw = ipc::write::StreamWriter::new(writer, options); + + sw.start(schema, None) + .map_err(CodecError::ArrowSerialization)?; + sw.write(data, None) + .map_err(CodecError::ArrowSerialization)?; + sw.finish().map_err(CodecError::ArrowSerialization)?; + + Ok(()) +} From 7a720f8ceffff59244f56fc784f92b002ac51217 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 19:52:45 +0100 Subject: [PATCH 12/47] fix check --- crates/top/rerun/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/top/rerun/src/lib.rs b/crates/top/rerun/src/lib.rs index 0a42e8232d63..4627f810d007 100644 --- a/crates/top/rerun/src/lib.rs +++ b/crates/top/rerun/src/lib.rs @@ -147,7 +147,6 @@ pub use re_entity_db::external::re_chunk_store::{ ChunkStore, ChunkStoreConfig, ChunkStoreDiff, ChunkStoreDiffKind, ChunkStoreEvent, ChunkStoreGeneration, ChunkStoreHandle, ChunkStoreSubscriber, }; -pub use re_log_encoding::VersionPolicy; pub use re_log_types::StoreKind; /// To register a new external data loader, simply add an executable in your $PATH whose name From a2d7ea4d36ea1e628c43dc1efe4c837ff1e7b285 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 6 Dec 2024 21:57:16 +0100 Subject: [PATCH 13/47] fix more lints --- crates/store/re_chunk_store/src/lib.rs | 2 +- .../store/re_data_loader/src/loader_external.rs | 2 +- crates/store/re_data_loader/src/loader_rrd.rs | 8 ++++---- crates/store/re_data_source/src/load_stdin.rs | 2 +- .../store/re_entity_db/examples/memory_usage.rs | 2 +- crates/store/re_entity_db/src/store_bundle.rs | 2 +- .../benches/msg_encode_benchmark.rs | 2 +- crates/store/re_log_encoding/src/decoder/mod.rs | 15 +-------------- crates/store/re_log_encoding/src/lib.rs | 16 ++++++++++++++-- crates/store/re_log_encoding/src/protobuf.rs | 8 ++------ .../re_log_encoding/src/protobuf/decoder.rs | 5 ++++- .../re_log_encoding/src/protobuf/encoder.rs | 5 ++++- .../re_log_encoding/src/stream_rrd_from_http.rs | 4 ++-- crates/store/re_sdk_comms/src/server.rs | 2 +- crates/top/rerun/Cargo.toml | 8 +++----- crates/top/rerun/src/commands/rrd/compare.rs | 2 +- crates/top/rerun/src/commands/rrd/filter.rs | 2 +- .../top/rerun/src/commands/rrd/merge_compact.rs | 2 +- crates/top/rerun/src/commands/rrd/print.rs | 2 +- crates/top/rerun/src/commands/stdio.rs | 2 +- crates/top/rerun/src/lib.rs | 2 ++ crates/viewer/re_viewer/src/loading.rs | 2 +- 22 files changed, 49 insertions(+), 48 deletions(-) diff --git a/crates/store/re_chunk_store/src/lib.rs b/crates/store/re_chunk_store/src/lib.rs index 9ed27a730fe9..eb07efafdf0a 100644 --- a/crates/store/re_chunk_store/src/lib.rs +++ b/crates/store/re_chunk_store/src/lib.rs @@ -50,7 +50,7 @@ pub use re_chunk::{ UnitChunkShared, }; // #[doc(no_inline)] -// pub use re_log_encoding::decoder::VersionPolicy; +// pub use re_log_encoding::VersionPolicy; #[doc(no_inline)] pub use re_log_types::{ResolvedTimeRange, TimeInt, TimeType, Timeline}; diff --git a/crates/store/re_data_loader/src/loader_external.rs b/crates/store/re_data_loader/src/loader_external.rs index be4dacea5e61..51e4ac10f1d9 100644 --- a/crates/store/re_data_loader/src/loader_external.rs +++ b/crates/store/re_data_loader/src/loader_external.rs @@ -182,7 +182,7 @@ impl crate::DataLoader for ExternalLoader { // streaming data to stdout. let is_sending_data = Arc::new(AtomicBool::new(false)); - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let stdout = std::io::BufReader::new(stdout); match re_log_encoding::decoder::Decoder::new(version_policy, stdout) { Ok(decoder) => { diff --git a/crates/store/re_data_loader/src/loader_rrd.rs b/crates/store/re_data_loader/src/loader_rrd.rs index a7df3b51acb7..2c4157f08afd 100644 --- a/crates/store/re_data_loader/src/loader_rrd.rs +++ b/crates/store/re_data_loader/src/loader_rrd.rs @@ -40,7 +40,7 @@ impl crate::DataLoader for RrdLoader { "Loading rrd data from filesystem…", ); - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; match extension.as_str() { "rbl" => { @@ -118,7 +118,7 @@ impl crate::DataLoader for RrdLoader { return Err(crate::DataLoaderError::Incompatible(filepath)); } - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let contents = std::io::Cursor::new(contents); let decoder = match re_log_encoding::decoder::Decoder::new(version_policy, contents) { Ok(decoder) => decoder, @@ -308,7 +308,7 @@ impl RetryableFileReader { mod tests { use re_build_info::CrateVersion; use re_chunk::RowId; - use re_log_encoding::{decoder, encoder::DroppableEncoder}; + use re_log_encoding::{encoder::DroppableEncoder, VersionPolicy}; use re_log_types::{ ApplicationId, LogMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; @@ -372,7 +372,7 @@ mod tests { encoder.flush_blocking().expect("failed to flush messages"); let reader = RetryableFileReader::new(&rrd_file_path).unwrap(); - let mut decoder = Decoder::new(decoder::VersionPolicy::Warn, reader).unwrap(); + let mut decoder = Decoder::new(VersionPolicy::Warn, reader).unwrap(); // we should be able to read 5 messages that we wrote let decoded_messages = (0..5) diff --git a/crates/store/re_data_source/src/load_stdin.rs b/crates/store/re_data_source/src/load_stdin.rs index 0573c20750ae..1c4f5ac437f5 100644 --- a/crates/store/re_data_source/src/load_stdin.rs +++ b/crates/store/re_data_source/src/load_stdin.rs @@ -6,7 +6,7 @@ use re_smart_channel::Sender; /// This fails synchronously iff the standard input stream could not be opened, otherwise errors /// are handled asynchronously (as in: they're logged). pub fn load_stdin(tx: Sender) -> anyhow::Result<()> { - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let stdin = std::io::BufReader::new(std::io::stdin()); let decoder = re_log_encoding::decoder::Decoder::new_concatenated(version_policy, stdin)?; diff --git a/crates/store/re_entity_db/examples/memory_usage.rs b/crates/store/re_entity_db/examples/memory_usage.rs index 7c4c524dc10e..585b94a6189c 100644 --- a/crates/store/re_entity_db/examples/memory_usage.rs +++ b/crates/store/re_entity_db/examples/memory_usage.rs @@ -78,7 +78,7 @@ fn log_messages() { } fn decode_log_msg(mut bytes: &[u8]) -> LogMsg { - let version_policy = re_log_encoding::decoder::VersionPolicy::Error; + let version_policy = re_log_encoding::VersionPolicy::Error; let mut messages = re_log_encoding::decoder::Decoder::new(version_policy, &mut bytes) .unwrap() .collect::, _>>() diff --git a/crates/store/re_entity_db/src/store_bundle.rs b/crates/store/re_entity_db/src/store_bundle.rs index 45fa833e02ee..a6645d606c45 100644 --- a/crates/store/re_entity_db/src/store_bundle.rs +++ b/crates/store/re_entity_db/src/store_bundle.rs @@ -1,7 +1,7 @@ use itertools::Itertools as _; use crate::EntityDb; -use re_log_encoding::decoder::VersionPolicy; +use re_log_encoding::VersionPolicy; use re_log_types::{StoreId, StoreKind}; #[derive(thiserror::Error, Debug)] diff --git a/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs b/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs index d2455b765530..1b73e51dc4b5 100644 --- a/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs +++ b/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs @@ -46,7 +46,7 @@ fn encode_log_msgs(messages: &[LogMsg]) -> Vec { } fn decode_log_msgs(mut bytes: &[u8]) -> Vec { - let version_policy = re_log_encoding::decoder::VersionPolicy::Error; + let version_policy = re_log_encoding::VersionPolicy::Error; let messages = re_log_encoding::decoder::Decoder::new(version_policy, &mut bytes) .unwrap() .collect::, _>>() diff --git a/crates/store/re_log_encoding/src/decoder/mod.rs b/crates/store/re_log_encoding/src/decoder/mod.rs index f6cd36578d12..7fe791920cfa 100644 --- a/crates/store/re_log_encoding/src/decoder/mod.rs +++ b/crates/store/re_log_encoding/src/decoder/mod.rs @@ -11,25 +11,12 @@ use re_log_types::LogMsg; use crate::codec; use crate::FileHeader; use crate::MessageHeader; +use crate::VersionPolicy; use crate::OLD_RRD_HEADERS; use crate::{Compression, EncodingOptions, Serializer}; // ---------------------------------------------------------------------------- -/// How to handle version mismatches during decoding. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum VersionPolicy { - /// Warn if the versions don't match, but continue loading. - /// - /// We usually use this for loading `.rrd` recordings. - Warn, - - /// Return [`DecodeError::IncompatibleRerunVersion`] if the versions aren't compatible. - /// - /// We usually use this for tests, and for loading `.rbl` blueprint files. - Error, -} - fn warn_on_version_mismatch( version_policy: VersionPolicy, encoded_version: [u8; 4], diff --git a/crates/store/re_log_encoding/src/lib.rs b/crates/store/re_log_encoding/src/lib.rs index 54bb4396744f..6a70e578f58c 100644 --- a/crates/store/re_log_encoding/src/lib.rs +++ b/crates/store/re_log_encoding/src/lib.rs @@ -2,8 +2,6 @@ #[cfg(feature = "decoder")] pub mod decoder; -#[cfg(feature = "decoder")] -pub use decoder::VersionPolicy; #[cfg(feature = "encoder")] pub mod encoder; @@ -19,6 +17,20 @@ mod file_sink; #[cfg(feature = "stream_from_http")] pub mod stream_rrd_from_http; +/// How to handle version mismatches during decoding. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VersionPolicy { + /// Warn if the versions don't match, but continue loading. + /// + /// We usually use this for loading `.rrd` recordings. + Warn, + + /// Return an error if the versions aren't compatible. + /// + /// We usually use this for tests, and for loading `.rbl` blueprint files. + Error, +} + // --------------------------------------------------------------------- #[cfg(feature = "encoder")] diff --git a/crates/store/re_log_encoding/src/protobuf.rs b/crates/store/re_log_encoding/src/protobuf.rs index cd1e2990b670..13be885e791c 100644 --- a/crates/store/re_log_encoding/src/protobuf.rs +++ b/crates/store/re_log_encoding/src/protobuf.rs @@ -1,9 +1,4 @@ -use re_log_types::LogMsg; -use re_protos::TypeConversionError; - -use crate::codec::CodecError; -use crate::Compression; - +#[allow(dead_code)] // used in encoder/decoder behind feature flag #[derive(Debug, Clone, Copy)] #[repr(u32)] pub(crate) enum MessageKind { @@ -13,6 +8,7 @@ pub(crate) enum MessageKind { End = 255, } +#[allow(dead_code)] // used in encoder/decoder behind feature flag #[derive(Debug, Clone, Copy)] pub(crate) struct MessageHeader { pub(crate) kind: MessageKind, diff --git a/crates/store/re_log_encoding/src/protobuf/decoder.rs b/crates/store/re_log_encoding/src/protobuf/decoder.rs index 30d2a7b13015..9a73bd54ed72 100644 --- a/crates/store/re_log_encoding/src/protobuf/decoder.rs +++ b/crates/store/re_log_encoding/src/protobuf/decoder.rs @@ -1,5 +1,8 @@ -use super::*; +use super::{MessageHeader, MessageKind}; use crate::decoder::DecodeError; +use crate::{codec::CodecError, Compression}; +use re_log_types::LogMsg; +use re_protos::TypeConversionError; impl MessageKind { pub(crate) fn decode(data: &mut impl std::io::Read) -> Result { diff --git a/crates/store/re_log_encoding/src/protobuf/encoder.rs b/crates/store/re_log_encoding/src/protobuf/encoder.rs index 340949e7f6c2..22e967ea533a 100644 --- a/crates/store/re_log_encoding/src/protobuf/encoder.rs +++ b/crates/store/re_log_encoding/src/protobuf/encoder.rs @@ -1,5 +1,8 @@ -use super::*; +use super::{MessageHeader, MessageKind}; +use crate::codec::CodecError; use crate::encoder::EncodeError; +use crate::Compression; +use re_log_types::LogMsg; impl MessageKind { pub(crate) fn encode(&self, buf: &mut impl std::io::Write) -> Result<(), EncodeError> { diff --git a/crates/store/re_log_encoding/src/stream_rrd_from_http.rs b/crates/store/re_log_encoding/src/stream_rrd_from_http.rs index 718a8f8bb479..a52283c4fb05 100644 --- a/crates/store/re_log_encoding/src/stream_rrd_from_http.rs +++ b/crates/store/re_log_encoding/src/stream_rrd_from_http.rs @@ -71,7 +71,7 @@ pub fn stream_rrd_from_http(url: String, on_msg: Arc) { re_log::debug!("Downloading .rrd file from {url:?}…"); ehttp::streaming::fetch(ehttp::Request::get(&url), { - let version_policy = crate::decoder::VersionPolicy::Warn; + let version_policy = crate::VersionPolicy::Warn; let decoder = RefCell::new(StreamDecoder::new(version_policy)); move |part| match part { Ok(part) => match part { @@ -184,7 +184,7 @@ pub mod web_decode { async fn decode_rrd_async(rrd_bytes: Vec, on_msg: Arc) { let mut last_yield = web_time::Instant::now(); - let version_policy = crate::decoder::VersionPolicy::Warn; + let version_policy = crate::VersionPolicy::Warn; match crate::decoder::Decoder::new(version_policy, rrd_bytes.as_slice()) { Ok(decoder) => { for msg in decoder { diff --git a/crates/store/re_sdk_comms/src/server.rs b/crates/store/re_sdk_comms/src/server.rs index fc7ceea9037c..48a1c381c700 100644 --- a/crates/store/re_sdk_comms/src/server.rs +++ b/crates/store/re_sdk_comms/src/server.rs @@ -238,7 +238,7 @@ fn run_client( congestion_manager.register_latency(tx.latency_sec()); - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; for msg in re_log_encoding::decoder::decode_bytes(version_policy, &packet)? { if congestion_manager.should_send(&msg) { tx.send(msg)?; diff --git a/crates/top/rerun/Cargo.toml b/crates/top/rerun/Cargo.toml index 068c98028fdd..d9739ec159f8 100644 --- a/crates/top/rerun/Cargo.toml +++ b/crates/top/rerun/Cargo.toml @@ -99,7 +99,8 @@ run = [ "unindent", "dep:re_chunk_store", "dep:re_data_source", - "dep:re_log_encoding", + "re_log_encoding/encoder", + "re_log_encoding/decoder", "dep:re_sdk_comms", "dep:re_ws_comms", ] @@ -128,6 +129,7 @@ re_crash_handler.workspace = true re_entity_db.workspace = true re_error.workspace = true re_format.workspace = true +re_log_encoding.workspace = true re_log_types.workspace = true re_video.workspace = true re_log.workspace = true @@ -145,10 +147,6 @@ re_analytics = { workspace = true, optional = true } re_chunk_store = { workspace = true, optional = true } re_data_source = { workspace = true, optional = true } re_dataframe = { workspace = true, optional = true } -re_log_encoding = { workspace = true, optional = true, features = [ - "decoder", - "encoder", -] } re_sdk = { workspace = true, optional = true } re_sdk_comms = { workspace = true, optional = true } re_types = { workspace = true, optional = true } diff --git a/crates/top/rerun/src/commands/rrd/compare.rs b/crates/top/rerun/src/commands/rrd/compare.rs index cd2c87b5aa09..3472eacd8f90 100644 --- a/crates/top/rerun/src/commands/rrd/compare.rs +++ b/crates/top/rerun/src/commands/rrd/compare.rs @@ -93,7 +93,7 @@ fn compute_uber_table( let rrd_file = std::io::BufReader::new(rrd_file); let mut stores: std::collections::HashMap = Default::default(); - let version_policy = re_log_encoding::decoder::VersionPolicy::Error; + let version_policy = re_log_encoding::VersionPolicy::Error; let decoder = re_log_encoding::decoder::Decoder::new(version_policy, rrd_file)?; for msg in decoder { let msg = msg.context("decode rrd message")?; diff --git a/crates/top/rerun/src/commands/rrd/filter.rs b/crates/top/rerun/src/commands/rrd/filter.rs index 53efe7cf9d4e..f3334d2f9328 100644 --- a/crates/top/rerun/src/commands/rrd/filter.rs +++ b/crates/top/rerun/src/commands/rrd/filter.rs @@ -60,7 +60,7 @@ impl FilterCommand { .collect(); // TODO(cmc): might want to make this configurable at some point. - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let (rx_decoder, rx_size_bytes) = read_rrd_streams_from_file_or_stdin(version_policy, path_to_input_rrds); diff --git a/crates/top/rerun/src/commands/rrd/merge_compact.rs b/crates/top/rerun/src/commands/rrd/merge_compact.rs index 9aa434f84b1a..16911fa92510 100644 --- a/crates/top/rerun/src/commands/rrd/merge_compact.rs +++ b/crates/top/rerun/src/commands/rrd/merge_compact.rs @@ -157,7 +157,7 @@ fn merge_and_compact( ); // TODO(cmc): might want to make this configurable at some point. - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let (rx, rx_size_bytes) = read_rrd_streams_from_file_or_stdin(version_policy, path_to_input_rrds); diff --git a/crates/top/rerun/src/commands/rrd/print.rs b/crates/top/rerun/src/commands/rrd/print.rs index e1ec9ab64588..d6ff0b52a31b 100644 --- a/crates/top/rerun/src/commands/rrd/print.rs +++ b/crates/top/rerun/src/commands/rrd/print.rs @@ -32,7 +32,7 @@ impl PrintCommand { } = self; // TODO(cmc): might want to make this configurable at some point. - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let (rx, _) = read_rrd_streams_from_file_or_stdin(version_policy, path_to_input_rrds); for res in rx { diff --git a/crates/top/rerun/src/commands/stdio.rs b/crates/top/rerun/src/commands/stdio.rs index 98599dd7b5eb..e34a67f155c6 100644 --- a/crates/top/rerun/src/commands/stdio.rs +++ b/crates/top/rerun/src/commands/stdio.rs @@ -23,7 +23,7 @@ use re_log_types::LogMsg; /// /// This function is capable of decoding multiple independent recordings from a single stream. pub fn read_rrd_streams_from_file_or_stdin( - version_policy: re_log_encoding::decoder::VersionPolicy, + version_policy: re_log_encoding::VersionPolicy, paths: &[String], ) -> ( channel::Receiver>, diff --git a/crates/top/rerun/src/lib.rs b/crates/top/rerun/src/lib.rs index 4627f810d007..2f6dbf8f1c8b 100644 --- a/crates/top/rerun/src/lib.rs +++ b/crates/top/rerun/src/lib.rs @@ -133,6 +133,8 @@ pub use log_integration::Logger; #[cfg(feature = "run")] pub use commands::{run, CallSource}; +pub use re_log_encoding::VersionPolicy; + #[cfg(feature = "sdk")] pub use sdk::*; diff --git a/crates/viewer/re_viewer/src/loading.rs b/crates/viewer/re_viewer/src/loading.rs index a6e2c5317718..476e3d969afb 100644 --- a/crates/viewer/re_viewer/src/loading.rs +++ b/crates/viewer/re_viewer/src/loading.rs @@ -25,7 +25,7 @@ pub fn load_blueprint_file( // Blueprint files change often. Be strict about the version, and then ignore any errors. // See https://github.com/rerun-io/rerun/issues/2830 - let version_policy = re_log_encoding::decoder::VersionPolicy::Error; + let version_policy = re_log_encoding::VersionPolicy::Error; Ok(StoreBundle::from_rrd(version_policy, file)?) } From 0d3ec408581b08d0496f58d7aa290dfa1ac70bbe Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 10 Dec 2024 11:42:06 +0100 Subject: [PATCH 14/47] update lockfile --- Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index dd7c54947f9c..749b50158287 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6004,7 +6004,6 @@ dependencies = [ "bytemuck", "clean-path", "criterion", - "crossbeam", "document-features", "fixed", "half", From 5d9051b3f5fc7cdc6fbe679d02f03fab29d2344e Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 10 Dec 2024 14:46:16 +0100 Subject: [PATCH 15/47] rename `OUTPUT_V0_RUST` --- .../src/bin/build_re_remote_store_types.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs b/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs index dc2e15c4d23b..d1bb11fc2a98 100644 --- a/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs +++ b/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs @@ -1,6 +1,6 @@ //! This binary runs the remote store gRPC service codegen manually. //! -//! It is easiest to call this using `pixi run codegen-protos`, +//! It is easiest to call this using `pixi run codegen-protos `, //! which will set up the necessary tools. #![allow(clippy::unwrap_used)] @@ -9,7 +9,7 @@ use camino::Utf8Path; const PROTOS_DIR: &str = "crates/store/re_protos/proto"; const INPUT_V0: &[&str] = &["rerun/v0/remote_store.proto", "rerun/v0/log_msg.proto"]; -const OUTPUT_V0_RUST: &str = "crates/store/re_protos/src/v0"; +const OUTPUT_V0_RUST_DIR: &str = "crates/store/re_protos/src/v0"; fn main() { re_log::setup_logging(); @@ -27,7 +27,7 @@ fn main() { ); let definitions_dir_path = workspace_dir.join(PROTOS_DIR); - let rust_generated_output_dir_path = workspace_dir.join(OUTPUT_V0_RUST); + let rust_generated_output_dir_path = workspace_dir.join(OUTPUT_V0_RUST_DIR); re_log::info!( definitions=?definitions_dir_path, From 84977138acf60f7f6eb2853e572584e30b910c56 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 10 Dec 2024 14:46:39 +0100 Subject: [PATCH 16/47] remove comments --- crates/store/re_chunk_store/src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/store/re_chunk_store/src/lib.rs b/crates/store/re_chunk_store/src/lib.rs index eb07efafdf0a..ef7fadd6e327 100644 --- a/crates/store/re_chunk_store/src/lib.rs +++ b/crates/store/re_chunk_store/src/lib.rs @@ -49,8 +49,7 @@ pub use re_chunk::{ Chunk, ChunkId, ChunkShared, LatestAtQuery, RangeQuery, RangeQueryOptions, RowId, UnitChunkShared, }; -// #[doc(no_inline)] -// pub use re_log_encoding::VersionPolicy; + #[doc(no_inline)] pub use re_log_types::{ResolvedTimeRange, TimeInt, TimeType, Timeline}; @@ -58,7 +57,6 @@ pub mod external { pub use arrow2; pub use re_chunk; - // pub use re_log_encoding; } // --- From 187bcb2a23856dbb85de63f16cb9203ec3d2b175 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 10 Dec 2024 14:47:57 +0100 Subject: [PATCH 17/47] rm dead code --- crates/store/re_log_encoding/src/decoder/mod.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/crates/store/re_log_encoding/src/decoder/mod.rs b/crates/store/re_log_encoding/src/decoder/mod.rs index 7fe791920cfa..23fab4fe3663 100644 --- a/crates/store/re_log_encoding/src/decoder/mod.rs +++ b/crates/store/re_log_encoding/src/decoder/mod.rs @@ -410,21 +410,6 @@ mod tests { ApplicationId, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; - // useful for debugging reads in the absence of a functional debugger - /* - struct PrintReader { - inner: R, - } - - impl std::io::Read for PrintReader { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - let read = self.inner.read(buf)?; - println!("read {} bytes: {:?}", read, &buf[..read]); - Ok(read) - } - } - */ - fn fake_log_message() -> LogMsg { LogMsg::SetStoreInfo(SetStoreInfo { row_id: *RowId::new(), From 8b8be710575d030120aed47e4fdbed8b7469e511 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 10 Dec 2024 17:28:39 +0100 Subject: [PATCH 18/47] docs --- .../re_protos/proto/rerun/v0/log_msg.proto | 93 +++++++++++++++++-- .../re_protos/src/v0/rerun.log_msg.v0.rs | 69 +++++++++++++- 2 files changed, 147 insertions(+), 15 deletions(-) diff --git a/crates/store/re_protos/proto/rerun/v0/log_msg.proto b/crates/store/re_protos/proto/rerun/v0/log_msg.proto index 1babb3312423..2accf5904da1 100644 --- a/crates/store/re_protos/proto/rerun/v0/log_msg.proto +++ b/crates/store/re_protos/proto/rerun/v0/log_msg.proto @@ -4,40 +4,65 @@ package rerun.log_msg.v0; import "rerun/v0/common.proto"; -// TODO(jan): documentation here - +// Corresponds to `LogMsg::SetStoreInfo`. Used to identify a recording. message SetStoreInfo { + // A time-based UID that is used to determine how a `StoreInfo` fits in the global ordering of events. rerun.common.v0.Tuid row_id = 1; + // The new store info. StoreInfo info = 2; } +// The type of compression used on the payload. enum Compression { + // No compression. NONE = 0; + + // LZ4 block compression. LZ4 = 1; } +// The encoding of the mesage payload. enum Encoding { + // We don't know what encoding the payload is in. UNKNOWN = 0; + + // The payload is encoded as Arrow-IPC. ARROW_IPC = 1; } +// Corresponds to `LogMsg::ArrowMsg`. Used to transmit actual data. message ArrowMsg { + // The ID of the store that this message is for. rerun.common.v0.StoreId store_id = 1; + + // Compression algorithm used. Compression compression = 2; + + // Encoding of the payload. Encoding encoding = 3; - // Arrow-IPC encoded schema and chunk, - // compressed according to `compression` + // Arrow-IPC encoded schema and chunk, compressed according to the `compression` field. bytes payload = 1000; } +// Corresponds to `LogMsg::BlueprintActivationCommand`. +// +// Used for activating a blueprint once it has been fully transmitted, +// because showing a blueprint before it is fully transmitted can lead to +// a confusing user experience, or inconsistent results due to heuristics. message BlueprintActivationCommand { + // The ID of the blueprint to activate. rerun.common.v0.StoreId blueprint_id = 1; + + // Whether to make the blueprint active immediately. bool make_active = 2; + + // Whether to make the blueprint the default. bool make_default = 3; } +// Information about a recording or blueprint. message StoreInfo { // User-chosen name of the application doing the logging. rerun.common.v0.ApplicationId application_id = 1; @@ -51,55 +76,103 @@ message StoreInfo { // When the recording started. rerun.common.v0.Time started = 4; + // Where the recording came from. StoreSource store_source = 5; } +// The source of a recording or blueprint. message StoreSource { // Determines what is encoded in `extra`. StoreSourceKind kind = 1; + + // Store source payload. See `StoreSourceKind` for what exactly is encoded here. StoreSourceExtra extra = 2; } +// A newtype for `StoreSource` payload. +// +// This exists to that we can implement conversions on the newtype for convenience. +message StoreSourceExtra { + bytes payload = 1; +} + +// What kind of source a recording comes from. enum StoreSourceKind { + // We don't know anything about the source of this recording. + // // `extra` is unused. UNKNOWN_KIND = 0; + + // The recording came from the C++ SDK. + // // `extra` is unused. C_SDK = 1; + + // The recording came from the Python SDK. + // // `extra` is `PythonVersion`. PYTHON_SDK = 2; + + // The recording came from the Rust SDK. + // // `extra` is `CrateInfo`. RUST_SDK = 3; + + // The recording came from a file. + // // `extra` is `FileSource`. FILE = 4; + + // The recording came from some action in the viewer. + // // `extra` is unused. VIEWER = 5; + + // The recording came from some other source. + // // `extra` is a string. OTHER = 6; } -message StoreSourceExtra { - bytes payload = 1; -} - +// Version of the Python SDK that created the recording. message PythonVersion { - // [u8; major, minor, patch, ..suffix] - bytes version = 1; + sint32 major = 1; + sint32 minor = 2; + sint32 patch = 3; + string suffix = 4; } +// Information about the Rust SDK that created the recording. message CrateInfo { + // Version of the Rust compiler used to compile the SDK. string rustc_version = 1; + + // Version of LLVM used by the Rust compiler. string llvm_version = 2; } +// A recording which came from a file. message FileSource { FileSourceKind kind = 1; } +// Determines where the file came from. enum FileSourceKind { + // We don't know where the file came from. UNKNOWN_SOURCE = 0; + + // The file came from the command line. CLI = 1; + + // The file was served over HTTP. URI = 2; + + // The file was dragged into the viewer. DRAG_AND_DROP = 3; + + // The file was opened using a file dialog. FILE_DIALOG = 4; + + // The recording was produced using a data loader, such as when logging a mesh file. SDK = 5; } diff --git a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs index 096b2926f720..a4946bb35557 100644 --- a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs @@ -1,33 +1,48 @@ // This file is @generated by prost-build. +/// Corresponds to `LogMsg::SetStoreInfo`. Used to identify a recording. #[derive(Clone, PartialEq, ::prost::Message)] pub struct SetStoreInfo { + /// A time-based UID that is used to determine how a `StoreInfo` fits in the global ordering of events. #[prost(message, optional, tag = "1")] pub row_id: ::core::option::Option, + /// The new store info. #[prost(message, optional, tag = "2")] pub info: ::core::option::Option, } +/// Corresponds to `LogMsg::ArrowMsg`. Used to transmit actual data. #[derive(Clone, PartialEq, ::prost::Message)] pub struct ArrowMsg { + /// The ID of the store that this message is for. #[prost(message, optional, tag = "1")] pub store_id: ::core::option::Option, + /// Compression algorithm used. #[prost(enumeration = "Compression", tag = "2")] pub compression: i32, + /// Encoding of the payload. #[prost(enumeration = "Encoding", tag = "3")] pub encoding: i32, - /// Arrow-IPC encoded schema and chunk, - /// compressed according to `compression` + /// Arrow-IPC encoded schema and chunk, compressed according to the `compression` field. #[prost(bytes = "vec", tag = "1000")] pub payload: ::prost::alloc::vec::Vec, } +/// Corresponds to `LogMsg::BlueprintActivationCommand`. +/// +/// Used for activating a blueprint once it has been fully transmitted, +/// because showing a blueprint before it is fully transmitted can lead to +/// a confusing user experience, or inconsistent results due to heuristics. #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlueprintActivationCommand { + /// The ID of the blueprint to activate. #[prost(message, optional, tag = "1")] pub blueprint_id: ::core::option::Option, + /// Whether to make the blueprint active immediately. #[prost(bool, tag = "2")] pub make_active: bool, + /// Whether to make the blueprint the default. #[prost(bool, tag = "3")] pub make_default: bool, } +/// Information about a recording or blueprint. #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreInfo { /// User-chosen name of the application doing the logging. @@ -42,44 +57,63 @@ pub struct StoreInfo { /// When the recording started. #[prost(message, optional, tag = "4")] pub started: ::core::option::Option, + /// Where the recording came from. #[prost(message, optional, tag = "5")] pub store_source: ::core::option::Option, } +/// The source of a recording or blueprint. #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreSource { /// Determines what is encoded in `extra`. #[prost(enumeration = "StoreSourceKind", tag = "1")] pub kind: i32, + /// Store source payload. See `StoreSourceKind` for what exactly is encoded here. #[prost(message, optional, tag = "2")] pub extra: ::core::option::Option, } +/// A newtype for `StoreSource` payload. +/// +/// This exists to that we can implement conversions on the newtype for convenience. #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreSourceExtra { #[prost(bytes = "vec", tag = "1")] pub payload: ::prost::alloc::vec::Vec, } +/// Version of the Python SDK that created the recording. #[derive(Clone, PartialEq, ::prost::Message)] pub struct PythonVersion { - /// \[u8; major, minor, patch, ..suffix\] - #[prost(bytes = "vec", tag = "1")] - pub version: ::prost::alloc::vec::Vec, + #[prost(sint32, tag = "1")] + pub major: i32, + #[prost(sint32, tag = "2")] + pub minor: i32, + #[prost(sint32, tag = "3")] + pub patch: i32, + #[prost(string, tag = "4")] + pub suffix: ::prost::alloc::string::String, } +/// Information about the Rust SDK that created the recording. #[derive(Clone, PartialEq, ::prost::Message)] pub struct CrateInfo { + /// Version of the Rust compiler used to compile the SDK. #[prost(string, tag = "1")] pub rustc_version: ::prost::alloc::string::String, + /// Version of LLVM used by the Rust compiler. #[prost(string, tag = "2")] pub llvm_version: ::prost::alloc::string::String, } +/// A recording which came from a file. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct FileSource { #[prost(enumeration = "FileSourceKind", tag = "1")] pub kind: i32, } +/// The type of compression used on the payload. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum Compression { + /// No compression. None = 0, + /// LZ4 block compression. Lz4 = 1, } impl Compression { @@ -102,10 +136,13 @@ impl Compression { } } } +/// The encoding of the mesage payload. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum Encoding { + /// We don't know what encoding the payload is in. Unknown = 0, + /// The payload is encoded as Arrow-IPC. ArrowIpc = 1, } impl Encoding { @@ -128,21 +165,36 @@ impl Encoding { } } } +/// What kind of source a recording comes from. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum StoreSourceKind { + /// We don't know anything about the source of this recording. + /// /// `extra` is unused. UnknownKind = 0, + /// The recording came from the C++ SDK. + /// /// `extra` is unused. CSdk = 1, + /// The recording came from the Python SDK. + /// /// `extra` is `PythonVersion`. PythonSdk = 2, + /// The recording came from the Rust SDK. + /// /// `extra` is `CrateInfo`. RustSdk = 3, + /// The recording came from a file. + /// /// `extra` is `FileSource`. File = 4, + /// The recording came from some action in the viewer. + /// /// `extra` is unused. Viewer = 5, + /// The recording came from some other source. + /// /// `extra` is a string. Other = 6, } @@ -176,14 +228,21 @@ impl StoreSourceKind { } } } +/// Determines where the file came from. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum FileSourceKind { + /// We don't know where the file came from. UnknownSource = 0, + /// The file came from the command line. Cli = 1, + /// The file was served over HTTP. Uri = 2, + /// The file was dragged into the viewer. DragAndDrop = 3, + /// The file was opened using a file dialog. FileDialog = 4, + /// The recording was produced using a data loader, such as when logging a mesh file. Sdk = 5, } impl FileSourceKind { From 76c973c642fc332d79a79f4ea6473f2a68f39e5c Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 10 Dec 2024 17:30:54 +0100 Subject: [PATCH 19/47] make `PythonVersion` less bad --- crates/store/re_log_encoding/src/codec/mod.rs | 12 ------ .../re_log_types/src/protobuf_conversions.rs | 40 +++++-------------- 2 files changed, 10 insertions(+), 42 deletions(-) diff --git a/crates/store/re_log_encoding/src/codec/mod.rs b/crates/store/re_log_encoding/src/codec/mod.rs index c332217e0e04..665e6ab716e8 100644 --- a/crates/store/re_log_encoding/src/codec/mod.rs +++ b/crates/store/re_log_encoding/src/codec/mod.rs @@ -20,18 +20,6 @@ pub enum CodecError { #[error("Unsupported encoding, expected Arrow IPC")] UnsupportedEncoding, - #[error("Invalid file header")] - InvalidFileHeader, - #[error("Unknown message header")] UnknownMessageHeader, - - #[error("Invalid message header")] - InvalidMessageHeader, - - #[error("Unknown message kind {0}")] - UnknownMessageKind(u8), - - #[error("Invalid argument: {0}")] - InvalidArgument(String), } diff --git a/crates/store/re_log_types/src/protobuf_conversions.rs b/crates/store/re_log_types/src/protobuf_conversions.rs index 6eafa0b019f7..0f4675a112b8 100644 --- a/crates/store/re_log_types/src/protobuf_conversions.rs +++ b/crates/store/re_log_types/src/protobuf_conversions.rs @@ -307,13 +307,12 @@ impl TryFrom for crate::StoreSource { impl From for re_protos::log_msg::v0::PythonVersion { #[inline] fn from(value: crate::PythonVersion) -> Self { - let mut version = Vec::new(); - version.push(value.major); - version.push(value.minor); - version.push(value.patch); - version.extend_from_slice(value.suffix.as_bytes()); - - Self { version } + Self { + major: value.major as i32, + minor: value.minor as i32, + patch: value.patch as i32, + suffix: value.suffix, + } } } @@ -322,30 +321,11 @@ impl TryFrom for crate::PythonVersion { #[inline] fn try_from(value: re_protos::log_msg::v0::PythonVersion) -> Result { - if value.version.len() < 3 { - return Err(TypeConversionError::InvalidField { - type_name: "rerun.log_msg.v0.PythonVersion", - field_name: "version", - reason: "expected at least 3 bytes".to_owned(), - }); - } - - let major = value.version[0]; - let minor = value.version[1]; - let patch = value.version[2]; - let suffix = std::str::from_utf8(&value.version[3..]) - .map_err(|err| TypeConversionError::InvalidField { - type_name: "rerun.log_msg.v0.PythonVersion", - field_name: "version", - reason: err.to_string(), - })? - .to_owned(); - Ok(Self { - major, - minor, - patch, - suffix, + major: value.major as u8, + minor: value.minor as u8, + patch: value.patch as u8, + suffix: value.suffix, }) } } From c367fc3ec16294fe85fee0de575e870c94914ca1 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 11:16:59 +0100 Subject: [PATCH 20/47] update module structure --- .../store/re_log_encoding/src/codec/arrow.rs | 52 ++++ .../src/{protobuf => codec/file}/decoder.rs | 31 +-- .../src/{protobuf => codec/file}/encoder.rs | 23 +- .../src/{protobuf.rs => codec/file/mod.rs} | 11 +- crates/store/re_log_encoding/src/codec/mod.rs | 2 + .../store/re_log_encoding/src/codec/wire.rs | 261 ------------------ .../re_log_encoding/src/codec/wire/decoder.rs | 55 ++++ .../re_log_encoding/src/codec/wire/encoder.rs | 58 ++++ .../re_log_encoding/src/codec/wire/mod.rs | 121 ++++++++ .../store/re_log_encoding/src/decoder/mod.rs | 3 +- crates/store/re_log_encoding/src/encoder.rs | 11 +- crates/store/re_log_encoding/src/lib.rs | 2 - 12 files changed, 301 insertions(+), 329 deletions(-) create mode 100644 crates/store/re_log_encoding/src/codec/arrow.rs rename crates/store/re_log_encoding/src/{protobuf => codec/file}/decoder.rs (78%) rename crates/store/re_log_encoding/src/{protobuf => codec/file}/encoder.rs (80%) rename crates/store/re_log_encoding/src/{protobuf.rs => codec/file/mod.rs} (99%) delete mode 100644 crates/store/re_log_encoding/src/codec/wire.rs create mode 100644 crates/store/re_log_encoding/src/codec/wire/decoder.rs create mode 100644 crates/store/re_log_encoding/src/codec/wire/encoder.rs create mode 100644 crates/store/re_log_encoding/src/codec/wire/mod.rs diff --git a/crates/store/re_log_encoding/src/codec/arrow.rs b/crates/store/re_log_encoding/src/codec/arrow.rs new file mode 100644 index 000000000000..42178d96389c --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/arrow.rs @@ -0,0 +1,52 @@ +use super::CodecError; + +/// Helper function that serializes given arrow schema and record batch into bytes +/// using Arrow IPC format. +pub(crate) fn write_arrow_to_bytes( + writer: &mut W, + schema: &arrow2::datatypes::Schema, + data: &arrow2::chunk::Chunk>, +) -> Result<(), CodecError> { + use arrow2::io::ipc; + + let options = ipc::write::WriteOptions { compression: None }; + let mut sw = ipc::write::StreamWriter::new(writer, options); + + sw.start(schema, None) + .map_err(CodecError::ArrowSerialization)?; + sw.write(data, None) + .map_err(CodecError::ArrowSerialization)?; + sw.finish().map_err(CodecError::ArrowSerialization)?; + + Ok(()) +} + +/// Helper function that deserializes raw bytes into arrow schema and record batch +/// using Arrow IPC format. +pub(crate) fn read_arrow_from_bytes( + reader: &mut R, +) -> Result< + ( + arrow2::datatypes::Schema, + arrow2::chunk::Chunk>, + ), + CodecError, +> { + use arrow2::io::ipc; + + let metadata = + ipc::read::read_stream_metadata(reader).map_err(CodecError::ArrowSerialization)?; + let mut stream = ipc::read::StreamReader::new(reader, metadata, None); + + let schema = stream.schema().clone(); + // there should be at least one record batch in the stream + let stream_state = stream + .next() + .ok_or(CodecError::MissingRecordBatch)? + .map_err(CodecError::ArrowSerialization)?; + + match stream_state { + ipc::read::StreamState::Waiting => Err(CodecError::UnexpectedStreamState), + ipc::read::StreamState::Some(chunk) => Ok((schema, chunk)), + } +} diff --git a/crates/store/re_log_encoding/src/protobuf/decoder.rs b/crates/store/re_log_encoding/src/codec/file/decoder.rs similarity index 78% rename from crates/store/re_log_encoding/src/protobuf/decoder.rs rename to crates/store/re_log_encoding/src/codec/file/decoder.rs index 9a73bd54ed72..e6b840ac6c8d 100644 --- a/crates/store/re_log_encoding/src/protobuf/decoder.rs +++ b/crates/store/re_log_encoding/src/codec/file/decoder.rs @@ -1,4 +1,5 @@ use super::{MessageHeader, MessageKind}; +use crate::codec::arrow::read_arrow_from_bytes; use crate::decoder::DecodeError; use crate::{codec::CodecError, Compression}; use re_log_types::LogMsg; @@ -104,33 +105,3 @@ fn decode_arrow( Ok(read_arrow_from_bytes(&mut &data[..])?) } - -/// Helper function that deserializes raw bytes into arrow schema and record batch -/// using Arrow IPC format. -fn read_arrow_from_bytes( - reader: &mut R, -) -> Result< - ( - arrow2::datatypes::Schema, - arrow2::chunk::Chunk>, - ), - CodecError, -> { - use arrow2::io::ipc; - - let metadata = - ipc::read::read_stream_metadata(reader).map_err(CodecError::ArrowSerialization)?; - let mut stream = ipc::read::StreamReader::new(reader, metadata, None); - - let schema = stream.schema().clone(); - // there should be at least one record batch in the stream - let stream_state = stream - .next() - .ok_or(CodecError::MissingRecordBatch)? - .map_err(CodecError::ArrowSerialization)?; - - match stream_state { - ipc::read::StreamState::Waiting => Err(CodecError::UnexpectedStreamState), - ipc::read::StreamState::Some(chunk) => Ok((schema, chunk)), - } -} diff --git a/crates/store/re_log_encoding/src/protobuf/encoder.rs b/crates/store/re_log_encoding/src/codec/file/encoder.rs similarity index 80% rename from crates/store/re_log_encoding/src/protobuf/encoder.rs rename to crates/store/re_log_encoding/src/codec/file/encoder.rs index 22e967ea533a..6868e9ca792a 100644 --- a/crates/store/re_log_encoding/src/protobuf/encoder.rs +++ b/crates/store/re_log_encoding/src/codec/file/encoder.rs @@ -1,5 +1,5 @@ use super::{MessageHeader, MessageKind}; -use crate::codec::CodecError; +use crate::codec::arrow::write_arrow_to_bytes; use crate::encoder::EncodeError; use crate::Compression; use re_log_types::LogMsg; @@ -85,24 +85,3 @@ fn encode_arrow( crate::Compression::LZ4 => Ok(lz4_flex::block::compress(&uncompressed)), } } - -/// Helper function that serializes given arrow schema and record batch into bytes -/// using Arrow IPC format. -fn write_arrow_to_bytes( - writer: &mut W, - schema: &arrow2::datatypes::Schema, - data: &arrow2::chunk::Chunk>, -) -> Result<(), CodecError> { - use arrow2::io::ipc; - - let options = ipc::write::WriteOptions { compression: None }; - let mut sw = ipc::write::StreamWriter::new(writer, options); - - sw.start(schema, None) - .map_err(CodecError::ArrowSerialization)?; - sw.write(data, None) - .map_err(CodecError::ArrowSerialization)?; - sw.finish().map_err(CodecError::ArrowSerialization)?; - - Ok(()) -} diff --git a/crates/store/re_log_encoding/src/protobuf.rs b/crates/store/re_log_encoding/src/codec/file/mod.rs similarity index 99% rename from crates/store/re_log_encoding/src/protobuf.rs rename to crates/store/re_log_encoding/src/codec/file/mod.rs index 13be885e791c..ef058b013e85 100644 --- a/crates/store/re_log_encoding/src/protobuf.rs +++ b/crates/store/re_log_encoding/src/codec/file/mod.rs @@ -1,3 +1,8 @@ +#[cfg(feature = "decoder")] +pub(crate) mod decoder; +#[cfg(feature = "encoder")] +pub(crate) mod encoder; + #[allow(dead_code)] // used in encoder/decoder behind feature flag #[derive(Debug, Clone, Copy)] #[repr(u32)] @@ -14,9 +19,3 @@ pub(crate) struct MessageHeader { pub(crate) kind: MessageKind, pub(crate) len: u32, } - -#[cfg(feature = "encoder")] -pub(crate) mod encoder; - -#[cfg(feature = "decoder")] -pub(crate) mod decoder; diff --git a/crates/store/re_log_encoding/src/codec/mod.rs b/crates/store/re_log_encoding/src/codec/mod.rs index 665e6ab716e8..7c51b4b690e0 100644 --- a/crates/store/re_log_encoding/src/codec/mod.rs +++ b/crates/store/re_log_encoding/src/codec/mod.rs @@ -1,3 +1,5 @@ +mod arrow; +pub mod file; pub mod wire; #[derive(Debug, thiserror::Error)] diff --git a/crates/store/re_log_encoding/src/codec/wire.rs b/crates/store/re_log_encoding/src/codec/wire.rs deleted file mode 100644 index e0b0daabd1d1..000000000000 --- a/crates/store/re_log_encoding/src/codec/wire.rs +++ /dev/null @@ -1,261 +0,0 @@ -use arrow2::chunk::Chunk as Arrow2Chunk; -use arrow2::datatypes::Schema as Arrow2Schema; -use arrow2::io::ipc; -use re_chunk::Arrow2Array; -use re_chunk::TransportChunk; - -use super::CodecError; - -#[derive(Clone, Copy, PartialEq, Eq, Hash, Default)] -pub struct MessageHeader(pub u8); - -impl MessageHeader { - pub const NO_DATA: Self = Self(1); - pub const RECORD_BATCH: Self = Self(2); - - pub const SIZE_BYTES: usize = 1; -} - -impl MessageHeader { - fn decode(read: &mut impl std::io::Read) -> Result { - let mut buffer = [0_u8; Self::SIZE_BYTES]; - read.read_exact(&mut buffer) - .map_err(CodecError::HeaderDecoding)?; - - let header = u8::from_le(buffer[0]); - - Ok(Self(header)) - } - - fn encode(&self, write: &mut impl std::io::Write) -> Result<(), CodecError> { - write - .write_all(&[self.0]) - .map_err(CodecError::HeaderEncoding)?; - - Ok(()) - } -} - -#[derive(Debug)] -pub enum TransportMessageV0 { - NoData, - RecordBatch(TransportChunk), -} - -impl TransportMessageV0 { - fn to_bytes(&self) -> Result, CodecError> { - match self { - Self::NoData => { - let mut data: Vec = Vec::new(); - MessageHeader::NO_DATA.encode(&mut data)?; - Ok(data) - } - Self::RecordBatch(chunk) => { - let mut data: Vec = Vec::new(); - MessageHeader::RECORD_BATCH.encode(&mut data)?; - - write_arrow_to_bytes(&mut data, &chunk.schema, &chunk.data)?; - - Ok(data) - } - } - } - - fn from_bytes(data: &[u8]) -> Result { - let mut reader = std::io::Cursor::new(data); - let header = MessageHeader::decode(&mut reader)?; - - match header { - MessageHeader::NO_DATA => Ok(Self::NoData), - MessageHeader::RECORD_BATCH => { - let (schema, data) = read_arrow_from_bytes(&mut reader)?; - - let tc = TransportChunk { - schema: schema.clone(), - data, - }; - - Ok(Self::RecordBatch(tc)) - } - _ => Err(CodecError::UnknownMessageHeader), - } - } -} - -// TODO(zehiko) add support for separately encoding schema from the record batch to get rid of overhead -// of sending schema in each transport message for the same stream of batches. This will require codec -// to become stateful and keep track if schema was sent / received. -/// Encode a transport chunk into a byte stream. -pub fn encode( - version: re_protos::common::v0::EncoderVersion, - chunk: TransportChunk, -) -> Result, CodecError> { - match version { - re_protos::common::v0::EncoderVersion::V0 => { - TransportMessageV0::RecordBatch(chunk).to_bytes() - } - } -} - -/// Encode a `NoData` message into a byte stream. This can be used by the remote store -/// (i.e. data producer) to signal back to the client that there's no data available. -pub fn no_data(version: re_protos::common::v0::EncoderVersion) -> Result, CodecError> { - match version { - re_protos::common::v0::EncoderVersion::V0 => TransportMessageV0::NoData.to_bytes(), - } -} - -/// Decode transport data from a byte stream - if there's a record batch present, return it, otherwise return `None`. -pub fn decode( - version: re_protos::common::v0::EncoderVersion, - data: &[u8], -) -> Result, CodecError> { - match version { - re_protos::common::v0::EncoderVersion::V0 => { - let msg = TransportMessageV0::from_bytes(data)?; - match msg { - TransportMessageV0::RecordBatch(chunk) => Ok(Some(chunk)), - TransportMessageV0::NoData => Ok(None), - } - } - } -} - -/// Helper function that serializes given arrow schema and record batch into bytes -/// using Arrow IPC format. -pub fn write_arrow_to_bytes( - writer: &mut W, - schema: &Arrow2Schema, - data: &Arrow2Chunk>, -) -> Result<(), CodecError> { - let options = ipc::write::WriteOptions { compression: None }; - let mut sw = ipc::write::StreamWriter::new(writer, options); - - sw.start(schema, None) - .map_err(CodecError::ArrowSerialization)?; - sw.write(data, None) - .map_err(CodecError::ArrowSerialization)?; - sw.finish().map_err(CodecError::ArrowSerialization)?; - - Ok(()) -} - -/// Helper function that deserializes raw bytes into arrow schema and record batch -/// using Arrow IPC format. -pub fn read_arrow_from_bytes( - reader: &mut R, -) -> Result<(Arrow2Schema, Arrow2Chunk>), CodecError> { - let metadata = - ipc::read::read_stream_metadata(reader).map_err(CodecError::ArrowSerialization)?; - let mut stream = ipc::read::StreamReader::new(reader, metadata, None); - - let schema = stream.schema().clone(); - // there should be at least one record batch in the stream - let stream_state = stream - .next() - .ok_or(CodecError::MissingRecordBatch)? - .map_err(CodecError::ArrowSerialization)?; - - match stream_state { - ipc::read::StreamState::Waiting => Err(CodecError::UnexpectedStreamState), - ipc::read::StreamState::Some(chunk) => Ok((schema, chunk)), - } -} - -#[cfg(test)] -mod tests { - use crate::{ - codec::wire::{decode, encode, TransportMessageV0}, - codec::CodecError, - }; - use re_chunk::{Chunk, RowId}; - use re_log_types::{example_components::MyPoint, Timeline}; - use re_protos::common::v0::EncoderVersion; - - fn get_test_chunk() -> Chunk { - let row_id1 = RowId::new(); - let row_id2 = RowId::new(); - - let timepoint1 = [ - (Timeline::log_time(), 100), - (Timeline::new_sequence("frame"), 1), - ]; - let timepoint2 = [ - (Timeline::log_time(), 104), - (Timeline::new_sequence("frame"), 1), - ]; - - let points1 = &[MyPoint::new(1.0, 1.0)]; - let points2 = &[MyPoint::new(2.0, 2.0)]; - - Chunk::builder("mypoints".into()) - .with_component_batches(row_id1, timepoint1, [points1 as _]) - .with_component_batches(row_id2, timepoint2, [points2 as _]) - .build() - .unwrap() - } - - #[test] - fn test_message_v0_no_data() { - let msg = TransportMessageV0::NoData; - let data = msg.to_bytes().unwrap(); - let decoded = TransportMessageV0::from_bytes(&data).unwrap(); - assert!(matches!(decoded, TransportMessageV0::NoData)); - } - - #[test] - fn test_message_v0_record_batch() { - let expected_chunk = get_test_chunk(); - - let msg = TransportMessageV0::RecordBatch(expected_chunk.clone().to_transport().unwrap()); - let data = msg.to_bytes().unwrap(); - let decoded = TransportMessageV0::from_bytes(&data).unwrap(); - - #[allow(clippy::match_wildcard_for_single_variants)] - match decoded { - TransportMessageV0::RecordBatch(transport) => { - let decoded_chunk = Chunk::from_transport(&transport).unwrap(); - assert_eq!(expected_chunk, decoded_chunk); - } - _ => panic!("unexpected message type"), - } - } - - #[test] - fn test_invalid_batch_data() { - let data = vec![2, 3, 4]; // '1' is NO_DATA message header - let decoded = TransportMessageV0::from_bytes(&data); - - assert!(matches!( - decoded.err().unwrap(), - CodecError::ArrowSerialization(_) - )); - } - - #[test] - fn test_unknown_header() { - let data = vec![3]; - let decoded = TransportMessageV0::from_bytes(&data); - assert!(decoded.is_err()); - - assert!(matches!( - decoded.err().unwrap(), - CodecError::UnknownMessageHeader - )); - } - - #[test] - fn test_v0_codec() { - let expected_chunk = get_test_chunk(); - - let encoded = encode( - EncoderVersion::V0, - expected_chunk.clone().to_transport().unwrap(), - ) - .unwrap(); - let decoded = decode(EncoderVersion::V0, &encoded).unwrap().unwrap(); - let decoded_chunk = Chunk::from_transport(&decoded).unwrap(); - - assert_eq!(expected_chunk, decoded_chunk); - } -} diff --git a/crates/store/re_log_encoding/src/codec/wire/decoder.rs b/crates/store/re_log_encoding/src/codec/wire/decoder.rs new file mode 100644 index 000000000000..302af6d518a1 --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/wire/decoder.rs @@ -0,0 +1,55 @@ +use super::MessageHeader; +use super::TransportMessageV0; +use crate::codec::arrow::read_arrow_from_bytes; +use crate::codec::CodecError; +use re_chunk::TransportChunk; + +impl MessageHeader { + fn decode(read: &mut impl std::io::Read) -> Result { + let mut buffer = [0_u8; Self::SIZE_BYTES]; + read.read_exact(&mut buffer) + .map_err(CodecError::HeaderDecoding)?; + + let header = u8::from_le(buffer[0]); + + Ok(Self(header)) + } +} + +impl TransportMessageV0 { + fn from_bytes(data: &[u8]) -> Result { + let mut reader = std::io::Cursor::new(data); + let header = MessageHeader::decode(&mut reader)?; + + match header { + MessageHeader::NO_DATA => Ok(Self::NoData), + MessageHeader::RECORD_BATCH => { + let (schema, data) = read_arrow_from_bytes(&mut reader)?; + + let tc = TransportChunk { + schema: schema.clone(), + data, + }; + + Ok(Self::RecordBatch(tc)) + } + _ => Err(CodecError::UnknownMessageHeader), + } + } +} + +/// Decode transport data from a byte stream - if there's a record batch present, return it, otherwise return `None`. +pub fn decode( + version: re_protos::common::v0::EncoderVersion, + data: &[u8], +) -> Result, CodecError> { + match version { + re_protos::common::v0::EncoderVersion::V0 => { + let msg = TransportMessageV0::from_bytes(data)?; + match msg { + TransportMessageV0::RecordBatch(chunk) => Ok(Some(chunk)), + TransportMessageV0::NoData => Ok(None), + } + } + } +} diff --git a/crates/store/re_log_encoding/src/codec/wire/encoder.rs b/crates/store/re_log_encoding/src/codec/wire/encoder.rs new file mode 100644 index 000000000000..d848d1ad9f62 --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/wire/encoder.rs @@ -0,0 +1,58 @@ +use super::MessageHeader; +use super::TransportMessageV0; +use crate::codec::arrow::write_arrow_to_bytes; +use crate::codec::CodecError; +use re_chunk::TransportChunk; + +impl MessageHeader { + fn encode(&self, write: &mut impl std::io::Write) -> Result<(), CodecError> { + write + .write_all(&[self.0]) + .map_err(CodecError::HeaderEncoding)?; + + Ok(()) + } +} + +impl TransportMessageV0 { + fn to_bytes(&self) -> Result, CodecError> { + match self { + Self::NoData => { + let mut data: Vec = Vec::new(); + MessageHeader::NO_DATA.encode(&mut data)?; + Ok(data) + } + Self::RecordBatch(chunk) => { + let mut data: Vec = Vec::new(); + MessageHeader::RECORD_BATCH.encode(&mut data)?; + + write_arrow_to_bytes(&mut data, &chunk.schema, &chunk.data)?; + + Ok(data) + } + } + } +} + +/// Encode a `NoData` message into a byte stream. This can be used by the remote store +/// (i.e. data producer) to signal back to the client that there's no data available. +pub fn no_data(version: re_protos::common::v0::EncoderVersion) -> Result, CodecError> { + match version { + re_protos::common::v0::EncoderVersion::V0 => TransportMessageV0::NoData.to_bytes(), + } +} + +// TODO(zehiko) add support for separately encoding schema from the record batch to get rid of overhead +// of sending schema in each transport message for the same stream of batches. This will require codec +// to become stateful and keep track if schema was sent / received. +/// Encode a transport chunk into a byte stream. +pub fn encode( + version: re_protos::common::v0::EncoderVersion, + chunk: TransportChunk, +) -> Result, CodecError> { + match version { + re_protos::common::v0::EncoderVersion::V0 => { + TransportMessageV0::RecordBatch(chunk).to_bytes() + } + } +} diff --git a/crates/store/re_log_encoding/src/codec/wire/mod.rs b/crates/store/re_log_encoding/src/codec/wire/mod.rs new file mode 100644 index 000000000000..587e9e31e2ce --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/wire/mod.rs @@ -0,0 +1,121 @@ +pub mod decoder; +pub mod encoder; + +pub use decoder::decode; +pub use encoder::encode; + +use re_chunk::TransportChunk; + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Default)] +pub struct MessageHeader(pub u8); + +impl MessageHeader { + pub const NO_DATA: Self = Self(1); + pub const RECORD_BATCH: Self = Self(2); + + pub const SIZE_BYTES: usize = 1; +} + +#[derive(Debug)] +pub enum TransportMessageV0 { + NoData, + RecordBatch(TransportChunk), +} + +#[cfg(test)] +mod tests { + use crate::{ + codec::wire::{decode, encode, TransportMessageV0}, + codec::CodecError, + }; + use re_chunk::{Chunk, RowId}; + use re_log_types::{example_components::MyPoint, Timeline}; + use re_protos::common::v0::EncoderVersion; + + fn get_test_chunk() -> Chunk { + let row_id1 = RowId::new(); + let row_id2 = RowId::new(); + + let timepoint1 = [ + (Timeline::log_time(), 100), + (Timeline::new_sequence("frame"), 1), + ]; + let timepoint2 = [ + (Timeline::log_time(), 104), + (Timeline::new_sequence("frame"), 1), + ]; + + let points1 = &[MyPoint::new(1.0, 1.0)]; + let points2 = &[MyPoint::new(2.0, 2.0)]; + + Chunk::builder("mypoints".into()) + .with_component_batches(row_id1, timepoint1, [points1 as _]) + .with_component_batches(row_id2, timepoint2, [points2 as _]) + .build() + .unwrap() + } + + #[test] + fn test_message_v0_no_data() { + let msg = TransportMessageV0::NoData; + let data = msg.to_bytes().unwrap(); + let decoded = TransportMessageV0::from_bytes(&data).unwrap(); + assert!(matches!(decoded, TransportMessageV0::NoData)); + } + + #[test] + fn test_message_v0_record_batch() { + let expected_chunk = get_test_chunk(); + + let msg = TransportMessageV0::RecordBatch(expected_chunk.clone().to_transport().unwrap()); + let data = msg.to_bytes().unwrap(); + let decoded = TransportMessageV0::from_bytes(&data).unwrap(); + + #[allow(clippy::match_wildcard_for_single_variants)] + match decoded { + TransportMessageV0::RecordBatch(transport) => { + let decoded_chunk = Chunk::from_transport(&transport).unwrap(); + assert_eq!(expected_chunk, decoded_chunk); + } + _ => panic!("unexpected message type"), + } + } + + #[test] + fn test_invalid_batch_data() { + let data = vec![2, 3, 4]; // '1' is NO_DATA message header + let decoded = TransportMessageV0::from_bytes(&data); + + assert!(matches!( + decoded.err().unwrap(), + CodecError::ArrowSerialization(_) + )); + } + + #[test] + fn test_unknown_header() { + let data = vec![3]; + let decoded = TransportMessageV0::from_bytes(&data); + assert!(decoded.is_err()); + + assert!(matches!( + decoded.err().unwrap(), + CodecError::UnknownMessageHeader + )); + } + + #[test] + fn test_v0_codec() { + let expected_chunk = get_test_chunk(); + + let encoded = encode( + EncoderVersion::V0, + expected_chunk.clone().to_transport().unwrap(), + ) + .unwrap(); + let decoded = decode(EncoderVersion::V0, &encoded).unwrap().unwrap(); + let decoded_chunk = Chunk::from_transport(&decoded).unwrap(); + + assert_eq!(expected_chunk, decoded_chunk); + } +} diff --git a/crates/store/re_log_encoding/src/decoder/mod.rs b/crates/store/re_log_encoding/src/decoder/mod.rs index 23fab4fe3663..b909306cece6 100644 --- a/crates/store/re_log_encoding/src/decoder/mod.rs +++ b/crates/store/re_log_encoding/src/decoder/mod.rs @@ -9,6 +9,7 @@ use re_build_info::CrateVersion; use re_log_types::LogMsg; use crate::codec; +use crate::codec::file::decoder; use crate::FileHeader; use crate::MessageHeader; use crate::VersionPolicy; @@ -294,7 +295,7 @@ impl Iterator for Decoder { let msg = match self.options.serializer { Serializer::Protobuf => { - match crate::protobuf::decoder::decode(&mut self.read, self.options.compression) { + match decoder::decode(&mut self.read, self.options.compression) { Ok((read_bytes, msg)) => { self.size_bytes += read_bytes; msg diff --git a/crates/store/re_log_encoding/src/encoder.rs b/crates/store/re_log_encoding/src/encoder.rs index 247a3b97d2bd..90760800eb99 100644 --- a/crates/store/re_log_encoding/src/encoder.rs +++ b/crates/store/re_log_encoding/src/encoder.rs @@ -1,6 +1,7 @@ //! Encoding of [`LogMsg`]es as a binary stream, e.g. to store in an `.rrd` file, or send over network. use crate::codec; +use crate::codec::file::{self, encoder}; use crate::FileHeader; use crate::MessageHeader; use crate::Serializer; @@ -155,11 +156,7 @@ impl Encoder { self.uncompressed.clear(); match self.serializer { Serializer::Protobuf => { - crate::protobuf::encoder::encode( - &mut self.uncompressed, - message, - self.compression, - )?; + encoder::encode(&mut self.uncompressed, message, self.compression)?; self.write .write_all(&self.uncompressed) @@ -215,8 +212,8 @@ impl Encoder { MessageHeader::EndOfStream.encode(&mut self.write)?; } Serializer::Protobuf => { - crate::protobuf::MessageHeader { - kind: crate::protobuf::MessageKind::End, + file::MessageHeader { + kind: file::MessageKind::End, len: 0, } .encode(&mut self.write)?; diff --git a/crates/store/re_log_encoding/src/lib.rs b/crates/store/re_log_encoding/src/lib.rs index 6a70e578f58c..d3abf1c74d80 100644 --- a/crates/store/re_log_encoding/src/lib.rs +++ b/crates/store/re_log_encoding/src/lib.rs @@ -6,8 +6,6 @@ pub mod decoder; #[cfg(feature = "encoder")] pub mod encoder; -mod protobuf; - pub mod codec; #[cfg(feature = "encoder")] From 44f701795f1524f57eb0af328cfbb22f33e9219e Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 11:18:04 +0100 Subject: [PATCH 21/47] typo --- crates/store/re_protos/proto/rerun/v0/log_msg.proto | 2 +- crates/store/re_protos/src/v0/rerun.log_msg.v0.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/re_protos/proto/rerun/v0/log_msg.proto b/crates/store/re_protos/proto/rerun/v0/log_msg.proto index 2accf5904da1..c440dd9b230e 100644 --- a/crates/store/re_protos/proto/rerun/v0/log_msg.proto +++ b/crates/store/re_protos/proto/rerun/v0/log_msg.proto @@ -22,7 +22,7 @@ enum Compression { LZ4 = 1; } -// The encoding of the mesage payload. +// The encoding of the message payload. enum Encoding { // We don't know what encoding the payload is in. UNKNOWN = 0; diff --git a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs index a4946bb35557..de25726319c9 100644 --- a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs @@ -136,7 +136,7 @@ impl Compression { } } } -/// The encoding of the mesage payload. +/// The encoding of the message payload. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum Encoding { From ce14bf7699e600302345d378ba4d362ae1115481 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 11:48:22 +0100 Subject: [PATCH 22/47] undo header bump --- crates/store/re_log_encoding/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/re_log_encoding/src/lib.rs b/crates/store/re_log_encoding/src/lib.rs index d3abf1c74d80..5d895e8a4d1d 100644 --- a/crates/store/re_log_encoding/src/lib.rs +++ b/crates/store/re_log_encoding/src/lib.rs @@ -38,10 +38,10 @@ pub use file_sink::{FileSink, FileSinkError}; // ---------------------------------------------------------------------------- #[cfg(any(feature = "encoder", feature = "decoder"))] -const RRD_HEADER: &[u8; 4] = b"RRIO"; +const RRD_HEADER: &[u8; 4] = b"RRF2"; #[cfg(feature = "decoder")] -const OLD_RRD_HEADERS: &[[u8; 4]] = &[*b"RRF0", *b"RRF1", *b"RRF2"]; +const OLD_RRD_HEADERS: &[[u8; 4]] = &[*b"RRF0", *b"RRF1"]; // ---------------------------------------------------------------------------- From ad4fc56025e012504fdc15c0836242676eb63493 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 11:48:33 +0100 Subject: [PATCH 23/47] uncap max decode size --- crates/store/re_grpc_client/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs index 478312bb6aaa..0e22afbca90a 100644 --- a/crates/store/re_grpc_client/src/lib.rs +++ b/crates/store/re_grpc_client/src/lib.rs @@ -151,7 +151,7 @@ async fn stream_recording_async( .connect() .await?; - StorageNodeClient::new(tonic_client) + StorageNodeClient::new(tonic_client).max_decoding_message_size(usize::MAX) }; re_log::debug!("Fetching {recording_id}…"); From 4c2b644c85db8ed7480476c359f869d7d28e9919 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 12:46:10 +0100 Subject: [PATCH 24/47] fix lints --- crates/store/re_log_encoding/src/codec/wire/decoder.rs | 4 ++-- crates/store/re_log_encoding/src/codec/wire/encoder.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/store/re_log_encoding/src/codec/wire/decoder.rs b/crates/store/re_log_encoding/src/codec/wire/decoder.rs index 302af6d518a1..06b9bccdbc23 100644 --- a/crates/store/re_log_encoding/src/codec/wire/decoder.rs +++ b/crates/store/re_log_encoding/src/codec/wire/decoder.rs @@ -5,7 +5,7 @@ use crate::codec::CodecError; use re_chunk::TransportChunk; impl MessageHeader { - fn decode(read: &mut impl std::io::Read) -> Result { + pub(crate) fn decode(read: &mut impl std::io::Read) -> Result { let mut buffer = [0_u8; Self::SIZE_BYTES]; read.read_exact(&mut buffer) .map_err(CodecError::HeaderDecoding)?; @@ -17,7 +17,7 @@ impl MessageHeader { } impl TransportMessageV0 { - fn from_bytes(data: &[u8]) -> Result { + pub(crate) fn from_bytes(data: &[u8]) -> Result { let mut reader = std::io::Cursor::new(data); let header = MessageHeader::decode(&mut reader)?; diff --git a/crates/store/re_log_encoding/src/codec/wire/encoder.rs b/crates/store/re_log_encoding/src/codec/wire/encoder.rs index d848d1ad9f62..e6ae62c1e1b7 100644 --- a/crates/store/re_log_encoding/src/codec/wire/encoder.rs +++ b/crates/store/re_log_encoding/src/codec/wire/encoder.rs @@ -5,7 +5,7 @@ use crate::codec::CodecError; use re_chunk::TransportChunk; impl MessageHeader { - fn encode(&self, write: &mut impl std::io::Write) -> Result<(), CodecError> { + pub(crate) fn encode(&self, write: &mut impl std::io::Write) -> Result<(), CodecError> { write .write_all(&[self.0]) .map_err(CodecError::HeaderEncoding)?; @@ -15,7 +15,7 @@ impl MessageHeader { } impl TransportMessageV0 { - fn to_bytes(&self) -> Result, CodecError> { + pub(crate) fn to_bytes(&self) -> Result, CodecError> { match self { Self::NoData => { let mut data: Vec = Vec::new(); From a615187140236c10b71ab30d0f33905c485c2a58 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 12:58:46 +0100 Subject: [PATCH 25/47] add max decoding message size issue link --- crates/store/re_grpc_client/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs index 0e22afbca90a..49c3a58a698d 100644 --- a/crates/store/re_grpc_client/src/lib.rs +++ b/crates/store/re_grpc_client/src/lib.rs @@ -151,6 +151,7 @@ async fn stream_recording_async( .connect() .await?; + // TODO(#8411): figure out the right size for this StorageNodeClient::new(tonic_client).max_decoding_message_size(usize::MAX) }; From 77f41429d9726634046f7351055239f41d12f621 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 13:02:55 +0100 Subject: [PATCH 26/47] add todo for arrow ipc compression --- crates/store/re_log_encoding/src/codec/arrow.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/store/re_log_encoding/src/codec/arrow.rs b/crates/store/re_log_encoding/src/codec/arrow.rs index 42178d96389c..5a1e70a29a01 100644 --- a/crates/store/re_log_encoding/src/codec/arrow.rs +++ b/crates/store/re_log_encoding/src/codec/arrow.rs @@ -1,5 +1,7 @@ use super::CodecError; +// TODO(#8412): try using arrow ipc `compression` option instead of doing our own compression + /// Helper function that serializes given arrow schema and record batch into bytes /// using Arrow IPC format. pub(crate) fn write_arrow_to_bytes( From d79f3170794551e81147f14254a2f07b6aae2763 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 13:10:10 +0100 Subject: [PATCH 27/47] rename EncodingOptions constants --- crates/store/re_data_loader/src/loader_rrd.rs | 2 +- .../store/re_entity_db/examples/memory_usage.rs | 2 +- .../benches/msg_encode_benchmark.rs | 2 +- .../store/re_log_encoding/src/decoder/stream.rs | 16 ++++++++-------- crates/store/re_log_encoding/src/encoder.rs | 12 ++++++++++-- crates/store/re_log_encoding/src/file_sink.rs | 4 ++-- crates/store/re_log_encoding/src/lib.rs | 6 +++--- crates/store/re_sdk_comms/src/buffered_client.rs | 2 +- crates/top/re_sdk/src/binary_stream_sink.rs | 2 +- crates/top/rerun/src/commands/entrypoint.rs | 2 +- crates/top/rerun/src/commands/rrd/filter.rs | 2 +- .../top/rerun/src/commands/rrd/merge_compact.rs | 2 +- crates/viewer/re_viewer/src/saving.rs | 2 +- 13 files changed, 32 insertions(+), 24 deletions(-) diff --git a/crates/store/re_data_loader/src/loader_rrd.rs b/crates/store/re_data_loader/src/loader_rrd.rs index 2c4157f08afd..b421c6aa8d06 100644 --- a/crates/store/re_data_loader/src/loader_rrd.rs +++ b/crates/store/re_data_loader/src/loader_rrd.rs @@ -341,7 +341,7 @@ mod tests { let mut encoder = DroppableEncoder::new( re_build_info::CrateVersion::LOCAL, - re_log_encoding::EncodingOptions::UNCOMPRESSED, + re_log_encoding::EncodingOptions::MSGPACK_UNCOMPRESSED, rrd_file, ) .unwrap(); diff --git a/crates/store/re_entity_db/examples/memory_usage.rs b/crates/store/re_entity_db/examples/memory_usage.rs index 585b94a6189c..870e296bcf72 100644 --- a/crates/store/re_entity_db/examples/memory_usage.rs +++ b/crates/store/re_entity_db/examples/memory_usage.rs @@ -66,7 +66,7 @@ fn log_messages() { fn encode_log_msg(log_msg: &LogMsg) -> Vec { let mut bytes = vec![]; - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; re_log_encoding::encoder::encode_ref( re_build_info::CrateVersion::LOCAL, encoding_options, diff --git a/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs b/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs index 1b73e51dc4b5..36f966fea758 100644 --- a/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs +++ b/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs @@ -32,7 +32,7 @@ criterion_group!( criterion_main!(benches); fn encode_log_msgs(messages: &[LogMsg]) -> Vec { - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; let mut bytes = vec![]; re_log_encoding::encoder::encode_ref( re_build_info::CrateVersion::LOCAL, diff --git a/crates/store/re_log_encoding/src/decoder/stream.rs b/crates/store/re_log_encoding/src/decoder/stream.rs index ff85e734581b..0682c57a0609 100644 --- a/crates/store/re_log_encoding/src/decoder/stream.rs +++ b/crates/store/re_log_encoding/src/decoder/stream.rs @@ -317,7 +317,7 @@ mod tests { #[test] fn stream_whole_chunks_uncompressed() { - let (input, data) = test_data(EncodingOptions::UNCOMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_UNCOMPRESSED, 16); let mut decoder = StreamDecoder::new(VersionPolicy::Error); @@ -334,7 +334,7 @@ mod tests { #[test] fn stream_byte_chunks_uncompressed() { - let (input, data) = test_data(EncodingOptions::UNCOMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_UNCOMPRESSED, 16); let mut decoder = StreamDecoder::new(VersionPolicy::Error); @@ -353,8 +353,8 @@ mod tests { #[test] fn two_concatenated_streams() { - let (input1, data1) = test_data(EncodingOptions::UNCOMPRESSED, 16); - let (input2, data2) = test_data(EncodingOptions::UNCOMPRESSED, 16); + let (input1, data1) = test_data(EncodingOptions::MSGPACK_UNCOMPRESSED, 16); + let (input2, data2) = test_data(EncodingOptions::MSGPACK_UNCOMPRESSED, 16); let input = input1.into_iter().chain(input2).collect::>(); let mut decoder = StreamDecoder::new(VersionPolicy::Error); @@ -373,7 +373,7 @@ mod tests { #[test] fn stream_whole_chunks_compressed() { - let (input, data) = test_data(EncodingOptions::COMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_COMPRESSED, 16); let mut decoder = StreamDecoder::new(VersionPolicy::Error); @@ -390,7 +390,7 @@ mod tests { #[test] fn stream_byte_chunks_compressed() { - let (input, data) = test_data(EncodingOptions::COMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_COMPRESSED, 16); let mut decoder = StreamDecoder::new(VersionPolicy::Error); @@ -409,7 +409,7 @@ mod tests { #[test] fn stream_3x16_chunks() { - let (input, data) = test_data(EncodingOptions::COMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_COMPRESSED, 16); let mut decoder = StreamDecoder::new(VersionPolicy::Error); let mut decoded_messages = vec![]; @@ -438,7 +438,7 @@ mod tests { fn stream_irregular_chunks() { // this attempts to stress-test `try_read` with chunks of various sizes - let (input, data) = test_data(EncodingOptions::COMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_COMPRESSED, 16); let mut data = Cursor::new(data); let mut decoder = StreamDecoder::new(VersionPolicy::Error); diff --git a/crates/store/re_log_encoding/src/encoder.rs b/crates/store/re_log_encoding/src/encoder.rs index 90760800eb99..47fd4bdffa2e 100644 --- a/crates/store/re_log_encoding/src/encoder.rs +++ b/crates/store/re_log_encoding/src/encoder.rs @@ -282,12 +282,20 @@ pub fn encode_as_bytes( #[inline] pub fn local_encoder() -> Result>, EncodeError> { - DroppableEncoder::new(CrateVersion::LOCAL, EncodingOptions::COMPRESSED, Vec::new()) + DroppableEncoder::new( + CrateVersion::LOCAL, + EncodingOptions::MSGPACK_COMPRESSED, + Vec::new(), + ) } #[inline] pub fn local_raw_encoder() -> Result>, EncodeError> { - Encoder::new(CrateVersion::LOCAL, EncodingOptions::COMPRESSED, Vec::new()) + Encoder::new( + CrateVersion::LOCAL, + EncodingOptions::MSGPACK_COMPRESSED, + Vec::new(), + ) } #[inline] diff --git a/crates/store/re_log_encoding/src/file_sink.rs b/crates/store/re_log_encoding/src/file_sink.rs index cd5e7d2d953f..1a80625987f3 100644 --- a/crates/store/re_log_encoding/src/file_sink.rs +++ b/crates/store/re_log_encoding/src/file_sink.rs @@ -61,7 +61,7 @@ impl FileSink { /// Start writing log messages to a file at the given path. pub fn new(path: impl Into) -> Result { // We always compress on disk - let encoding_options = crate::EncodingOptions::COMPRESSED; + let encoding_options = crate::EncodingOptions::MSGPACK_COMPRESSED; let (tx, rx) = std::sync::mpsc::channel(); @@ -91,7 +91,7 @@ impl FileSink { /// Start writing log messages to standard output. pub fn stdout() -> Result { - let encoding_options = crate::EncodingOptions::COMPRESSED; + let encoding_options = crate::EncodingOptions::MSGPACK_COMPRESSED; let (tx, rx) = std::sync::mpsc::channel(); diff --git a/crates/store/re_log_encoding/src/lib.rs b/crates/store/re_log_encoding/src/lib.rs index 5d895e8a4d1d..b2ae061e35fa 100644 --- a/crates/store/re_log_encoding/src/lib.rs +++ b/crates/store/re_log_encoding/src/lib.rs @@ -70,15 +70,15 @@ pub struct EncodingOptions { } impl EncodingOptions { - pub const UNCOMPRESSED: Self = Self { + pub const MSGPACK_UNCOMPRESSED: Self = Self { compression: Compression::Off, serializer: Serializer::MsgPack, }; - pub const COMPRESSED: Self = Self { + pub const MSGPACK_COMPRESSED: Self = Self { compression: Compression::LZ4, serializer: Serializer::MsgPack, }; - pub const PROTOBUF: Self = Self { + pub const PROTOBUF_COMPRESSED: Self = Self { compression: Compression::LZ4, serializer: Serializer::Protobuf, }; diff --git a/crates/store/re_sdk_comms/src/buffered_client.rs b/crates/store/re_sdk_comms/src/buffered_client.rs index 6a48e553bd4b..4616bcce5f7f 100644 --- a/crates/store/re_sdk_comms/src/buffered_client.rs +++ b/crates/store/re_sdk_comms/src/buffered_client.rs @@ -71,7 +71,7 @@ impl Client { // We don't compress the stream because we assume the SDK // and server are on the same machine and compression // can be expensive, see https://github.com/rerun-io/rerun/issues/2216 - let encoding_options = re_log_encoding::EncodingOptions::UNCOMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_UNCOMPRESSED; let encode_join = std::thread::Builder::new() .name("msg_encoder".into()) diff --git a/crates/top/re_sdk/src/binary_stream_sink.rs b/crates/top/re_sdk/src/binary_stream_sink.rs index a673ea24992a..102f453271aa 100644 --- a/crates/top/re_sdk/src/binary_stream_sink.rs +++ b/crates/top/re_sdk/src/binary_stream_sink.rs @@ -129,7 +129,7 @@ impl BinaryStreamSink { // We always compress when writing to a stream // TODO(jleibs): Make this configurable - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; let (tx, rx) = std::sync::mpsc::channel(); diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index b34289ac1d09..bc437f46ce35 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -1007,7 +1007,7 @@ fn stream_to_rrd_on_disk( re_log::info!("Saving incoming log stream to {path:?}. Abort with Ctrl-C."); - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; let file = std::fs::File::create(path).map_err(|err| FileSinkError::CreateFile(path.clone(), err))?; let mut encoder = re_log_encoding::encoder::DroppableEncoder::new( diff --git a/crates/top/rerun/src/commands/rrd/filter.rs b/crates/top/rerun/src/commands/rrd/filter.rs index f3334d2f9328..b1e9ca2c0e5e 100644 --- a/crates/top/rerun/src/commands/rrd/filter.rs +++ b/crates/top/rerun/src/commands/rrd/filter.rs @@ -83,7 +83,7 @@ impl FilterCommand { let mut encoder = { // TODO(cmc): encoding options & version should match the original. let version = CrateVersion::LOCAL; - let options = re_log_encoding::EncodingOptions::COMPRESSED; + let options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; re_log_encoding::encoder::DroppableEncoder::new(version, options, &mut rrd_out) .context("couldn't init encoder")? }; diff --git a/crates/top/rerun/src/commands/rrd/merge_compact.rs b/crates/top/rerun/src/commands/rrd/merge_compact.rs index 16911fa92510..82a2ffb3cd69 100644 --- a/crates/top/rerun/src/commands/rrd/merge_compact.rs +++ b/crates/top/rerun/src/commands/rrd/merge_compact.rs @@ -215,7 +215,7 @@ fn merge_and_compact( .flat_map(|entity_db| entity_db.to_messages(None /* time selection */)); // TODO(cmc): encoding options should match the original. - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; let version = entity_dbs .values() .next() diff --git a/crates/viewer/re_viewer/src/saving.rs b/crates/viewer/re_viewer/src/saving.rs index 142fda1997df..001912449730 100644 --- a/crates/viewer/re_viewer/src/saving.rs +++ b/crates/viewer/re_viewer/src/saving.rs @@ -67,7 +67,7 @@ pub fn encode_to_file( let mut file = std::fs::File::create(path) .with_context(|| format!("Failed to create file at {path:?}"))?; - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; re_log_encoding::encoder::encode(version, encoding_options, messages, &mut file) .map(|_| ()) .context("Message encode") From 0b56c95416487d2bff4cd733cf614aa6287c3830 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 14:25:55 +0100 Subject: [PATCH 28/47] more thorough testing --- Cargo.lock | 24 ++++ crates/store/re_log_encoding/Cargo.toml | 4 + .../store/re_log_encoding/src/codec/arrow.rs | 52 ++++++++ .../re_log_encoding/src/codec/file/decoder.rs | 37 ++---- .../re_log_encoding/src/codec/file/encoder.rs | 20 +-- .../store/re_log_encoding/src/decoder/mod.rs | 122 ++++++++++++------ crates/store/re_log_encoding/src/lib.rs | 2 + .../src/protobuf_conversions.rs | 17 +++ .../re_log_types/src/protobuf_conversions.rs | 2 +- .../re_protos/proto/rerun/v0/log_msg.proto | 10 +- .../re_protos/src/v0/rerun.log_msg.v0.rs | 10 +- 11 files changed, 204 insertions(+), 96 deletions(-) create mode 100644 crates/store/re_log_encoding/src/protobuf_conversions.rs diff --git a/Cargo.lock b/Cargo.lock index ea42c6d9aaf7..23d2158f05bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1819,6 +1819,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edf215dbb8cb1409cca7645aaed35f9e39fb0a21855bba1ac48bc0334903bf66" +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "dify" version = "0.7.4" @@ -5113,6 +5119,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "prettyplease" version = "0.2.25" @@ -5976,6 +5992,7 @@ dependencies = [ "lz4_flex", "mimalloc", "parking_lot", + "pretty_assertions", "re_arrow2", "re_build_info", "re_chunk", @@ -5985,6 +6002,7 @@ dependencies = [ "re_smart_channel", "re_tracing", "re_types", + "re_types_core", "rmp-serde", "serde_test", "thiserror", @@ -10232,6 +10250,12 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.7.5" diff --git a/crates/store/re_log_encoding/Cargo.toml b/crates/store/re_log_encoding/Cargo.toml index 5fd58ed1ae61..a88e09c15565 100644 --- a/crates/store/re_log_encoding/Cargo.toml +++ b/crates/store/re_log_encoding/Cargo.toml @@ -49,6 +49,7 @@ re_log.workspace = true re_protos.workspace = true re_smart_channel.workspace = true re_tracing.workspace = true +re_types_core.workspace = true # External: arrow2.workspace = true @@ -75,6 +76,9 @@ criterion.workspace = true mimalloc.workspace = true serde_test.workspace = true +# TEMP +pretty_assertions = "1.4.1" + [lib] bench = false diff --git a/crates/store/re_log_encoding/src/codec/arrow.rs b/crates/store/re_log_encoding/src/codec/arrow.rs index 5a1e70a29a01..6c2c69c75b57 100644 --- a/crates/store/re_log_encoding/src/codec/arrow.rs +++ b/crates/store/re_log_encoding/src/codec/arrow.rs @@ -1,3 +1,6 @@ +use crate::decoder::DecodeError; +use crate::encoder::EncodeError; + use super::CodecError; // TODO(#8412): try using arrow ipc `compression` option instead of doing our own compression @@ -52,3 +55,52 @@ pub(crate) fn read_arrow_from_bytes( ipc::read::StreamState::Some(chunk) => Ok((schema, chunk)), } } + +pub(crate) struct Payload { + pub uncompressed_size: usize, + pub data: Vec, +} + +pub(crate) fn encode_arrow( + schema: &arrow2::datatypes::Schema, + chunk: &arrow2::chunk::Chunk>, + compression: crate::Compression, +) -> Result { + let mut uncompressed = Vec::new(); + write_arrow_to_bytes(&mut uncompressed, schema, chunk)?; + let uncompressed_size = uncompressed.len(); + + let data = match compression { + crate::Compression::Off => uncompressed, + crate::Compression::LZ4 => lz4_flex::block::compress(&uncompressed), + }; + + Ok(Payload { + uncompressed_size, + data, + }) +} + +pub(crate) fn decode_arrow( + data: &[u8], + uncompressed_size: usize, + compression: crate::Compression, +) -> Result< + ( + arrow2::datatypes::Schema, + arrow2::chunk::Chunk>, + ), + DecodeError, +> { + let mut uncompressed = Vec::new(); + let data = match compression { + crate::Compression::Off => data, + crate::Compression::LZ4 => { + uncompressed.resize(uncompressed_size, 0); + lz4_flex::block::decompress_into(data, &mut uncompressed)?; + uncompressed.as_slice() + } + }; + + Ok(read_arrow_from_bytes(&mut &data[..])?) +} diff --git a/crates/store/re_log_encoding/src/codec/file/decoder.rs b/crates/store/re_log_encoding/src/codec/file/decoder.rs index e6b840ac6c8d..fabe6795d2fd 100644 --- a/crates/store/re_log_encoding/src/codec/file/decoder.rs +++ b/crates/store/re_log_encoding/src/codec/file/decoder.rs @@ -1,7 +1,7 @@ use super::{MessageHeader, MessageKind}; -use crate::codec::arrow::read_arrow_from_bytes; +use crate::codec::arrow::decode_arrow; +use crate::codec::CodecError; use crate::decoder::DecodeError; -use crate::{codec::CodecError, Compression}; use re_log_types::LogMsg; use re_protos::TypeConversionError; @@ -31,10 +31,7 @@ impl MessageHeader { } } -pub(crate) fn decode( - data: &mut impl std::io::Read, - compression: Compression, -) -> Result<(u64, Option), DecodeError> { +pub(crate) fn decode(data: &mut impl std::io::Read) -> Result<(u64, Option), DecodeError> { use re_protos::external::prost::Message; use re_protos::log_msg::v0::{ArrowMsg, BlueprintActivationCommand, Encoding, SetStoreInfo}; @@ -56,7 +53,11 @@ pub(crate) fn decode( return Err(DecodeError::Codec(CodecError::UnsupportedEncoding)); } - let (schema, chunk) = decode_arrow(&arrow_msg.payload, compression)?; + let (schema, chunk) = decode_arrow( + &arrow_msg.payload, + arrow_msg.uncompressed_size as usize, + arrow_msg.compression().into(), + )?; let store_id: re_log_types::StoreId = arrow_msg .store_id @@ -83,25 +84,3 @@ pub(crate) fn decode( Ok((read_bytes, msg)) } - -fn decode_arrow( - data: &[u8], - compression: crate::Compression, -) -> Result< - ( - arrow2::datatypes::Schema, - arrow2::chunk::Chunk>, - ), - DecodeError, -> { - let mut uncompressed = Vec::new(); - let data = match compression { - crate::Compression::Off => data, - crate::Compression::LZ4 => { - lz4_flex::block::decompress_into(data, &mut uncompressed)?; - uncompressed.as_slice() - } - }; - - Ok(read_arrow_from_bytes(&mut &data[..])?) -} diff --git a/crates/store/re_log_encoding/src/codec/file/encoder.rs b/crates/store/re_log_encoding/src/codec/file/encoder.rs index 6868e9ca792a..dce303b1caa9 100644 --- a/crates/store/re_log_encoding/src/codec/file/encoder.rs +++ b/crates/store/re_log_encoding/src/codec/file/encoder.rs @@ -1,5 +1,5 @@ use super::{MessageHeader, MessageKind}; -use crate::codec::arrow::write_arrow_to_bytes; +use crate::codec::arrow::encode_arrow; use crate::encoder::EncodeError; use crate::Compression; use re_log_types::LogMsg; @@ -41,14 +41,16 @@ pub(crate) fn encode( set_store_info.encode(buf)?; } LogMsg::ArrowMsg(store_id, arrow_msg) => { + let payload = encode_arrow(&arrow_msg.schema, &arrow_msg.chunk, compression)?; let arrow_msg = ArrowMsg { store_id: Some(store_id.clone().into()), compression: match compression { Compression::Off => proto::Compression::None as i32, Compression::LZ4 => proto::Compression::Lz4 as i32, }, + uncompressed_size: payload.uncompressed_size as i32, encoding: Encoding::ArrowIpc as i32, - payload: encode_arrow(&arrow_msg.schema, &arrow_msg.chunk, compression)?, + payload: payload.data, }; let header = MessageHeader { kind: MessageKind::ArrowMsg, @@ -71,17 +73,3 @@ pub(crate) fn encode( Ok(()) } - -fn encode_arrow( - schema: &arrow2::datatypes::Schema, - chunk: &arrow2::chunk::Chunk>, - compression: crate::Compression, -) -> Result, EncodeError> { - let mut uncompressed = Vec::new(); - write_arrow_to_bytes(&mut uncompressed, schema, chunk)?; - - match compression { - crate::Compression::Off => Ok(uncompressed), - crate::Compression::LZ4 => Ok(lz4_flex::block::compress(&uncompressed)), - } -} diff --git a/crates/store/re_log_encoding/src/decoder/mod.rs b/crates/store/re_log_encoding/src/decoder/mod.rs index b909306cece6..e313473565e7 100644 --- a/crates/store/re_log_encoding/src/decoder/mod.rs +++ b/crates/store/re_log_encoding/src/decoder/mod.rs @@ -294,15 +294,13 @@ impl Iterator for Decoder { } let msg = match self.options.serializer { - Serializer::Protobuf => { - match decoder::decode(&mut self.read, self.options.compression) { - Ok((read_bytes, msg)) => { - self.size_bytes += read_bytes; - msg - } - Err(err) => return Some(Err(err)), + Serializer::Protobuf => match decoder::decode(&mut self.read) { + Ok((read_bytes, msg)) => { + self.size_bytes += read_bytes; + msg } - } + Err(err) => return Some(Err(err)), + }, Serializer::MsgPack => { let header = match MessageHeader::decode(&mut self.read) { Ok(header) => header, @@ -408,32 +406,73 @@ mod tests { use re_build_info::CrateVersion; use re_chunk::RowId; use re_log_types::{ - ApplicationId, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, + ApplicationId, ArrowMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; - fn fake_log_message() -> LogMsg { - LogMsg::SetStoreInfo(SetStoreInfo { - row_id: *RowId::new(), - info: StoreInfo { - application_id: ApplicationId("test".to_owned()), - store_id: StoreId::random(StoreKind::Recording), - cloned_from: None, - is_official_example: true, - started: Time::now(), - store_source: StoreSource::RustSdk { - rustc_version: String::new(), - llvm_version: String::new(), + use pretty_assertions::assert_eq; + + fn fake_log_messages() -> Vec { + let store_id = StoreId::random(StoreKind::Blueprint); + vec![ + LogMsg::SetStoreInfo(SetStoreInfo { + row_id: *RowId::new(), + info: StoreInfo { + application_id: ApplicationId("test".to_owned()), + store_id: store_id.clone(), + cloned_from: None, + is_official_example: true, + started: Time::now(), + store_source: StoreSource::RustSdk { + rustc_version: String::new(), + llvm_version: String::new(), + }, + store_version: Some(CrateVersion::LOCAL), }, - store_version: Some(CrateVersion::LOCAL), - }, - }) + }), + LogMsg::ArrowMsg( + store_id.clone(), + re_chunk::Chunk::builder("test_entity".into()) + .with_archetype( + re_chunk::RowId::new(), + re_log_types::TimePoint::default().with( + re_log_types::Timeline::new_sequence("blueprint"), + re_log_types::TimeInt::from_milliseconds(re_log_types::NonMinI64::MIN), + ), + &re_types::blueprint::archetypes::Background::new( + re_types::blueprint::components::BackgroundKind::SolidColor, + ) + .with_color([255, 0, 0]), + ) + .build() + .unwrap() + .to_arrow_msg() + .unwrap(), + ), + LogMsg::BlueprintActivationCommand(re_log_types::BlueprintActivationCommand { + blueprint_id: store_id, + make_active: true, + make_default: true, + }), + ] + } + + fn clear_arrow_extension_metadata(messages: &mut Vec) { + for msg in messages { + if let LogMsg::ArrowMsg(_, arrow_msg) = msg { + for field in &mut arrow_msg.schema.fields { + field + .metadata + .retain(|k, _| !k.starts_with("ARROW:extension")); + } + } + } } #[test] fn test_encode_decode() { let rrd_version = CrateVersion::LOCAL; - let messages = vec![fake_log_message()]; + let messages = fake_log_messages(); let options = [ EncodingOptions { @@ -459,11 +498,13 @@ mod tests { crate::encoder::encode_ref(rrd_version, options, messages.iter().map(Ok), &mut file) .unwrap(); - let decoded_messages = Decoder::new(VersionPolicy::Error, &mut file.as_slice()) + let mut decoded_messages = Decoder::new(VersionPolicy::Error, &mut file.as_slice()) .unwrap() .collect::, DecodeError>>() .unwrap(); + clear_arrow_extension_metadata(&mut decoded_messages); + assert_eq!(messages, decoded_messages); } } @@ -490,22 +531,20 @@ mod tests { ]; for options in options { + println!("{options:?}"); + let mut data = vec![]; // write "2 files" i.e. 2 streams that end with end-of-stream marker - let messages = vec![ - fake_log_message(), - fake_log_message(), - fake_log_message(), - fake_log_message(), - ]; + let messages = fake_log_messages(); // (2 encoders as each encoder writes a file header) let writer = std::io::Cursor::new(&mut data); let mut encoder1 = crate::encoder::Encoder::new(CrateVersion::LOCAL, options, writer).unwrap(); - encoder1.append(&messages[0]).unwrap(); - encoder1.append(&messages[1]).unwrap(); + for message in &messages { + encoder1.append(message).unwrap(); + } encoder1.finish().unwrap(); let written = data.len() as u64; @@ -513,9 +552,9 @@ mod tests { writer.set_position(written); let mut encoder2 = crate::encoder::Encoder::new(CrateVersion::LOCAL, options, writer).unwrap(); - - encoder2.append(&messages[2]).unwrap(); - encoder2.append(&messages[3]).unwrap(); + for message in &messages { + encoder2.append(message).unwrap(); + } encoder2.finish().unwrap(); let decoder = Decoder::new_concatenated( @@ -524,12 +563,11 @@ mod tests { ) .unwrap(); - let mut decoded_messages = vec![]; - for msg in decoder { - decoded_messages.push(msg.unwrap()); - } + let mut decoded_messages = decoder.into_iter().collect::, _>>().unwrap(); - assert_eq!(messages, decoded_messages); + clear_arrow_extension_metadata(&mut decoded_messages); + + assert_eq!(vec![messages.clone(), messages].concat(), decoded_messages); } } } diff --git a/crates/store/re_log_encoding/src/lib.rs b/crates/store/re_log_encoding/src/lib.rs index b2ae061e35fa..d35b8e331342 100644 --- a/crates/store/re_log_encoding/src/lib.rs +++ b/crates/store/re_log_encoding/src/lib.rs @@ -8,6 +8,8 @@ pub mod encoder; pub mod codec; +mod protobuf_conversions; + #[cfg(feature = "encoder")] #[cfg(not(target_arch = "wasm32"))] mod file_sink; diff --git a/crates/store/re_log_encoding/src/protobuf_conversions.rs b/crates/store/re_log_encoding/src/protobuf_conversions.rs new file mode 100644 index 000000000000..9b613a1e3937 --- /dev/null +++ b/crates/store/re_log_encoding/src/protobuf_conversions.rs @@ -0,0 +1,17 @@ +impl From for crate::Compression { + fn from(value: re_protos::log_msg::v0::Compression) -> Self { + match value { + re_protos::log_msg::v0::Compression::None => Self::Off, + re_protos::log_msg::v0::Compression::Lz4 => Self::LZ4, + } + } +} + +impl From for re_protos::log_msg::v0::Compression { + fn from(value: crate::Compression) -> Self { + match value { + crate::Compression::Off => Self::None, + crate::Compression::LZ4 => Self::Lz4, + } + } +} diff --git a/crates/store/re_log_types/src/protobuf_conversions.rs b/crates/store/re_log_types/src/protobuf_conversions.rs index 0f4675a112b8..5bb66da2a9c7 100644 --- a/crates/store/re_log_types/src/protobuf_conversions.rs +++ b/crates/store/re_log_types/src/protobuf_conversions.rs @@ -154,7 +154,7 @@ impl From for crate::StoreId { #[inline] fn from(value: re_protos::common::v0::StoreId) -> Self { Self { - kind: crate::StoreKind::Recording, + kind: value.kind().into(), id: Arc::new(value.id), } } diff --git a/crates/store/re_protos/proto/rerun/v0/log_msg.proto b/crates/store/re_protos/proto/rerun/v0/log_msg.proto index c440dd9b230e..d9d9c3d7256a 100644 --- a/crates/store/re_protos/proto/rerun/v0/log_msg.proto +++ b/crates/store/re_protos/proto/rerun/v0/log_msg.proto @@ -39,8 +39,10 @@ message ArrowMsg { // Compression algorithm used. Compression compression = 2; + int32 uncompressed_size = 3; + // Encoding of the payload. - Encoding encoding = 3; + Encoding encoding = 4; // Arrow-IPC encoded schema and chunk, compressed according to the `compression` field. bytes payload = 1000; @@ -136,9 +138,9 @@ enum StoreSourceKind { // Version of the Python SDK that created the recording. message PythonVersion { - sint32 major = 1; - sint32 minor = 2; - sint32 patch = 3; + int32 major = 1; + int32 minor = 2; + int32 patch = 3; string suffix = 4; } diff --git a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs index de25726319c9..08cbb32ee443 100644 --- a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs @@ -18,8 +18,10 @@ pub struct ArrowMsg { /// Compression algorithm used. #[prost(enumeration = "Compression", tag = "2")] pub compression: i32, + #[prost(int32, tag = "3")] + pub uncompressed_size: i32, /// Encoding of the payload. - #[prost(enumeration = "Encoding", tag = "3")] + #[prost(enumeration = "Encoding", tag = "4")] pub encoding: i32, /// Arrow-IPC encoded schema and chunk, compressed according to the `compression` field. #[prost(bytes = "vec", tag = "1000")] @@ -82,11 +84,11 @@ pub struct StoreSourceExtra { /// Version of the Python SDK that created the recording. #[derive(Clone, PartialEq, ::prost::Message)] pub struct PythonVersion { - #[prost(sint32, tag = "1")] + #[prost(int32, tag = "1")] pub major: i32, - #[prost(sint32, tag = "2")] + #[prost(int32, tag = "2")] pub minor: i32, - #[prost(sint32, tag = "3")] + #[prost(int32, tag = "3")] pub patch: i32, #[prost(string, tag = "4")] pub suffix: ::prost::alloc::string::String, From b1d8ffa0672215c454e43bf2bdf1b75538b7cd93 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 15:04:26 +0100 Subject: [PATCH 29/47] add conversion unit tests --- .../re_log_types/src/protobuf_conversions.rs | 231 +++++++++++++++++- .../re_protos/proto/rerun/v0/log_msg.proto | 10 + .../re_protos/src/v0/rerun.log_msg.v0.rs | 11 + .../utils/re_tuid/src/protobuf_conversions.rs | 11 + 4 files changed, 250 insertions(+), 13 deletions(-) diff --git a/crates/store/re_log_types/src/protobuf_conversions.rs b/crates/store/re_log_types/src/protobuf_conversions.rs index 5bb66da2a9c7..00f2b0d0ae92 100644 --- a/crates/store/re_log_types/src/protobuf_conversions.rs +++ b/crates/store/re_log_types/src/protobuf_conversions.rs @@ -90,27 +90,39 @@ impl TryFrom for crate::ResolvedTimeRange { } } +impl From for crate::Timeline { + fn from(value: re_protos::common::v0::Timeline) -> Self { + // TODO(cmc): QueryExpression::filtered_index gotta be a selector + #[allow(clippy::match_same_arms)] + match value.name.as_str() { + "log_time" => Self::new_temporal(value.name), + "log_tick" => Self::new_sequence(value.name), + "frame" => Self::new_sequence(value.name), + "frame_nr" => Self::new_sequence(value.name), + _ => Self::new_temporal(value.name), + } + } +} + +impl From for re_protos::common::v0::Timeline { + fn from(value: crate::Timeline) -> Self { + Self { + name: value.name().to_string(), + } + } +} + impl TryFrom for crate::Timeline { type Error = TypeConversionError; fn try_from(value: re_protos::common::v0::IndexColumnSelector) -> Result { - let timeline_name = value + let timeline = value .timeline .ok_or(TypeConversionError::missing_field( "rerun.common.v0.IndexColumnSelector", "timeline", ))? - .name; - - // TODO(cmc): QueryExpression::filtered_index gotta be a selector - #[allow(clippy::match_same_arms)] - let timeline = match timeline_name.as_str() { - "log_time" => Self::new_temporal(timeline_name), - "log_tick" => Self::new_sequence(timeline_name), - "frame" => Self::new_sequence(timeline_name), - "frame_nr" => Self::new_sequence(timeline_name), - _ => Self::new_temporal(timeline_name), - }; + .into(); Ok(timeline) } @@ -388,6 +400,11 @@ impl From for re_protos::log_msg::v0::StoreInfo { is_official_example: value.is_official_example, started: Some(value.started.into()), store_source: Some(value.store_source.into()), + store_version: value + .store_version + .map(|v| re_protos::log_msg::v0::StoreVersion { + crate_version_bits: i32::from_le_bytes(v.to_bytes()), + }), } } } @@ -426,6 +443,9 @@ impl TryFrom for crate::StoreInfo { "store_source", ))? .try_into()?; + let store_version = value + .store_version + .map(|v| re_build_info::CrateVersion::from_bytes(v.crate_version_bits.to_le_bytes())); Ok(Self { application_id, @@ -434,7 +454,7 @@ impl TryFrom for crate::StoreInfo { is_official_example, started, store_source, - store_version: Some(re_build_info::CrateVersion::LOCAL), + store_version, }) } } @@ -508,3 +528,188 @@ impl TryFrom }) } } + +#[cfg(test)] +mod tests { + #[test] + fn entity_path_conversion() { + let entity_path = crate::EntityPath::parse_strict("a/b/c").unwrap(); + let proto_entity_path: re_protos::common::v0::EntityPath = entity_path.clone().into(); + let entity_path2: crate::EntityPath = proto_entity_path.try_into().unwrap(); + assert_eq!(entity_path, entity_path2); + } + + #[test] + fn time_conversion() { + let time = crate::Time::from_ns_since_epoch(123456789); + let proto_time: re_protos::common::v0::Time = time.clone().into(); + let time2: crate::Time = proto_time.into(); + assert_eq!(time, time2); + } + + #[test] + fn time_int_conversion() { + let time_int = crate::TimeInt::new_temporal(123456789); + let proto_time_int: re_protos::common::v0::TimeInt = time_int.clone().into(); + let time_int2: crate::TimeInt = proto_time_int.into(); + assert_eq!(time_int, time_int2); + } + + #[test] + fn time_range_conversion() { + let time_range = crate::ResolvedTimeRange::new( + crate::TimeInt::new_temporal(123456789), + crate::TimeInt::new_temporal(987654321), + ); + let proto_time_range: re_protos::common::v0::TimeRange = time_range.clone().into(); + let time_range2: crate::ResolvedTimeRange = proto_time_range.into(); + assert_eq!(time_range, time_range2); + } + + #[test] + fn index_range_conversion() { + let time_range = crate::ResolvedTimeRange::new( + crate::TimeInt::new_temporal(123456789), + crate::TimeInt::new_temporal(987654321), + ); + let proto_index_range: re_protos::common::v0::IndexRange = time_range.clone().into(); + let time_range2: crate::ResolvedTimeRange = proto_index_range.try_into().unwrap(); + assert_eq!(time_range, time_range2); + } + + #[test] + fn index_column_selector_conversion() { + let timeline = crate::Timeline::new_temporal("log_time"); + let proto_index_column_selector: re_protos::common::v0::IndexColumnSelector = + re_protos::common::v0::IndexColumnSelector { + timeline: Some(timeline.into()), + }; + let timeline2: crate::Timeline = proto_index_column_selector.try_into().unwrap(); + assert_eq!(timeline, timeline2); + } + + #[test] + fn application_id_conversion() { + let application_id = crate::ApplicationId("test".to_owned()); + let proto_application_id: re_protos::common::v0::ApplicationId = + application_id.clone().into(); + let application_id2: crate::ApplicationId = proto_application_id.into(); + assert_eq!(application_id, application_id2); + } + + #[test] + fn store_kind_conversion() { + let store_kind = crate::StoreKind::Recording; + let proto_store_kind: re_protos::common::v0::StoreKind = store_kind.clone().into(); + let store_kind2: crate::StoreKind = proto_store_kind.into(); + assert_eq!(store_kind, store_kind2); + } + + #[test] + fn store_id_conversion() { + let store_id = + crate::StoreId::from_string(crate::StoreKind::Recording, "test_recording".to_owned()); + let proto_store_id: re_protos::common::v0::StoreId = store_id.clone().into(); + let store_id2: crate::StoreId = proto_store_id.into(); + assert_eq!(store_id, store_id2); + } + + #[test] + fn recording_id_conversion() { + let store_id = + crate::StoreId::from_string(crate::StoreKind::Recording, "test_recording".to_owned()); + let proto_recording_id: re_protos::common::v0::RecordingId = store_id.clone().into(); + let store_id2: crate::StoreId = proto_recording_id.into(); + assert_eq!(store_id, store_id2); + } + + #[test] + fn store_source_conversion() { + let store_source = crate::StoreSource::PythonSdk(crate::PythonVersion { + major: 3, + minor: 8, + patch: 0, + suffix: "a".to_owned(), + }); + let proto_store_source: re_protos::log_msg::v0::StoreSource = store_source.clone().into(); + let store_source2: crate::StoreSource = proto_store_source.try_into().unwrap(); + assert_eq!(store_source, store_source2); + } + + #[test] + fn file_source_conversion() { + let file_source = crate::FileSource::Uri; + let proto_file_source: re_protos::log_msg::v0::FileSource = file_source.clone().into(); + let file_source2: crate::FileSource = proto_file_source.try_into().unwrap(); + assert_eq!(file_source, file_source2); + } + + #[test] + fn store_info_conversion() { + let store_info = crate::StoreInfo { + application_id: crate::ApplicationId("test".to_owned()), + store_id: crate::StoreId::from_string( + crate::StoreKind::Recording, + "test_recording".to_owned(), + ), + cloned_from: None, + is_official_example: false, + started: crate::Time::now(), + store_source: crate::StoreSource::PythonSdk(crate::PythonVersion { + major: 3, + minor: 8, + patch: 0, + suffix: "a".to_owned(), + }), + store_version: None, + }; + let proto_store_info: re_protos::log_msg::v0::StoreInfo = store_info.clone().into(); + let store_info2: crate::StoreInfo = proto_store_info.try_into().unwrap(); + assert_eq!(store_info, store_info2); + } + + #[test] + fn set_store_info_conversion() { + let set_store_info = crate::SetStoreInfo { + row_id: re_tuid::Tuid::new(), + info: crate::StoreInfo { + application_id: crate::ApplicationId("test".to_owned()), + store_id: crate::StoreId::from_string( + crate::StoreKind::Recording, + "test_recording".to_owned(), + ), + cloned_from: None, + is_official_example: false, + started: crate::Time::now(), + store_source: crate::StoreSource::PythonSdk(crate::PythonVersion { + major: 3, + minor: 8, + patch: 0, + suffix: "a".to_owned(), + }), + store_version: None, + }, + }; + let proto_set_store_info: re_protos::log_msg::v0::SetStoreInfo = + set_store_info.clone().into(); + let set_store_info2: crate::SetStoreInfo = proto_set_store_info.try_into().unwrap(); + assert_eq!(set_store_info, set_store_info2); + } + + #[test] + fn blueprint_activation_command_conversion() { + let blueprint_activation_command = crate::BlueprintActivationCommand { + blueprint_id: crate::StoreId::from_string( + crate::StoreKind::Blueprint, + "test".to_owned(), + ), + make_active: true, + make_default: false, + }; + let proto_blueprint_activation_command: re_protos::log_msg::v0::BlueprintActivationCommand = + blueprint_activation_command.clone().into(); + let blueprint_activation_command2: crate::BlueprintActivationCommand = + proto_blueprint_activation_command.try_into().unwrap(); + assert_eq!(blueprint_activation_command, blueprint_activation_command2); + } +} diff --git a/crates/store/re_protos/proto/rerun/v0/log_msg.proto b/crates/store/re_protos/proto/rerun/v0/log_msg.proto index d9d9c3d7256a..b35abec50a58 100644 --- a/crates/store/re_protos/proto/rerun/v0/log_msg.proto +++ b/crates/store/re_protos/proto/rerun/v0/log_msg.proto @@ -80,6 +80,9 @@ message StoreInfo { // Where the recording came from. StoreSource store_source = 5; + + // Version of the store crate. + StoreVersion store_version = 6; } // The source of a recording or blueprint. @@ -178,3 +181,10 @@ enum FileSourceKind { // The recording was produced using a data loader, such as when logging a mesh file. SDK = 5; } + +message StoreVersion { + // Crate version encoded using our custom scheme. + // + // See `CrateVersion` in `re_build_info`. + int32 crate_version_bits = 1; +} diff --git a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs index 08cbb32ee443..3a0044e099b4 100644 --- a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs @@ -62,6 +62,9 @@ pub struct StoreInfo { /// Where the recording came from. #[prost(message, optional, tag = "5")] pub store_source: ::core::option::Option, + /// Version of the store crate. + #[prost(message, optional, tag = "6")] + pub store_version: ::core::option::Option, } /// The source of a recording or blueprint. #[derive(Clone, PartialEq, ::prost::Message)] @@ -109,6 +112,14 @@ pub struct FileSource { #[prost(enumeration = "FileSourceKind", tag = "1")] pub kind: i32, } +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct StoreVersion { + /// Crate version encoded using our custom scheme. + /// + /// See `CrateVersion` in `re_build_info`. + #[prost(int32, tag = "1")] + pub crate_version_bits: i32, +} /// The type of compression used on the payload. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/crates/utils/re_tuid/src/protobuf_conversions.rs b/crates/utils/re_tuid/src/protobuf_conversions.rs index 2dfd04627d4b..a04dba09ddbf 100644 --- a/crates/utils/re_tuid/src/protobuf_conversions.rs +++ b/crates/utils/re_tuid/src/protobuf_conversions.rs @@ -15,3 +15,14 @@ impl From for re_protos::common::v0::Tuid { } } } + +#[cfg(test)] +mod tests { + #[test] + fn test_tuid_conversion() { + let tuid = crate::Tuid::new(); + let proto_tuid: re_protos::common::v0::Tuid = tuid.into(); + let tuid2: crate::Tuid = proto_tuid.into(); + assert_eq!(tuid, tuid2); + } +} From 3c9a6826abb90aca617eebf5bc9a3ee8dbe58bce Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 15:05:17 +0100 Subject: [PATCH 30/47] fix compile error --- crates/viewer/re_viewer/src/app.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/viewer/re_viewer/src/app.rs b/crates/viewer/re_viewer/src/app.rs index b8d36e4d2899..1a5c4f1e4639 100644 --- a/crates/viewer/re_viewer/src/app.rs +++ b/crates/viewer/re_viewer/src/app.rs @@ -2283,7 +2283,7 @@ async fn async_save_dialog( let bytes = re_log_encoding::encoder::encode_as_bytes( rrd_version, - re_log_encoding::EncodingOptions::COMPRESSED, + re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED, messages, )?; file_handle.write(&bytes).await.context("Failed to save") From 21258a0aa04df49a2b2cf13d9b621c1f1c56420d Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 15:07:52 +0100 Subject: [PATCH 31/47] remove temp --- Cargo.lock | 23 ------------------- crates/store/re_log_encoding/Cargo.toml | 3 --- .../store/re_log_encoding/src/decoder/mod.rs | 2 -- 3 files changed, 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23d2158f05bc..910cede30193 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1819,12 +1819,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edf215dbb8cb1409cca7645aaed35f9e39fb0a21855bba1ac48bc0334903bf66" -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - [[package]] name = "dify" version = "0.7.4" @@ -5119,16 +5113,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "pretty_assertions" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" -dependencies = [ - "diff", - "yansi", -] - [[package]] name = "prettyplease" version = "0.2.25" @@ -5992,7 +5976,6 @@ dependencies = [ "lz4_flex", "mimalloc", "parking_lot", - "pretty_assertions", "re_arrow2", "re_build_info", "re_chunk", @@ -10250,12 +10233,6 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984" -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "yoke" version = "0.7.5" diff --git a/crates/store/re_log_encoding/Cargo.toml b/crates/store/re_log_encoding/Cargo.toml index a88e09c15565..20ac933112e1 100644 --- a/crates/store/re_log_encoding/Cargo.toml +++ b/crates/store/re_log_encoding/Cargo.toml @@ -76,9 +76,6 @@ criterion.workspace = true mimalloc.workspace = true serde_test.workspace = true -# TEMP -pretty_assertions = "1.4.1" - [lib] bench = false diff --git a/crates/store/re_log_encoding/src/decoder/mod.rs b/crates/store/re_log_encoding/src/decoder/mod.rs index e313473565e7..e9eefd14b348 100644 --- a/crates/store/re_log_encoding/src/decoder/mod.rs +++ b/crates/store/re_log_encoding/src/decoder/mod.rs @@ -409,8 +409,6 @@ mod tests { ApplicationId, ArrowMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; - use pretty_assertions::assert_eq; - fn fake_log_messages() -> Vec { let store_id = StoreId::random(StoreKind::Blueprint); vec![ From d1dc3f838d6ae59fe41c472c46f316dc03b60253 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 20:45:43 +0100 Subject: [PATCH 32/47] use types instead of strings --- Cargo.lock | 1 + Cargo.toml | 1 + crates/build/re_protos_builder/Cargo.toml | 1 + crates/build/re_protos_builder/src/lib.rs | 5 +- .../src/protobuf_conversions.rs | 34 ++- .../re_log_encoding/src/codec/file/decoder.rs | 6 +- .../store/re_log_encoding/src/decoder/mod.rs | 6 +- .../re_log_types/src/protobuf_conversions.rs | 95 ++++--- crates/store/re_protos/src/lib.rs | 31 ++- .../store/re_protos/src/v0/rerun.common.v0.rs | 240 ++++++++++++++++++ .../re_protos/src/v0/rerun.log_msg.v0.rs | 100 ++++++++ .../re_protos/src/v0/rerun.remote_store.v0.rs | 120 +++++++++ 12 files changed, 561 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7af1bc0dd16b..c6b543b04047 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6094,6 +6094,7 @@ name = "re_protos_builder" version = "0.21.0-alpha.1+dev" dependencies = [ "camino", + "prost-build", "re_log", "tonic-build", ] diff --git a/Cargo.toml b/Cargo.toml index 7e6a16ea007a..266a827a965f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -242,6 +242,7 @@ prettyplease = "0.2" proc-macro2 = { version = "1.0", default-features = false } profiling = { version = "1.0.12", default-features = false } prost = "0.13.3" +prost-build = "0.13.3" puffin = "0.19.1" puffin_http = "0.16" pyo3 = "0.22.5" diff --git a/crates/build/re_protos_builder/Cargo.toml b/crates/build/re_protos_builder/Cargo.toml index 87ab72d8efb8..266537383c4e 100644 --- a/crates/build/re_protos_builder/Cargo.toml +++ b/crates/build/re_protos_builder/Cargo.toml @@ -29,3 +29,4 @@ camino.workspace = true tonic-build = { workspace = true, default-features = false, features = [ "prost", ] } +prost-build = { workspace = true, default-features = false } diff --git a/crates/build/re_protos_builder/src/lib.rs b/crates/build/re_protos_builder/src/lib.rs index a1d2f744e1a0..6f382e89fb48 100644 --- a/crates/build/re_protos_builder/src/lib.rs +++ b/crates/build/re_protos_builder/src/lib.rs @@ -16,12 +16,15 @@ pub fn generate_rust_code( proto_paths: &[impl AsRef], output_dir: impl AsRef, ) { + let mut prost_config = prost_build::Config::new(); + prost_config.enable_type_names(); // tonic doesn't expose this option + if let Err(err) = tonic_build::configure() .out_dir(output_dir.as_ref()) .build_client(true) .build_server(true) .build_transport(false) // Small convenience, but doesn't work on web - .compile_protos(proto_paths, &[definitions_dir]) + .compile_protos_with_config(prost_config, proto_paths, &[definitions_dir]) { match err.kind() { std::io::ErrorKind::Other => { diff --git a/crates/store/re_chunk_store/src/protobuf_conversions.rs b/crates/store/re_chunk_store/src/protobuf_conversions.rs index 67ef038284e0..4f37e7b3bf5a 100644 --- a/crates/store/re_chunk_store/src/protobuf_conversions.rs +++ b/crates/store/re_chunk_store/src/protobuf_conversions.rs @@ -1,8 +1,8 @@ +use re_protos::missing_field; +use re_protos::TypeConversionError; use std::collections::BTreeMap; use std::collections::BTreeSet; -use re_protos::TypeConversionError; - impl TryFrom for crate::ComponentColumnSelector { type Error = TypeConversionError; @@ -11,16 +11,16 @@ impl TryFrom for crate::Componen ) -> Result { let entity_path = value .entity_path - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.ComponentColumnSelector", + .ok_or(missing_field!( + re_protos::common::v0::ComponentColumnSelector, "entity_path", ))? .try_into()?; let component_name = value .component - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.ComponentColumnSelector", + .ok_or(missing_field!( + re_protos::common::v0::ComponentColumnSelector, "component", ))? .name; @@ -36,8 +36,8 @@ impl TryFrom for crate::TimeColumnSel type Error = TypeConversionError; fn try_from(value: re_protos::common::v0::TimeColumnSelector) -> Result { - let timeline = value.timeline.ok_or(TypeConversionError::missing_field( - "rerun.common.v0.TimeColumnSelector", + let timeline = value.timeline.ok_or(missing_field!( + re_protos::common::v0::TimeColumnSelector, "timeline", ))?; @@ -51,12 +51,10 @@ impl TryFrom for crate::ColumnSelector { type Error = TypeConversionError; fn try_from(value: re_protos::common::v0::ColumnSelector) -> Result { - match value - .selector_type - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.ColumnSelector", - "selector_type", - ))? { + match value.selector_type.ok_or(missing_field!( + re_protos::common::v0::ColumnSelector, + "selector_type", + ))? { re_protos::common::v0::column_selector::SelectorType::ComponentColumn( component_column_selector, ) => { @@ -115,8 +113,8 @@ impl TryFrom for crate::ViewContentsSelecto .map(|part| { let entity_path: re_log_types::EntityPath = part .path - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.ViewContentsPart", + .ok_or(missing_field!( + re_protos::common::v0::ViewContentsPart, "path", ))? .try_into()?; @@ -139,8 +137,8 @@ impl TryFrom for crate::QueryExpression { fn try_from(value: re_protos::common::v0::Query) -> Result { let filtered_index = value .filtered_index - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.Query", + .ok_or(missing_field!( + re_protos::common::v0::Query, "filtered_index", ))? .try_into()?; diff --git a/crates/store/re_log_encoding/src/codec/file/decoder.rs b/crates/store/re_log_encoding/src/codec/file/decoder.rs index fabe6795d2fd..3ef7112b808c 100644 --- a/crates/store/re_log_encoding/src/codec/file/decoder.rs +++ b/crates/store/re_log_encoding/src/codec/file/decoder.rs @@ -3,7 +3,7 @@ use crate::codec::arrow::decode_arrow; use crate::codec::CodecError; use crate::decoder::DecodeError; use re_log_types::LogMsg; -use re_protos::TypeConversionError; +use re_protos::missing_field; impl MessageKind { pub(crate) fn decode(data: &mut impl std::io::Read) -> Result { @@ -61,9 +61,7 @@ pub(crate) fn decode(data: &mut impl std::io::Read) -> Result<(u64, Option Iterator for Decoder { #[cfg(all(test, feature = "decoder", feature = "encoder"))] mod tests { + #![allow(clippy::unwrap_used)] // acceptable for tests + use super::*; use re_build_info::CrateVersion; use re_chunk::RowId; use re_log_types::{ - ApplicationId, ArrowMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, + ApplicationId, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; fn fake_log_messages() -> Vec { @@ -565,7 +567,7 @@ mod tests { clear_arrow_extension_metadata(&mut decoded_messages); - assert_eq!(vec![messages.clone(), messages].concat(), decoded_messages); + assert_eq!([messages.clone(), messages].concat(), decoded_messages); } } } diff --git a/crates/store/re_log_types/src/protobuf_conversions.rs b/crates/store/re_log_types/src/protobuf_conversions.rs index 00f2b0d0ae92..e6dcc56d3bb0 100644 --- a/crates/store/re_log_types/src/protobuf_conversions.rs +++ b/crates/store/re_log_types/src/protobuf_conversions.rs @@ -1,6 +1,7 @@ -use std::sync::Arc; - +use re_protos::invalid_field; +use re_protos::missing_field; use re_protos::TypeConversionError; +use std::sync::Arc; impl From for re_protos::common::v0::EntityPath { fn from(value: crate::EntityPath) -> Self { @@ -82,9 +83,9 @@ impl TryFrom for crate::ResolvedTimeRange { fn try_from(value: re_protos::common::v0::IndexRange) -> Result { value .time_range - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.IndexRange", - "time_range", + .ok_or(missing_field!( + re_protos::common::v0::IndexRange, + "time_range" )) .map(|time_range| Self::new(time_range.start, time_range.end)) } @@ -118,9 +119,9 @@ impl TryFrom for crate::Timeline { fn try_from(value: re_protos::common::v0::IndexColumnSelector) -> Result { let timeline = value .timeline - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.IndexColumnSelector", - "timeline", + .ok_or(missing_field!( + re_protos::common::v0::IndexColumnSelector, + "timeline" ))? .into(); @@ -264,10 +265,9 @@ impl TryFrom for crate::StoreSource { StoreSourceKind::UnknownKind => Ok(Self::Unknown), StoreSourceKind::CSdk => Ok(Self::CSdk), StoreSourceKind::PythonSdk => { - let extra = value.extra.ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.StoreSource", - "extra", - ))?; + let extra = value + .extra + .ok_or(missing_field!(re_protos::log_msg::v0::StoreSource, "extra"))?; let python_version = re_protos::log_msg::v0::PythonVersion::decode(&mut &extra.payload[..])?; Ok(Self::PythonSdk(crate::PythonVersion::try_from( @@ -275,10 +275,9 @@ impl TryFrom for crate::StoreSource { )?)) } StoreSourceKind::RustSdk => { - let extra = value.extra.ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.StoreSource", - "extra", - ))?; + let extra = value + .extra + .ok_or(missing_field!(re_protos::log_msg::v0::StoreSource, "extra"))?; let crate_info = re_protos::log_msg::v0::CrateInfo::decode(&mut &extra.payload[..])?; Ok(Self::RustSdk { @@ -287,10 +286,9 @@ impl TryFrom for crate::StoreSource { }) } StoreSourceKind::File => { - let extra = value.extra.ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.StoreSource", - "extra", - ))?; + let extra = value + .extra + .ok_or(missing_field!(re_protos::log_msg::v0::StoreSource, "extra"))?; let file_source = re_protos::log_msg::v0::FileSource::decode(&mut &extra.payload[..])?; Ok(Self::File { @@ -299,16 +297,15 @@ impl TryFrom for crate::StoreSource { } StoreSourceKind::Viewer => Ok(Self::Viewer), StoreSourceKind::Other => { - let description = value.extra.ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.StoreSource", - "extra", - ))?; + let description = value + .extra + .ok_or(missing_field!(re_protos::log_msg::v0::StoreSource, "extra"))?; let description = String::from_utf8(description.payload).map_err(|err| { - TypeConversionError::InvalidField { - type_name: "rerun.log_msg.v0.StoreSource", - field_name: "extra", - reason: err.to_string(), - } + invalid_field!( + re_protos::log_msg::v0::StoreSource, + "extra", + err.to_string() + ) })?; Ok(Self::Other(description)) } @@ -416,30 +413,27 @@ impl TryFrom for crate::StoreInfo { fn try_from(value: re_protos::log_msg::v0::StoreInfo) -> Result { let application_id: crate::ApplicationId = value .application_id - .ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.StoreInfo", + .ok_or(missing_field!( + re_protos::log_msg::v0::StoreInfo, "application_id", ))? .into(); let store_id: crate::StoreId = value .store_id - .ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.StoreInfo", + .ok_or(missing_field!( + re_protos::log_msg::v0::StoreInfo, "store_id", ))? .into(); let is_official_example = value.is_official_example; let started: crate::Time = value .started - .ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.StoreInfo", - "started", - ))? + .ok_or(missing_field!(re_protos::log_msg::v0::StoreInfo, "started"))? .into(); let store_source: crate::StoreSource = value .store_source - .ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.StoreInfo", + .ok_or(missing_field!( + re_protos::log_msg::v0::StoreInfo, "store_source", ))? .try_into()?; @@ -477,17 +471,14 @@ impl TryFrom for crate::SetStoreInfo { Ok(Self { row_id: value .row_id - .ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.SetStoreInfo", + .ok_or(missing_field!( + re_protos::log_msg::v0::SetStoreInfo, "row_id", ))? .into(), info: value .info - .ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.SetStoreInfo", - "info", - ))? + .ok_or(missing_field!(re_protos::log_msg::v0::SetStoreInfo, "info"))? .try_into()?, }) } @@ -518,8 +509,8 @@ impl TryFrom Ok(Self { blueprint_id: value .blueprint_id - .ok_or(TypeConversionError::missing_field( - "rerun.log_msg.v0.BlueprintActivationCommand", + .ok_or(missing_field!( + re_protos::log_msg::v0::BlueprintActivationCommand, "blueprint_id", ))? .into(), @@ -542,7 +533,7 @@ mod tests { #[test] fn time_conversion() { let time = crate::Time::from_ns_since_epoch(123456789); - let proto_time: re_protos::common::v0::Time = time.clone().into(); + let proto_time: re_protos::common::v0::Time = time.into(); let time2: crate::Time = proto_time.into(); assert_eq!(time, time2); } @@ -550,7 +541,7 @@ mod tests { #[test] fn time_int_conversion() { let time_int = crate::TimeInt::new_temporal(123456789); - let proto_time_int: re_protos::common::v0::TimeInt = time_int.clone().into(); + let proto_time_int: re_protos::common::v0::TimeInt = time_int.into(); let time_int2: crate::TimeInt = proto_time_int.into(); assert_eq!(time_int, time_int2); } @@ -561,7 +552,7 @@ mod tests { crate::TimeInt::new_temporal(123456789), crate::TimeInt::new_temporal(987654321), ); - let proto_time_range: re_protos::common::v0::TimeRange = time_range.clone().into(); + let proto_time_range: re_protos::common::v0::TimeRange = time_range.into(); let time_range2: crate::ResolvedTimeRange = proto_time_range.into(); assert_eq!(time_range, time_range2); } @@ -572,7 +563,7 @@ mod tests { crate::TimeInt::new_temporal(123456789), crate::TimeInt::new_temporal(987654321), ); - let proto_index_range: re_protos::common::v0::IndexRange = time_range.clone().into(); + let proto_index_range: re_protos::common::v0::IndexRange = time_range.into(); let time_range2: crate::ResolvedTimeRange = proto_index_range.try_into().unwrap(); assert_eq!(time_range, time_range2); } @@ -600,7 +591,7 @@ mod tests { #[test] fn store_kind_conversion() { let store_kind = crate::StoreKind::Recording; - let proto_store_kind: re_protos::common::v0::StoreKind = store_kind.clone().into(); + let proto_store_kind: re_protos::common::v0::StoreKind = store_kind.into(); let store_kind2: crate::StoreKind = proto_store_kind.into(); assert_eq!(store_kind, store_kind2); } diff --git a/crates/store/re_protos/src/lib.rs b/crates/store/re_protos/src/lib.rs index 6a3f37430fb8..4e58c4f7de10 100644 --- a/crates/store/re_protos/src/lib.rs +++ b/crates/store/re_protos/src/lib.rs @@ -54,6 +54,7 @@ pub mod remote_store { pub enum TypeConversionError { #[error("missing required field: {type_name}.{field_name}")] MissingField { + package_name: &'static str, type_name: &'static str, field_name: &'static str, }, @@ -76,10 +77,36 @@ pub enum TypeConversionError { } impl TypeConversionError { - pub fn missing_field(type_name: &'static str, field_name: &'static str) -> Self { + #[inline] + pub fn missing_field(field_name: &'static str) -> Self { Self::MissingField { - type_name, + package_name: T::PACKAGE, + type_name: T::NAME, field_name, } } + + #[allow(clippy::needless_pass_by_value)] // false-positive + #[inline] + pub fn invalid_field(field_name: &'static str, reason: &impl ToString) -> Self { + Self::InvalidField { + type_name: T::NAME, + field_name, + reason: reason.to_string(), + } + } +} + +#[macro_export] +macro_rules! missing_field { + ($type:ty, $field:expr $(,)?) => { + $crate::TypeConversionError::missing_field::<$type>($field) + }; +} + +#[macro_export] +macro_rules! invalid_field { + ($type:ty, $field:expr, $reason:expr $(,)?) => { + $crate::TypeConversionError::invalid_field::<$type>($field, &$reason) + }; } diff --git a/crates/store/re_protos/src/v0/rerun.common.v0.rs b/crates/store/re_protos/src/v0/rerun.common.v0.rs index d06d839e0c65..99bcbbd61774 100644 --- a/crates/store/re_protos/src/v0/rerun.common.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.common.v0.rs @@ -11,18 +11,48 @@ pub struct RerunChunk { #[prost(bytes = "vec", tag = "1000")] pub payload: ::prost::alloc::vec::Vec, } +impl ::prost::Name for RerunChunk { + const NAME: &'static str = "RerunChunk"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.RerunChunk".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.RerunChunk".into() + } +} /// unique recording identifier. At this point in time it is the same id as the ChunkStore's StoreId #[derive(Clone, PartialEq, ::prost::Message)] pub struct RecordingId { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, } +impl ::prost::Name for RecordingId { + const NAME: &'static str = "RecordingId"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.RecordingId".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.RecordingId".into() + } +} /// A recording can have multiple timelines, each is identified by a name, for example `log_tick`, `log_time`, etc. #[derive(Clone, PartialEq, ::prost::Message)] pub struct Timeline { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, } +impl ::prost::Name for Timeline { + const NAME: &'static str = "Timeline"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Timeline".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Timeline".into() + } +} /// A time range between start and end time points. Each 64 bit number can represent different time point data /// depending on the timeline it is associated with. Time range is inclusive for both start and end time points. #[derive(Clone, Copy, PartialEq, ::prost::Message)] @@ -32,12 +62,32 @@ pub struct TimeRange { #[prost(int64, tag = "2")] pub end: i64, } +impl ::prost::Name for TimeRange { + const NAME: &'static str = "TimeRange"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.TimeRange".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.TimeRange".into() + } +} /// arrow IPC serialized schema #[derive(Clone, PartialEq, ::prost::Message)] pub struct Schema { #[prost(bytes = "vec", tag = "1")] pub arrow_schema: ::prost::alloc::vec::Vec, } +impl ::prost::Name for Schema { + const NAME: &'static str = "Schema"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Schema".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Schema".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct Query { /// The subset of the database that the query will run on: a set of EntityPath(s) and their @@ -100,11 +150,31 @@ pub struct Query { #[prost(enumeration = "SparseFillStrategy", tag = "11")] pub sparse_fill_strategy: i32, } +impl ::prost::Name for Query { + const NAME: &'static str = "Query"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Query".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Query".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ColumnSelection { #[prost(message, repeated, tag = "1")] pub columns: ::prost::alloc::vec::Vec, } +impl ::prost::Name for ColumnSelection { + const NAME: &'static str = "ColumnSelection"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ColumnSelection".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ColumnSelection".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ColumnSelector { #[prost(oneof = "column_selector::SelectorType", tags = "2, 3")] @@ -120,40 +190,110 @@ pub mod column_selector { TimeColumn(super::TimeColumnSelector), } } +impl ::prost::Name for ColumnSelector { + const NAME: &'static str = "ColumnSelector"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ColumnSelector".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ColumnSelector".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct IndexColumnSelector { /// TODO(zehiko) we need to add support for other types of index selectors #[prost(message, optional, tag = "1")] pub timeline: ::core::option::Option, } +impl ::prost::Name for IndexColumnSelector { + const NAME: &'static str = "IndexColumnSelector"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.IndexColumnSelector".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.IndexColumnSelector".into() + } +} #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct IndexRange { /// TODO(zehiko) support for other ranges for other index selectors #[prost(message, optional, tag = "1")] pub time_range: ::core::option::Option, } +impl ::prost::Name for IndexRange { + const NAME: &'static str = "IndexRange"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.IndexRange".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.IndexRange".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct IndexValues { /// TODO(zehiko) we need to add support for other types of index selectors #[prost(message, repeated, tag = "1")] pub time_points: ::prost::alloc::vec::Vec, } +impl ::prost::Name for IndexValues { + const NAME: &'static str = "IndexValues"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.IndexValues".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.IndexValues".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct SampledIndexValues { #[prost(message, repeated, tag = "1")] pub sample_points: ::prost::alloc::vec::Vec, } +impl ::prost::Name for SampledIndexValues { + const NAME: &'static str = "SampledIndexValues"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.SampledIndexValues".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.SampledIndexValues".into() + } +} /// A 64-bit number describing either nanoseconds, sequence numbers or fully static data. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct TimeInt { #[prost(int64, tag = "1")] pub time: i64, } +impl ::prost::Name for TimeInt { + const NAME: &'static str = "TimeInt"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.TimeInt".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.TimeInt".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ViewContents { #[prost(message, repeated, tag = "1")] pub contents: ::prost::alloc::vec::Vec, } +impl ::prost::Name for ViewContents { + const NAME: &'static str = "ViewContents"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ViewContents".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ViewContents".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ViewContentsPart { #[prost(message, optional, tag = "1")] @@ -161,11 +301,31 @@ pub struct ViewContentsPart { #[prost(message, optional, tag = "2")] pub components: ::core::option::Option, } +impl ::prost::Name for ViewContentsPart { + const NAME: &'static str = "ViewContentsPart"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ViewContentsPart".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ViewContentsPart".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ComponentsSet { #[prost(message, repeated, tag = "1")] pub components: ::prost::alloc::vec::Vec, } +impl ::prost::Name for ComponentsSet { + const NAME: &'static str = "ComponentsSet"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ComponentsSet".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ComponentsSet".into() + } +} /// The unique identifier of an entity, e.g. `camera/3/points` /// See <> for more on entity paths. #[derive(Clone, PartialEq, ::prost::Message)] @@ -173,6 +333,16 @@ pub struct EntityPath { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, } +impl ::prost::Name for EntityPath { + const NAME: &'static str = "EntityPath"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.EntityPath".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.EntityPath".into() + } +} /// Component describes semantic data that can be used by any number of rerun's archetypes. #[derive(Clone, PartialEq, ::prost::Message)] pub struct Component { @@ -180,12 +350,32 @@ pub struct Component { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, } +impl ::prost::Name for Component { + const NAME: &'static str = "Component"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Component".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Component".into() + } +} /// Used to telect a time column. #[derive(Clone, PartialEq, ::prost::Message)] pub struct TimeColumnSelector { #[prost(message, optional, tag = "1")] pub timeline: ::core::option::Option, } +impl ::prost::Name for TimeColumnSelector { + const NAME: &'static str = "TimeColumnSelector"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.TimeColumnSelector".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.TimeColumnSelector".into() + } +} /// Used to select a component based on its EntityPath and Component name. #[derive(Clone, PartialEq, ::prost::Message)] pub struct ComponentColumnSelector { @@ -194,11 +384,31 @@ pub struct ComponentColumnSelector { #[prost(message, optional, tag = "2")] pub component: ::core::option::Option, } +impl ::prost::Name for ComponentColumnSelector { + const NAME: &'static str = "ComponentColumnSelector"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ComponentColumnSelector".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ComponentColumnSelector".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ApplicationId { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, } +impl ::prost::Name for ApplicationId { + const NAME: &'static str = "ApplicationId"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ApplicationId".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ApplicationId".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreId { #[prost(enumeration = "StoreKind", tag = "1")] @@ -206,12 +416,32 @@ pub struct StoreId { #[prost(string, tag = "2")] pub id: ::prost::alloc::string::String, } +impl ::prost::Name for StoreId { + const NAME: &'static str = "StoreId"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.StoreId".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.StoreId".into() + } +} /// A date-time represented as nanoseconds since unix epoch #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct Time { #[prost(int64, tag = "1")] pub nanos_since_epoch: i64, } +impl ::prost::Name for Time { + const NAME: &'static str = "Time"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Time".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Time".into() + } +} #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct Tuid { /// Approximate nanoseconds since epoch. @@ -222,6 +452,16 @@ pub struct Tuid { #[prost(fixed64, tag = "2")] pub inc: u64, } +impl ::prost::Name for Tuid { + const NAME: &'static str = "Tuid"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Tuid".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Tuid".into() + } +} /// supported encoder versions for encoding data /// See `RerunData` and `RerunChunkData` for its usage #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] diff --git a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs index 3a0044e099b4..3b415876fbc0 100644 --- a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs @@ -9,6 +9,16 @@ pub struct SetStoreInfo { #[prost(message, optional, tag = "2")] pub info: ::core::option::Option, } +impl ::prost::Name for SetStoreInfo { + const NAME: &'static str = "SetStoreInfo"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.SetStoreInfo".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.SetStoreInfo".into() + } +} /// Corresponds to `LogMsg::ArrowMsg`. Used to transmit actual data. #[derive(Clone, PartialEq, ::prost::Message)] pub struct ArrowMsg { @@ -27,6 +37,16 @@ pub struct ArrowMsg { #[prost(bytes = "vec", tag = "1000")] pub payload: ::prost::alloc::vec::Vec, } +impl ::prost::Name for ArrowMsg { + const NAME: &'static str = "ArrowMsg"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.ArrowMsg".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.ArrowMsg".into() + } +} /// Corresponds to `LogMsg::BlueprintActivationCommand`. /// /// Used for activating a blueprint once it has been fully transmitted, @@ -44,6 +64,16 @@ pub struct BlueprintActivationCommand { #[prost(bool, tag = "3")] pub make_default: bool, } +impl ::prost::Name for BlueprintActivationCommand { + const NAME: &'static str = "BlueprintActivationCommand"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.BlueprintActivationCommand".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.BlueprintActivationCommand".into() + } +} /// Information about a recording or blueprint. #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreInfo { @@ -66,6 +96,16 @@ pub struct StoreInfo { #[prost(message, optional, tag = "6")] pub store_version: ::core::option::Option, } +impl ::prost::Name for StoreInfo { + const NAME: &'static str = "StoreInfo"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.StoreInfo".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.StoreInfo".into() + } +} /// The source of a recording or blueprint. #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreSource { @@ -76,6 +116,16 @@ pub struct StoreSource { #[prost(message, optional, tag = "2")] pub extra: ::core::option::Option, } +impl ::prost::Name for StoreSource { + const NAME: &'static str = "StoreSource"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.StoreSource".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.StoreSource".into() + } +} /// A newtype for `StoreSource` payload. /// /// This exists to that we can implement conversions on the newtype for convenience. @@ -84,6 +134,16 @@ pub struct StoreSourceExtra { #[prost(bytes = "vec", tag = "1")] pub payload: ::prost::alloc::vec::Vec, } +impl ::prost::Name for StoreSourceExtra { + const NAME: &'static str = "StoreSourceExtra"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.StoreSourceExtra".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.StoreSourceExtra".into() + } +} /// Version of the Python SDK that created the recording. #[derive(Clone, PartialEq, ::prost::Message)] pub struct PythonVersion { @@ -96,6 +156,16 @@ pub struct PythonVersion { #[prost(string, tag = "4")] pub suffix: ::prost::alloc::string::String, } +impl ::prost::Name for PythonVersion { + const NAME: &'static str = "PythonVersion"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.PythonVersion".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.PythonVersion".into() + } +} /// Information about the Rust SDK that created the recording. #[derive(Clone, PartialEq, ::prost::Message)] pub struct CrateInfo { @@ -106,12 +176,32 @@ pub struct CrateInfo { #[prost(string, tag = "2")] pub llvm_version: ::prost::alloc::string::String, } +impl ::prost::Name for CrateInfo { + const NAME: &'static str = "CrateInfo"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.CrateInfo".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.CrateInfo".into() + } +} /// A recording which came from a file. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct FileSource { #[prost(enumeration = "FileSourceKind", tag = "1")] pub kind: i32, } +impl ::prost::Name for FileSource { + const NAME: &'static str = "FileSource"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.FileSource".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.FileSource".into() + } +} #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct StoreVersion { /// Crate version encoded using our custom scheme. @@ -120,6 +210,16 @@ pub struct StoreVersion { #[prost(int32, tag = "1")] pub crate_version_bits: i32, } +impl ::prost::Name for StoreVersion { + const NAME: &'static str = "StoreVersion"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.StoreVersion".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.StoreVersion".into() + } +} /// The type of compression used on the payload. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs index 144ddc584ef7..1429146e776d 100644 --- a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs @@ -9,6 +9,16 @@ pub struct DataframePart { #[prost(bytes = "vec", tag = "1000")] pub payload: ::prost::alloc::vec::Vec, } +impl ::prost::Name for DataframePart { + const NAME: &'static str = "DataframePart"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.DataframePart".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.DataframePart".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct RegisterRecordingRequest { /// human readable description of the recording @@ -25,6 +35,16 @@ pub struct RegisterRecordingRequest { #[prost(message, optional, tag = "4")] pub metadata: ::core::option::Option, } +impl ::prost::Name for RegisterRecordingRequest { + const NAME: &'static str = "RegisterRecordingRequest"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.RegisterRecordingRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.RegisterRecordingRequest".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct UpdateCatalogRequest { #[prost(message, optional, tag = "1")] @@ -32,8 +52,28 @@ pub struct UpdateCatalogRequest { #[prost(message, optional, tag = "2")] pub metadata: ::core::option::Option, } +impl ::prost::Name for UpdateCatalogRequest { + const NAME: &'static str = "UpdateCatalogRequest"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.UpdateCatalogRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.UpdateCatalogRequest".into() + } +} #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct UpdateCatalogResponse {} +impl ::prost::Name for UpdateCatalogResponse { + const NAME: &'static str = "UpdateCatalogResponse"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.UpdateCatalogResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.UpdateCatalogResponse".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct QueryRequest { /// unique identifier of the recording @@ -43,6 +83,16 @@ pub struct QueryRequest { #[prost(message, optional, tag = "3")] pub query: ::core::option::Option, } +impl ::prost::Name for QueryRequest { + const NAME: &'static str = "QueryRequest"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.QueryRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.QueryRequest".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct QueryCatalogRequest { /// Column projection - define which columns should be returned. @@ -53,11 +103,31 @@ pub struct QueryCatalogRequest { #[prost(message, optional, tag = "2")] pub filter: ::core::option::Option, } +impl ::prost::Name for QueryCatalogRequest { + const NAME: &'static str = "QueryCatalogRequest"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.QueryCatalogRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.QueryCatalogRequest".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ColumnProjection { #[prost(string, repeated, tag = "1")] pub columns: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } +impl ::prost::Name for ColumnProjection { + const NAME: &'static str = "ColumnProjection"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.ColumnProjection".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.ColumnProjection".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct CatalogFilter { /// Filtering is very simple right now, we can only select @@ -65,6 +135,16 @@ pub struct CatalogFilter { #[prost(message, repeated, tag = "1")] pub recording_ids: ::prost::alloc::vec::Vec, } +impl ::prost::Name for CatalogFilter { + const NAME: &'static str = "CatalogFilter"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.CatalogFilter".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.CatalogFilter".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct QueryCatalogResponse { #[prost(enumeration = "super::super::common::v0::EncoderVersion", tag = "1")] @@ -73,11 +153,31 @@ pub struct QueryCatalogResponse { #[prost(bytes = "vec", tag = "2")] pub payload: ::prost::alloc::vec::Vec, } +impl ::prost::Name for QueryCatalogResponse { + const NAME: &'static str = "QueryCatalogResponse"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.QueryCatalogResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.QueryCatalogResponse".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct FetchRecordingRequest { #[prost(message, optional, tag = "1")] pub recording_id: ::core::option::Option, } +impl ::prost::Name for FetchRecordingRequest { + const NAME: &'static str = "FetchRecordingRequest"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.FetchRecordingRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.FetchRecordingRequest".into() + } +} /// TODO(jleibs): Eventually this becomes either query-mediated in some way, but for now /// it's useful to be able to just get back the whole RRD somehow. #[derive(Clone, PartialEq, ::prost::Message)] @@ -91,6 +191,16 @@ pub struct FetchRecordingResponse { #[prost(bytes = "vec", tag = "2")] pub payload: ::prost::alloc::vec::Vec, } +impl ::prost::Name for FetchRecordingResponse { + const NAME: &'static str = "FetchRecordingResponse"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.FetchRecordingResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.FetchRecordingResponse".into() + } +} /// Application level error - used as `details` in the `google.rpc.Status` message #[derive(Clone, PartialEq, ::prost::Message)] pub struct RemoteStoreError { @@ -104,6 +214,16 @@ pub struct RemoteStoreError { #[prost(string, tag = "3")] pub message: ::prost::alloc::string::String, } +impl ::prost::Name for RemoteStoreError { + const NAME: &'static str = "RemoteStoreError"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.RemoteStoreError".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.RemoteStoreError".into() + } +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum RecordingType { From 90360cfe1515f69b98bdc235fba723d19c834b46 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 20:47:46 +0100 Subject: [PATCH 33/47] move `MessageKind`/`MessageHeader` impls back into declaration site --- .../re_log_encoding/src/codec/file/decoder.rs | 26 --------- .../re_log_encoding/src/codec/file/encoder.rs | 16 ------ .../re_log_encoding/src/codec/file/mod.rs | 54 +++++++++++++++++++ 3 files changed, 54 insertions(+), 42 deletions(-) diff --git a/crates/store/re_log_encoding/src/codec/file/decoder.rs b/crates/store/re_log_encoding/src/codec/file/decoder.rs index 3ef7112b808c..0eecdb59a52b 100644 --- a/crates/store/re_log_encoding/src/codec/file/decoder.rs +++ b/crates/store/re_log_encoding/src/codec/file/decoder.rs @@ -5,32 +5,6 @@ use crate::decoder::DecodeError; use re_log_types::LogMsg; use re_protos::missing_field; -impl MessageKind { - pub(crate) fn decode(data: &mut impl std::io::Read) -> Result { - let mut buf = [0; 4]; - data.read_exact(&mut buf)?; - - match u32::from_le_bytes(buf) { - 1 => Ok(Self::SetStoreInfo), - 2 => Ok(Self::ArrowMsg), - 3 => Ok(Self::BlueprintActivationCommand), - 255 => Ok(Self::End), - _ => Err(DecodeError::Codec(CodecError::UnknownMessageHeader)), - } - } -} - -impl MessageHeader { - pub(crate) fn decode(data: &mut impl std::io::Read) -> Result { - let kind = MessageKind::decode(data)?; - let mut buf = [0; 4]; - data.read_exact(&mut buf)?; - let len = u32::from_le_bytes(buf); - - Ok(Self { kind, len }) - } -} - pub(crate) fn decode(data: &mut impl std::io::Read) -> Result<(u64, Option), DecodeError> { use re_protos::external::prost::Message; use re_protos::log_msg::v0::{ArrowMsg, BlueprintActivationCommand, Encoding, SetStoreInfo}; diff --git a/crates/store/re_log_encoding/src/codec/file/encoder.rs b/crates/store/re_log_encoding/src/codec/file/encoder.rs index dce303b1caa9..9b2783d06d3f 100644 --- a/crates/store/re_log_encoding/src/codec/file/encoder.rs +++ b/crates/store/re_log_encoding/src/codec/file/encoder.rs @@ -4,22 +4,6 @@ use crate::encoder::EncodeError; use crate::Compression; use re_log_types::LogMsg; -impl MessageKind { - pub(crate) fn encode(&self, buf: &mut impl std::io::Write) -> Result<(), EncodeError> { - let kind: u32 = *self as u32; - buf.write_all(&kind.to_le_bytes())?; - Ok(()) - } -} - -impl MessageHeader { - pub(crate) fn encode(&self, buf: &mut impl std::io::Write) -> Result<(), EncodeError> { - self.kind.encode(buf)?; - buf.write_all(&self.len.to_le_bytes())?; - Ok(()) - } -} - pub(crate) fn encode( buf: &mut Vec, message: &LogMsg, diff --git a/crates/store/re_log_encoding/src/codec/file/mod.rs b/crates/store/re_log_encoding/src/codec/file/mod.rs index ef058b013e85..925bdc70da3b 100644 --- a/crates/store/re_log_encoding/src/codec/file/mod.rs +++ b/crates/store/re_log_encoding/src/codec/file/mod.rs @@ -19,3 +19,57 @@ pub(crate) struct MessageHeader { pub(crate) kind: MessageKind, pub(crate) len: u32, } + +impl MessageKind { + #[cfg(feature = "encoder")] + pub(crate) fn encode( + &self, + buf: &mut impl std::io::Write, + ) -> Result<(), crate::encoder::EncodeError> { + let kind: u32 = *self as u32; + buf.write_all(&kind.to_le_bytes())?; + Ok(()) + } + + #[cfg(feature = "decoder")] + pub(crate) fn decode( + data: &mut impl std::io::Read, + ) -> Result { + let mut buf = [0; 4]; + data.read_exact(&mut buf)?; + + match u32::from_le_bytes(buf) { + 1 => Ok(Self::SetStoreInfo), + 2 => Ok(Self::ArrowMsg), + 3 => Ok(Self::BlueprintActivationCommand), + 255 => Ok(Self::End), + _ => Err(crate::decoder::DecodeError::Codec( + crate::codec::CodecError::UnknownMessageHeader, + )), + } + } +} + +impl MessageHeader { + #[cfg(feature = "encoder")] + pub(crate) fn encode( + &self, + buf: &mut impl std::io::Write, + ) -> Result<(), crate::encoder::EncodeError> { + self.kind.encode(buf)?; + buf.write_all(&self.len.to_le_bytes())?; + Ok(()) + } + + #[cfg(feature = "decoder")] + pub(crate) fn decode( + data: &mut impl std::io::Read, + ) -> Result { + let kind = MessageKind::decode(data)?; + let mut buf = [0; 4]; + data.read_exact(&mut buf)?; + let len = u32::from_le_bytes(buf); + + Ok(Self { kind, len }) + } +} From c880b6a0e79481b4c99486d687ad458432f22f6d Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 20:57:38 +0100 Subject: [PATCH 34/47] 64-bit length --- crates/store/re_log_encoding/src/codec/file/decoder.rs | 2 +- crates/store/re_log_encoding/src/codec/file/encoder.rs | 6 +++--- crates/store/re_log_encoding/src/codec/file/mod.rs | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/store/re_log_encoding/src/codec/file/decoder.rs b/crates/store/re_log_encoding/src/codec/file/decoder.rs index 0eecdb59a52b..16b6f0ba4728 100644 --- a/crates/store/re_log_encoding/src/codec/file/decoder.rs +++ b/crates/store/re_log_encoding/src/codec/file/decoder.rs @@ -11,7 +11,7 @@ pub(crate) fn decode(data: &mut impl std::io::Read) -> Result<(u64, Option() as u64 + header.len as u64; + read_bytes += std::mem::size_of::() as u64 + header.len; let mut buf = vec![0; header.len as usize]; data.read_exact(&mut buf[..])?; diff --git a/crates/store/re_log_encoding/src/codec/file/encoder.rs b/crates/store/re_log_encoding/src/codec/file/encoder.rs index 9b2783d06d3f..3456df76b37b 100644 --- a/crates/store/re_log_encoding/src/codec/file/encoder.rs +++ b/crates/store/re_log_encoding/src/codec/file/encoder.rs @@ -19,7 +19,7 @@ pub(crate) fn encode( let set_store_info: SetStoreInfo = set_store_info.clone().into(); let header = MessageHeader { kind: MessageKind::SetStoreInfo, - len: set_store_info.encoded_len() as u32, + len: set_store_info.encoded_len() as u64, }; header.encode(buf)?; set_store_info.encode(buf)?; @@ -38,7 +38,7 @@ pub(crate) fn encode( }; let header = MessageHeader { kind: MessageKind::ArrowMsg, - len: arrow_msg.encoded_len() as u32, + len: arrow_msg.encoded_len() as u64, }; header.encode(buf)?; arrow_msg.encode(buf)?; @@ -48,7 +48,7 @@ pub(crate) fn encode( blueprint_activation_command.clone().into(); let header = MessageHeader { kind: MessageKind::BlueprintActivationCommand, - len: blueprint_activation_command.encoded_len() as u32, + len: blueprint_activation_command.encoded_len() as u64, }; header.encode(buf)?; blueprint_activation_command.encode(buf)?; diff --git a/crates/store/re_log_encoding/src/codec/file/mod.rs b/crates/store/re_log_encoding/src/codec/file/mod.rs index 925bdc70da3b..2575f39b3e32 100644 --- a/crates/store/re_log_encoding/src/codec/file/mod.rs +++ b/crates/store/re_log_encoding/src/codec/file/mod.rs @@ -17,7 +17,7 @@ pub(crate) enum MessageKind { #[derive(Debug, Clone, Copy)] pub(crate) struct MessageHeader { pub(crate) kind: MessageKind, - pub(crate) len: u32, + pub(crate) len: u64, } impl MessageKind { @@ -66,9 +66,9 @@ impl MessageHeader { data: &mut impl std::io::Read, ) -> Result { let kind = MessageKind::decode(data)?; - let mut buf = [0; 4]; + let mut buf = [0; 8]; data.read_exact(&mut buf)?; - let len = u32::from_le_bytes(buf); + let len = u64::from_le_bytes(buf); Ok(Self { kind, len }) } From 5ce153110a28d3db4ebed3b2f5196b55a333e80f Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 21:00:42 +0100 Subject: [PATCH 35/47] fix --- .../re_log_types/src/protobuf_conversions.rs | 26 +++++++------------ crates/store/re_protos/src/lib.rs | 6 +++-- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/crates/store/re_log_types/src/protobuf_conversions.rs b/crates/store/re_log_types/src/protobuf_conversions.rs index e6dcc56d3bb0..aab629292512 100644 --- a/crates/store/re_log_types/src/protobuf_conversions.rs +++ b/crates/store/re_log_types/src/protobuf_conversions.rs @@ -1,6 +1,5 @@ -use re_protos::invalid_field; -use re_protos::missing_field; use re_protos::TypeConversionError; +use re_protos::{invalid_field, missing_field}; use std::sync::Arc; impl From for re_protos::common::v0::EntityPath { @@ -15,11 +14,8 @@ impl TryFrom for crate::EntityPath { type Error = TypeConversionError; fn try_from(value: re_protos::common::v0::EntityPath) -> Result { - Self::parse_strict(&value.path).map_err(|err| TypeConversionError::InvalidField { - type_name: "rerun.common.v0.EntityPath", - field_name: "path", - reason: err.to_string(), - }) + Self::parse_strict(&value.path) + .map_err(|err| invalid_field!(re_protos::common::v0::EntityPath, "path", err)) } } @@ -301,11 +297,7 @@ impl TryFrom for crate::StoreSource { .extra .ok_or(missing_field!(re_protos::log_msg::v0::StoreSource, "extra"))?; let description = String::from_utf8(description.payload).map_err(|err| { - invalid_field!( - re_protos::log_msg::v0::StoreSource, - "extra", - err.to_string() - ) + invalid_field!(re_protos::log_msg::v0::StoreSource, "extra", err) })?; Ok(Self::Other(description)) } @@ -379,11 +371,11 @@ impl TryFrom for crate::FileSource { force_store_info: false, }), FileSourceKind::Sdk => Ok(Self::Sdk), - FileSourceKind::UnknownSource => Err(TypeConversionError::InvalidField { - type_name: "rerun.log_msg.v0.FileSource", - field_name: "kind", - reason: "unknown kind".to_owned(), - }), + FileSourceKind::UnknownSource => Err(invalid_field!( + re_protos::log_msg::v0::FileSource, + "kind", + "unknown kind", + )), } } } diff --git a/crates/store/re_protos/src/lib.rs b/crates/store/re_protos/src/lib.rs index 4e58c4f7de10..545198db999b 100644 --- a/crates/store/re_protos/src/lib.rs +++ b/crates/store/re_protos/src/lib.rs @@ -52,15 +52,16 @@ pub mod remote_store { #[derive(Debug, thiserror::Error)] pub enum TypeConversionError { - #[error("missing required field: {type_name}.{field_name}")] + #[error("missing required field: {package_name}.{type_name}.{field_name}")] MissingField { package_name: &'static str, type_name: &'static str, field_name: &'static str, }, - #[error("invalid value for field {type_name}.{field_name}: {reason}")] + #[error("invalid value for field {package_name}.{type_name}.{field_name}: {reason}")] InvalidField { + package_name: &'static str, type_name: &'static str, field_name: &'static str, reason: String, @@ -90,6 +91,7 @@ impl TypeConversionError { #[inline] pub fn invalid_field(field_name: &'static str, reason: &impl ToString) -> Self { Self::InvalidField { + package_name: T::PACKAGE, type_name: T::NAME, field_name, reason: reason.to_string(), From abbd4c9533ddb61640586c90edcaea612632f1a2 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 21:38:19 +0100 Subject: [PATCH 36/47] less hardcoding, more type-safety --- .../re_log_encoding/src/codec/file/mod.rs | 79 +++++++++---------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/crates/store/re_log_encoding/src/codec/file/mod.rs b/crates/store/re_log_encoding/src/codec/file/mod.rs index 2575f39b3e32..e491777409ca 100644 --- a/crates/store/re_log_encoding/src/codec/file/mod.rs +++ b/crates/store/re_log_encoding/src/codec/file/mod.rs @@ -3,31 +3,41 @@ pub(crate) mod decoder; #[cfg(feature = "encoder")] pub(crate) mod encoder; -#[allow(dead_code)] // used in encoder/decoder behind feature flag -#[derive(Debug, Clone, Copy)] -#[repr(u32)] +#[allow(dead_code)] // used behind feature flag +#[derive(Default, Debug, Clone, Copy)] +#[repr(u64)] pub(crate) enum MessageKind { - SetStoreInfo = 1, - ArrowMsg = 2, - BlueprintActivationCommand = 3, - End = 255, + #[default] + End = Self::END, + SetStoreInfo = Self::SET_STORE_INFO, + ArrowMsg = Self::ARROW_MSG, + BlueprintActivationCommand = Self::BLUEPRINT_ACTIVATION_COMMAND, +} + +#[allow(dead_code)] // used behind feature flag +impl MessageKind { + const END: u64 = 0; + const SET_STORE_INFO: u64 = 1; + const ARROW_MSG: u64 = 2; + const BLUEPRINT_ACTIVATION_COMMAND: u64 = 3; } -#[allow(dead_code)] // used in encoder/decoder behind feature flag +#[allow(dead_code)] // used behind feature flag #[derive(Debug, Clone, Copy)] pub(crate) struct MessageHeader { pub(crate) kind: MessageKind, pub(crate) len: u64, } -impl MessageKind { +impl MessageHeader { #[cfg(feature = "encoder")] pub(crate) fn encode( &self, buf: &mut impl std::io::Write, ) -> Result<(), crate::encoder::EncodeError> { - let kind: u32 = *self as u32; - buf.write_all(&kind.to_le_bytes())?; + let Self { kind, len } = *self; + buf.write_all(&(kind as u64).to_le_bytes())?; + buf.write_all(&len.to_le_bytes())?; Ok(()) } @@ -35,40 +45,25 @@ impl MessageKind { pub(crate) fn decode( data: &mut impl std::io::Read, ) -> Result { - let mut buf = [0; 4]; + let mut buf = [0; std::mem::size_of::()]; data.read_exact(&mut buf)?; - match u32::from_le_bytes(buf) { - 1 => Ok(Self::SetStoreInfo), - 2 => Ok(Self::ArrowMsg), - 3 => Ok(Self::BlueprintActivationCommand), - 255 => Ok(Self::End), - _ => Err(crate::decoder::DecodeError::Codec( - crate::codec::CodecError::UnknownMessageHeader, - )), - } - } -} - -impl MessageHeader { - #[cfg(feature = "encoder")] - pub(crate) fn encode( - &self, - buf: &mut impl std::io::Write, - ) -> Result<(), crate::encoder::EncodeError> { - self.kind.encode(buf)?; - buf.write_all(&self.len.to_le_bytes())?; - Ok(()) - } + #[allow(clippy::unwrap_used)] // cannot fail + let kind = u64::from_le_bytes(buf[0..8].try_into().unwrap()); + let kind = match kind { + MessageKind::END => MessageKind::End, + MessageKind::SET_STORE_INFO => MessageKind::SetStoreInfo, + MessageKind::ARROW_MSG => MessageKind::ArrowMsg, + MessageKind::BLUEPRINT_ACTIVATION_COMMAND => MessageKind::BlueprintActivationCommand, + _ => { + return Err(crate::decoder::DecodeError::Codec( + crate::codec::CodecError::UnknownMessageHeader, + )) + } + }; - #[cfg(feature = "decoder")] - pub(crate) fn decode( - data: &mut impl std::io::Read, - ) -> Result { - let kind = MessageKind::decode(data)?; - let mut buf = [0; 8]; - data.read_exact(&mut buf)?; - let len = u64::from_le_bytes(buf); + #[allow(clippy::unwrap_used)] // cannot fail + let len = u64::from_le_bytes(buf[8..16].try_into().unwrap()); Ok(Self { kind, len }) } From 764d3d639ef2d6fc20dea44fdd3a40ce30a88f82 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 21:42:11 +0100 Subject: [PATCH 37/47] type aliases --- .../store/re_log_encoding/src/codec/arrow.rs | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/crates/store/re_log_encoding/src/codec/arrow.rs b/crates/store/re_log_encoding/src/codec/arrow.rs index 6c2c69c75b57..d745719ed517 100644 --- a/crates/store/re_log_encoding/src/codec/arrow.rs +++ b/crates/store/re_log_encoding/src/codec/arrow.rs @@ -3,17 +3,21 @@ use crate::encoder::EncodeError; use super::CodecError; +use arrow2::array::Array as Arrow2Array; +use arrow2::datatypes::Schema as Arrow2Schema; +use arrow2::io::ipc; + +type Arrow2Chunk = arrow2::chunk::Chunk>; + // TODO(#8412): try using arrow ipc `compression` option instead of doing our own compression /// Helper function that serializes given arrow schema and record batch into bytes /// using Arrow IPC format. pub(crate) fn write_arrow_to_bytes( writer: &mut W, - schema: &arrow2::datatypes::Schema, - data: &arrow2::chunk::Chunk>, + schema: &Arrow2Schema, + data: &Arrow2Chunk, ) -> Result<(), CodecError> { - use arrow2::io::ipc; - let options = ipc::write::WriteOptions { compression: None }; let mut sw = ipc::write::StreamWriter::new(writer, options); @@ -30,13 +34,7 @@ pub(crate) fn write_arrow_to_bytes( /// using Arrow IPC format. pub(crate) fn read_arrow_from_bytes( reader: &mut R, -) -> Result< - ( - arrow2::datatypes::Schema, - arrow2::chunk::Chunk>, - ), - CodecError, -> { +) -> Result<(Arrow2Schema, Arrow2Chunk), CodecError> { use arrow2::io::ipc; let metadata = @@ -62,8 +60,8 @@ pub(crate) struct Payload { } pub(crate) fn encode_arrow( - schema: &arrow2::datatypes::Schema, - chunk: &arrow2::chunk::Chunk>, + schema: &Arrow2Schema, + chunk: &Arrow2Chunk, compression: crate::Compression, ) -> Result { let mut uncompressed = Vec::new(); @@ -85,13 +83,7 @@ pub(crate) fn decode_arrow( data: &[u8], uncompressed_size: usize, compression: crate::Compression, -) -> Result< - ( - arrow2::datatypes::Schema, - arrow2::chunk::Chunk>, - ), - DecodeError, -> { +) -> Result<(Arrow2Schema, Arrow2Chunk), DecodeError> { let mut uncompressed = Vec::new(); let data = match compression { crate::Compression::Off => data, From 922ef0e52d8167cb30fb760e8778e81b3c16f210 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 21:42:19 +0100 Subject: [PATCH 38/47] remove dead comment --- crates/store/re_log_encoding/src/codec/arrow.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/store/re_log_encoding/src/codec/arrow.rs b/crates/store/re_log_encoding/src/codec/arrow.rs index d745719ed517..e2762c737cb3 100644 --- a/crates/store/re_log_encoding/src/codec/arrow.rs +++ b/crates/store/re_log_encoding/src/codec/arrow.rs @@ -9,8 +9,6 @@ use arrow2::io::ipc; type Arrow2Chunk = arrow2::chunk::Chunk>; -// TODO(#8412): try using arrow ipc `compression` option instead of doing our own compression - /// Helper function that serializes given arrow schema and record batch into bytes /// using Arrow IPC format. pub(crate) fn write_arrow_to_bytes( From a51ffbda2080642bd54c55ce3619b190b52da139 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 22:11:50 +0100 Subject: [PATCH 39/47] remove spaces --- .../re_protos_builder/src/bin/build_re_remote_store_types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs b/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs index d1bb11fc2a98..9ed06af18e62 100644 --- a/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs +++ b/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs @@ -1,6 +1,6 @@ //! This binary runs the remote store gRPC service codegen manually. //! -//! It is easiest to call this using `pixi run codegen-protos `, +//! It is easiest to call this using `pixi run codegen-protos`, //! which will set up the necessary tools. #![allow(clippy::unwrap_used)] From d3183f0fabada197f0993688ba19e83a6e136ec8 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 22:20:40 +0100 Subject: [PATCH 40/47] add full bench --- .../benches/msg_encode_benchmark.rs | 217 ++++++++++++++---- 1 file changed, 167 insertions(+), 50 deletions(-) diff --git a/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs b/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs index 36f966fea758..40c72e5a906b 100644 --- a/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs +++ b/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs @@ -14,6 +14,10 @@ use re_log_types::{ LogMsg, StoreId, StoreKind, TimeInt, TimeType, Timeline, }; +use re_log_encoding::EncodingOptions; +const MSGPACK_COMPRESSED: EncodingOptions = EncodingOptions::MSGPACK_COMPRESSED; +const PROTOBUF_COMPRESSED: EncodingOptions = EncodingOptions::PROTOBUF_COMPRESSED; + use criterion::{criterion_group, criterion_main, Criterion}; #[cfg(not(debug_assertions))] @@ -31,8 +35,10 @@ criterion_group!( ); criterion_main!(benches); -fn encode_log_msgs(messages: &[LogMsg]) -> Vec { - let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; +fn encode_log_msgs( + messages: &[LogMsg], + encoding_options: re_log_encoding::EncodingOptions, +) -> Vec { let mut bytes = vec![]; re_log_encoding::encoder::encode_ref( re_build_info::CrateVersion::LOCAL, @@ -111,30 +117,67 @@ fn mono_points_arrow(c: &mut Criterion) { }); let messages = generate_messages(&store_id, &chunks); group.bench_function("encode_log_msg", |b| { - b.iter(|| encode_log_msgs(&messages)); + b.iter(|| encode_log_msgs(&messages, MSGPACK_COMPRESSED)); }); - group.bench_function("encode_total", |b| { - b.iter(|| encode_log_msgs(&generate_messages(&store_id, &generate_chunks()))); + group.bench_function("encode_log_msg(protobuf)", |b| { + b.iter(|| encode_log_msgs(&messages, PROTOBUF_COMPRESSED)); }); - - let encoded = encode_log_msgs(&messages); - group.bench_function("decode_log_msg", |b| { + group.bench_function("encode_total", |b| { b.iter(|| { - let decoded = decode_log_msgs(&encoded); - assert_eq!(decoded.len(), messages.len()); - decoded + encode_log_msgs( + &generate_messages(&store_id, &generate_chunks()), + MSGPACK_COMPRESSED, + ) }); }); - group.bench_function("decode_message_bundles", |b| { + group.bench_function("encode_total(protobuf)", |b| { b.iter(|| { - let chunks = decode_chunks(&messages); - assert_eq!(chunks.len(), messages.len()); - chunks + encode_log_msgs( + &generate_messages(&store_id, &generate_chunks()), + PROTOBUF_COMPRESSED, + ) }); }); - group.bench_function("decode_total", |b| { - b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); - }); + + { + let encoded = encode_log_msgs(&messages, MSGPACK_COMPRESSED); + group.bench_function("decode_log_msg", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles", |b| { + b.iter(|| { + let chunks = decode_chunks(&messages); + assert_eq!(chunks.len(), messages.len()); + chunks + }); + }); + group.bench_function("decode_total", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + + let encoded = encode_log_msgs(&messages, PROTOBUF_COMPRESSED); + group.bench_function("decode_log_msg(protobuf)", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles(protobuf)", |b| { + b.iter(|| { + let chunks = decode_chunks(&messages); + assert_eq!(chunks.len(), messages.len()); + chunks + }); + }); + group.bench_function("decode_total(protobuf)", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + } } } @@ -167,30 +210,67 @@ fn mono_points_arrow_batched(c: &mut Criterion) { }); let messages = generate_messages(&store_id, &chunks); group.bench_function("encode_log_msg", |b| { - b.iter(|| encode_log_msgs(&messages)); + b.iter(|| encode_log_msgs(&messages, MSGPACK_COMPRESSED)); }); - group.bench_function("encode_total", |b| { - b.iter(|| encode_log_msgs(&generate_messages(&store_id, &[generate_chunk()]))); + group.bench_function("encode_log_msg(protobuf)", |b| { + b.iter(|| encode_log_msgs(&messages, PROTOBUF_COMPRESSED)); }); - - let encoded = encode_log_msgs(&messages); - group.bench_function("decode_log_msg", |b| { + group.bench_function("encode_total", |b| { b.iter(|| { - let decoded = decode_log_msgs(&encoded); - assert_eq!(decoded.len(), messages.len()); - decoded + encode_log_msgs( + &generate_messages(&store_id, &[generate_chunk()]), + MSGPACK_COMPRESSED, + ) }); }); - group.bench_function("decode_message_bundles", |b| { + group.bench_function("encode_total(protobuf)", |b| { b.iter(|| { - let bundles = decode_chunks(&messages); - assert_eq!(bundles.len(), messages.len()); - bundles + encode_log_msgs( + &generate_messages(&store_id, &[generate_chunk()]), + PROTOBUF_COMPRESSED, + ) }); }); - group.bench_function("decode_total", |b| { - b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); - }); + + { + let encoded = encode_log_msgs(&messages, MSGPACK_COMPRESSED); + group.bench_function("decode_log_msg", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles", |b| { + b.iter(|| { + let bundles = decode_chunks(&messages); + assert_eq!(bundles.len(), messages.len()); + bundles + }); + }); + group.bench_function("decode_total", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + + let encoded = encode_log_msgs(&messages, PROTOBUF_COMPRESSED); + group.bench_function("decode_log_msg(protobuf)", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles(protobuf)", |b| { + b.iter(|| { + let bundles = decode_chunks(&messages); + assert_eq!(bundles.len(), messages.len()); + bundles + }); + }); + group.bench_function("decode_total(protobuf)", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + } } } @@ -222,30 +302,67 @@ fn batch_points_arrow(c: &mut Criterion) { }); let messages = generate_messages(&store_id, &chunks); group.bench_function("encode_log_msg", |b| { - b.iter(|| encode_log_msgs(&messages)); + b.iter(|| encode_log_msgs(&messages, MSGPACK_COMPRESSED)); }); - group.bench_function("encode_total", |b| { - b.iter(|| encode_log_msgs(&generate_messages(&store_id, &generate_chunks()))); + group.bench_function("encode_log_msg(protobuf)", |b| { + b.iter(|| encode_log_msgs(&messages, PROTOBUF_COMPRESSED)); }); - - let encoded = encode_log_msgs(&messages); - group.bench_function("decode_log_msg", |b| { + group.bench_function("encode_total", |b| { b.iter(|| { - let decoded = decode_log_msgs(&encoded); - assert_eq!(decoded.len(), messages.len()); - decoded + encode_log_msgs( + &generate_messages(&store_id, &generate_chunks()), + MSGPACK_COMPRESSED, + ) }); }); - group.bench_function("decode_message_bundles", |b| { + group.bench_function("encode_total(protobuf)", |b| { b.iter(|| { - let chunks = decode_chunks(&messages); - assert_eq!(chunks.len(), messages.len()); - chunks + encode_log_msgs( + &generate_messages(&store_id, &generate_chunks()), + PROTOBUF_COMPRESSED, + ) }); }); - group.bench_function("decode_total", |b| { - b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); - }); + + { + let encoded = encode_log_msgs(&messages, MSGPACK_COMPRESSED); + group.bench_function("decode_log_msg", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles", |b| { + b.iter(|| { + let chunks = decode_chunks(&messages); + assert_eq!(chunks.len(), messages.len()); + chunks + }); + }); + group.bench_function("decode_total", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + + let encoded = encode_log_msgs(&messages, PROTOBUF_COMPRESSED); + group.bench_function("decode_log_msg(protobuf)", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles(protobuf)", |b| { + b.iter(|| { + let chunks = decode_chunks(&messages); + assert_eq!(chunks.len(), messages.len()); + chunks + }); + }); + group.bench_function("decode_total(protobuf)", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + } } } From fed700feac83189da8938ff4b9d6d4f24e50171c Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 22:20:43 +0100 Subject: [PATCH 41/47] fix warning --- crates/build/re_protos_builder/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/build/re_protos_builder/Cargo.toml b/crates/build/re_protos_builder/Cargo.toml index 266537383c4e..6f100bcd9d49 100644 --- a/crates/build/re_protos_builder/Cargo.toml +++ b/crates/build/re_protos_builder/Cargo.toml @@ -29,4 +29,4 @@ camino.workspace = true tonic-build = { workspace = true, default-features = false, features = [ "prost", ] } -prost-build = { workspace = true, default-features = false } +prost-build = { workspace = true } From 9683405b47ad62c9d7d426f7d6d8486e8f1300e0 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 22:37:00 +0100 Subject: [PATCH 42/47] remove unused dep --- Cargo.lock | 1 - crates/store/re_log_encoding/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c6b543b04047..57c976b2fe6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5985,7 +5985,6 @@ dependencies = [ "re_smart_channel", "re_tracing", "re_types", - "re_types_core", "rmp-serde", "serde_test", "thiserror", diff --git a/crates/store/re_log_encoding/Cargo.toml b/crates/store/re_log_encoding/Cargo.toml index 20ac933112e1..5fd58ed1ae61 100644 --- a/crates/store/re_log_encoding/Cargo.toml +++ b/crates/store/re_log_encoding/Cargo.toml @@ -49,7 +49,6 @@ re_log.workspace = true re_protos.workspace = true re_smart_channel.workspace = true re_tracing.workspace = true -re_types_core.workspace = true # External: arrow2.workspace = true From ec1c6b667a1f2610292837be0271224c960ebd01 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 11 Dec 2024 22:54:14 +0100 Subject: [PATCH 43/47] fix --- crates/store/re_log_encoding/src/codec/arrow.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/store/re_log_encoding/src/codec/arrow.rs b/crates/store/re_log_encoding/src/codec/arrow.rs index e2762c737cb3..ae7cddabe27e 100644 --- a/crates/store/re_log_encoding/src/codec/arrow.rs +++ b/crates/store/re_log_encoding/src/codec/arrow.rs @@ -1,6 +1,3 @@ -use crate::decoder::DecodeError; -use crate::encoder::EncodeError; - use super::CodecError; use arrow2::array::Array as Arrow2Array; @@ -52,16 +49,18 @@ pub(crate) fn read_arrow_from_bytes( } } +#[cfg(feature = "encoder")] pub(crate) struct Payload { pub uncompressed_size: usize, pub data: Vec, } +#[cfg(feature = "encoder")] pub(crate) fn encode_arrow( schema: &Arrow2Schema, chunk: &Arrow2Chunk, compression: crate::Compression, -) -> Result { +) -> Result { let mut uncompressed = Vec::new(); write_arrow_to_bytes(&mut uncompressed, schema, chunk)?; let uncompressed_size = uncompressed.len(); @@ -77,11 +76,12 @@ pub(crate) fn encode_arrow( }) } +#[cfg(feature = "decoder")] pub(crate) fn decode_arrow( data: &[u8], uncompressed_size: usize, compression: crate::Compression, -) -> Result<(Arrow2Schema, Arrow2Chunk), DecodeError> { +) -> Result<(Arrow2Schema, Arrow2Chunk), crate::decoder::DecodeError> { let mut uncompressed = Vec::new(); let data = match compression { crate::Compression::Off => data, From 297ab19167c9abdf5e889a5b3fb93bee834267e7 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 12 Dec 2024 12:01:37 +0100 Subject: [PATCH 44/47] add test for python version parsing --- crates/store/re_log_types/src/lib.rs | 72 ++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/crates/store/re_log_types/src/lib.rs b/crates/store/re_log_types/src/lib.rs index 26ea4105ca6d..57ea6d7167ca 100644 --- a/crates/store/re_log_types/src/lib.rs +++ b/crates/store/re_log_types/src/lib.rs @@ -411,26 +411,26 @@ impl std::str::FromStr for PythonVersion { type Err = PythonVersionParseError; fn from_str(s: &str) -> Result { - let (major, rest) = s - .split_once('.') - .ok_or(PythonVersionParseError::MissingMajor)?; - if major.is_empty() { + if s.is_empty() { return Err(PythonVersionParseError::MissingMajor); } - let (minor, rest) = rest + let (major, rest) = s .split_once('.') .ok_or(PythonVersionParseError::MissingMinor)?; - if minor.is_empty() { + if rest.is_empty() { return Err(PythonVersionParseError::MissingMinor); } + let (minor, rest) = rest + .split_once('.') + .ok_or(PythonVersionParseError::MissingPatch)?; + if rest.is_empty() { + return Err(PythonVersionParseError::MissingPatch); + } let pos = rest.bytes().position(|v| !v.is_ascii_digit()); let (patch, suffix) = match pos { Some(pos) => rest.split_at(pos), None => (rest, ""), }; - if patch.is_empty() { - return Err(PythonVersionParseError::MissingPatch); - } Ok(Self { major: major @@ -634,3 +634,57 @@ pub fn build_frame_nr(frame_nr: impl TryInto) -> (Timeline, TimeInt) { frame_nr.try_into().unwrap_or(TimeInt::MIN), ) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_python_version() { + macro_rules! assert_parse_err { + ($input:literal, $expected:pat) => { + let actual = $input.parse::(); + + assert!( + matches!(actual, Err($expected)), + "actual: {actual:?}, expected: {}", + stringify!($expected) + ); + }; + } + + macro_rules! assert_parse_ok { + ($input:literal, $expected:expr) => { + let actual = $input.parse::().expect("failed to parse"); + assert_eq!(actual, $expected); + }; + } + + assert_parse_err!("", PythonVersionParseError::MissingMajor); + assert_parse_err!("3", PythonVersionParseError::MissingMinor); + assert_parse_err!("3.", PythonVersionParseError::MissingMinor); + assert_parse_err!("3.11", PythonVersionParseError::MissingPatch); + assert_parse_err!("3.11.", PythonVersionParseError::MissingPatch); + assert_parse_err!("a.11.0", PythonVersionParseError::InvalidMajor(_)); + assert_parse_err!("3.b.0", PythonVersionParseError::InvalidMinor(_)); + assert_parse_err!("3.11.c", PythonVersionParseError::InvalidPatch(_)); + assert_parse_ok!( + "3.11.0", + PythonVersion { + major: 3, + minor: 11, + patch: 0, + suffix: "".to_owned(), + } + ); + assert_parse_ok!( + "3.11.0a1", + PythonVersion { + major: 3, + minor: 11, + patch: 0, + suffix: "a1".to_owned(), + } + ); + } +} From 1d7aecbcb52858bb504fd751b563352773559dba Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 12 Dec 2024 12:02:31 +0100 Subject: [PATCH 45/47] mark `src/v0` as generated --- crates/store/re_protos/.gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_protos/.gitattributes b/crates/store/re_protos/.gitattributes index be213900d102..80caba21b9b2 100644 --- a/crates/store/re_protos/.gitattributes +++ b/crates/store/re_protos/.gitattributes @@ -1 +1 @@ -src/v0/rerun.remote_store.v0.rs linguist-generated=true +src/v0 linguist-generated=true From df46547e4df90b47560ccd5b92db8a4560853a9e Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 12 Dec 2024 12:03:29 +0100 Subject: [PATCH 46/47] fix pattern --- crates/store/re_protos/.gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_protos/.gitattributes b/crates/store/re_protos/.gitattributes index 80caba21b9b2..7dd060db3f46 100644 --- a/crates/store/re_protos/.gitattributes +++ b/crates/store/re_protos/.gitattributes @@ -1 +1 @@ -src/v0 linguist-generated=true +src/v0/** linguist-generated=true From cd6ab2ee071d25215f591cb0101560d07521a875 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 12 Dec 2024 12:35:43 +0100 Subject: [PATCH 47/47] fix lint --- crates/store/re_log_types/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_log_types/src/lib.rs b/crates/store/re_log_types/src/lib.rs index 57ea6d7167ca..3174228b7750 100644 --- a/crates/store/re_log_types/src/lib.rs +++ b/crates/store/re_log_types/src/lib.rs @@ -674,7 +674,7 @@ mod tests { major: 3, minor: 11, patch: 0, - suffix: "".to_owned(), + suffix: String::new(), } ); assert_parse_ok!(