Move trait SizeBytes to new crate re_byte_size (#8544)

This is so that I can move `fn arrow_ui` (which depend on `SizeBytes`) into `re_ui`, as part of improved testing of it before migrating it to arrow1. And `re_byte_size` sounds nicer than `re_sizes_bytes`. Maybe we should rename the trait to `ByteSize` too. --------- Co-authored-by: Clement Rey <[email protected]>
rerun-io · Dec 20, 2024 · 8ff8336 · 8ff8336
1 parent 649d74c
commit 8ff8336
Show file tree

Hide file tree

Showing 337 changed files with 787 additions and 620 deletions.
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
@@ -209,7 +209,8 @@ Update instructions:
 | Crate              | Description                                                                          |
 |--------------------|--------------------------------------------------------------------------------------|
 | re_analytics       | Rerun's analytics SDK                                                                |
-| re_capabilities    | Capability tokens                                                                   |
+| re_byte_size       | Calculate the heap-allocated size of values at runtime                               |
+| re_capabilities    | Capability tokens                                                                    |
 | re_case            | Case conversions, the way Rerun likes them                                           |
 | re_crash_handler   | Detect panics and signals, logging them and optionally sending them to analytics.    |
 | re_error           | Helpers for handling errors.                                                         |

diff --git a/Cargo.lock b/Cargo.lock
@@ -5605,6 +5605,17 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "re_byte_size"
+version = "0.22.0-alpha.1+dev"
+dependencies = [
+ "arrow",
+ "half",
+ "re_arrow2",
+ "re_tuid",
+ "smallvec",
+]
+
 [[package]]
 name = "re_capabilities"
 version = "0.22.0-alpha.1+dev"
@@ -5637,6 +5648,7 @@ dependencies = [
  "nohash-hasher",
  "rand",
  "re_arrow2",
+ "re_byte_size",
  "re_error",
  "re_format",
  "re_format_arrow",
@@ -5667,6 +5679,7 @@ dependencies = [
  "parking_lot",
  "rand",
  "re_arrow2",
+ "re_byte_size",
  "re_chunk",
  "re_format",
  "re_log",
@@ -5689,6 +5702,7 @@ dependencies = [
  "egui",
  "egui_extras",
  "itertools 0.13.0",
+ "re_byte_size",
  "re_chunk_store",
  "re_format",
  "re_log_types",
@@ -5708,6 +5722,7 @@ dependencies = [
  "egui_plot",
  "itertools 0.13.0",
  "nohash-hasher",
+ "re_byte_size",
  "re_data_source",
  "re_data_ui",
  "re_format",
@@ -5814,6 +5829,7 @@ dependencies = [
  "image",
  "itertools 0.13.0",
  "nohash-hasher",
+ "re_byte_size",
  "re_capabilities",
  "re_chunk_store",
  "re_entity_db",
@@ -5897,6 +5913,7 @@ dependencies = [
  "parking_lot",
  "rand",
  "re_build_info",
+ "re_byte_size",
  "re_chunk",
  "re_chunk_store",
  "re_format",
@@ -6033,6 +6050,7 @@ dependencies = [
  "num-traits",
  "re_arrow2",
  "re_build_info",
+ "re_byte_size",
  "re_format",
  "re_log",
  "re_protos",
@@ -6133,6 +6151,7 @@ dependencies = [
  "paste",
  "rand",
  "re_arrow2",
+ "re_byte_size",
  "re_chunk",
  "re_chunk_store",
  "re_error",
@@ -6264,6 +6283,7 @@ dependencies = [
  "re_arrow2",
  "re_build_info",
  "re_build_tools",
+ "re_byte_size",
  "re_chunk",
  "re_chunk_store",
  "re_data_loader",
@@ -6428,6 +6448,7 @@ dependencies = [
  "rayon",
  "re_arrow2",
  "re_build_tools",
+ "re_byte_size",
  "re_format",
  "re_log",
  "re_log_types",
@@ -6484,18 +6505,17 @@ dependencies = [
  "bytemuck",
  "criterion",
  "document-features",
- "half",
  "itertools 0.13.0",
  "nohash-hasher",
  "once_cell",
  "re_arrow2",
+ "re_byte_size",
  "re_case",
  "re_error",
  "re_string_interner",
  "re_tracing",
  "re_tuid",
  "serde",
- "smallvec",
  "thiserror 1.0.65",
 ]
 
@@ -7203,6 +7223,7 @@ dependencies = [
  "re_analytics",
  "re_build_info",
  "re_build_tools",
+ "re_byte_size",
  "re_capabilities",
  "re_chunk",
  "re_chunk_store",

diff --git a/Cargo.toml b/Cargo.toml
@@ -80,6 +80,7 @@ re_format = { path = "crates/utils/re_format", version = "=0.22.0-alpha.1", defa
 re_int_histogram = { path = "crates/utils/re_int_histogram", version = "=0.22.0-alpha.1", default-features = false }
 re_log = { path = "crates/utils/re_log", version = "=0.22.0-alpha.1", default-features = false }
 re_memory = { path = "crates/utils/re_memory", version = "=0.22.0-alpha.1", default-features = false }
+re_byte_size = { path = "crates/utils/re_byte_size", version = "=0.22.0-alpha.1", default-features = false }
 re_smart_channel = { path = "crates/utils/re_smart_channel", version = "=0.22.0-alpha.1", default-features = false }
 re_string_interner = { path = "crates/utils/re_string_interner", version = "=0.22.0-alpha.1", default-features = false }
 re_tracing = { path = "crates/utils/re_tracing", version = "=0.22.0-alpha.1", default-features = false }

diff --git a/crates/build/re_types_builder/src/codegen/rust/api.rs b/crates/build/re_types_builder/src/codegen/rust/api.rs
@@ -331,7 +331,7 @@ fn quote_struct(
         };
 
         quote! {
-            impl ::re_types_core::SizeBytes for #name {
+            impl ::re_byte_size::SizeBytes for #name {
                 #[inline]
                 fn heap_size_bytes(&self) -> u64 {
                     #heap_size_bytes_impl
@@ -443,7 +443,7 @@ fn quote_union(
         };
 
         quote! {
-            impl ::re_types_core::SizeBytes for #name {
+            impl ::re_byte_size::SizeBytes for #name {
                 #[inline]
                 fn heap_size_bytes(&self) -> u64 {
                     #![allow(clippy::match_same_arms)]
@@ -630,7 +630,7 @@ fn quote_enum(
             }
         }
 
-        impl ::re_types_core::SizeBytes for #name {
+        impl ::re_byte_size::SizeBytes for #name {
             #[inline]
             fn heap_size_bytes(&self) -> u64 {
                 0

diff --git a/crates/store/re_chunk/Cargo.toml b/crates/store/re_chunk/Cargo.toml
@@ -37,6 +37,7 @@ arrow = ["arrow2/arrow"]
 [dependencies]
 
 # Rerun
+re_byte_size.workspace = true
 re_error.workspace = true
 re_format.workspace = true
 re_format_arrow.workspace = true

diff --git a/crates/store/re_chunk/src/batcher.rs b/crates/store/re_chunk/src/batcher.rs
@@ -8,8 +8,9 @@ use arrow2::array::{Array as Arrow2Array, PrimitiveArray as Arrow2PrimitiveArray
 use crossbeam::channel::{Receiver, Sender};
 use nohash_hasher::IntMap;
 
+use re_byte_size::SizeBytes as _;
 use re_log_types::{EntityPath, ResolvedTimeRange, TimeInt, TimePoint, Timeline};
-use re_types_core::{ComponentDescriptor, SizeBytes as _};
+use re_types_core::ComponentDescriptor;
 
 use crate::{chunk::ChunkComponents, Chunk, ChunkId, ChunkResult, RowId, TimeColumn};
 
@@ -695,7 +696,7 @@ impl PendingRow {
     }
 }
 
-impl re_types_core::SizeBytes for PendingRow {
+impl re_byte_size::SizeBytes for PendingRow {
     #[inline]
     fn heap_size_bytes(&self) -> u64 {
         let Self {

diff --git a/crates/store/re_chunk/src/chunk.rs b/crates/store/re_chunk/src/chunk.rs
@@ -8,14 +8,14 @@ use arrow2::{
     },
     Either,
 };
-
 use itertools::{izip, Itertools};
 use nohash_hasher::IntMap;
 
+use re_byte_size::SizeBytes as _;
 use re_log_types::{EntityPath, ResolvedTimeRange, Time, TimeInt, TimePoint, Timeline};
 use re_types_core::{
     ComponentDescriptor, ComponentName, DeserializationError, Loggable, LoggableBatch,
-    SerializationError, SizeBytes,
+    SerializationError,
 };
 
 use crate::{ChunkId, RowId};
@@ -1399,7 +1399,7 @@ impl TimeColumn {
     }
 }
 
-impl re_types_core::SizeBytes for Chunk {
+impl re_byte_size::SizeBytes for Chunk {
     #[inline]
     fn heap_size_bytes(&self) -> u64 {
         let Self {
@@ -1428,7 +1428,7 @@ impl re_types_core::SizeBytes for Chunk {
     }
 }
 
-impl re_types_core::SizeBytes for TimeColumn {
+impl re_byte_size::SizeBytes for TimeColumn {
     #[inline]
     fn heap_size_bytes(&self) -> u64 {
         let Self {

diff --git a/crates/store/re_chunk/src/helpers.rs b/crates/store/re_chunk/src/helpers.rs
@@ -4,7 +4,7 @@ use arrow::array::ArrayRef;
 use arrow2::array::Array as Arrow2Array;
 
 use re_log_types::{TimeInt, Timeline};
-use re_types_core::{Component, ComponentName, SizeBytes};
+use re_types_core::{Component, ComponentName};
 
 use crate::{Chunk, ChunkResult, RowId};
 
@@ -175,7 +175,7 @@ impl std::ops::Deref for UnitChunkShared {
     }
 }
 
-impl SizeBytes for UnitChunkShared {
+impl re_byte_size::SizeBytes for UnitChunkShared {
     #[inline]
     fn heap_size_bytes(&self) -> u64 {
         Chunk::heap_size_bytes(&self.0)

diff --git a/crates/store/re_chunk/src/id.rs b/crates/store/re_chunk/src/id.rs
@@ -86,7 +86,7 @@ impl ChunkId {
     }
 }
 
-impl re_types_core::SizeBytes for ChunkId {
+impl re_byte_size::SizeBytes for ChunkId {
     #[inline]
     fn heap_size_bytes(&self) -> u64 {
         0
@@ -223,7 +223,7 @@ impl RowId {
     }
 }
 
-impl re_types_core::SizeBytes for RowId {
+impl re_byte_size::SizeBytes for RowId {
     #[inline]
     fn heap_size_bytes(&self) -> u64 {
         0

diff --git a/crates/store/re_chunk/src/lib.rs b/crates/store/re_chunk/src/lib.rs
@@ -52,6 +52,7 @@ pub mod external {
     pub use arrow2;
     pub use nohash_hasher;
 
+    pub use re_byte_size;
     pub use re_log_types;
 
     #[cfg(not(target_arch = "wasm32"))]

diff --git a/crates/store/re_chunk/src/transport.rs b/crates/store/re_chunk/src/transport.rs
@@ -9,11 +9,12 @@ use arrow2::{
         Schema as Arrow2Schema, TimeUnit as ArrowTimeUnit,
     },
 };
-
 use itertools::Itertools;
 use nohash_hasher::IntMap;
+
+use re_byte_size::SizeBytes as _;
 use re_log_types::{EntityPath, Timeline};
-use re_types_core::{Component as _, ComponentDescriptor, Loggable as _, SizeBytes};
+use re_types_core::{Component as _, ComponentDescriptor, Loggable as _};
 
 use crate::{chunk::ChunkComponents, Chunk, ChunkError, ChunkId, ChunkResult, RowId, TimeColumn};
 

diff --git a/crates/store/re_chunk_store/Cargo.toml b/crates/store/re_chunk_store/Cargo.toml
@@ -27,6 +27,7 @@ deadlock_detection = ["parking_lot/deadlock_detection"]
 
 [dependencies]
 # Rerun dependencies:
+re_byte_size.workspace = true
 re_chunk.workspace = true
 re_format.workspace = true
 re_log = { workspace = true, features = ["setup"] }

diff --git a/crates/store/re_chunk_store/src/gc.rs b/crates/store/re_chunk_store/src/gc.rs
@@ -5,11 +5,12 @@ use std::{
 
 use ahash::{HashMap, HashSet};
 use nohash_hasher::IntMap;
+use re_byte_size::SizeBytes;
 use web_time::Instant;
 
 use re_chunk::{Chunk, ChunkId};
 use re_log_types::{EntityPath, ResolvedTimeRange, TimeInt, Timeline};
-use re_types_core::{ComponentName, SizeBytes};
+use re_types_core::ComponentName;
 
 use crate::{
     store::ChunkIdSetPerTime, ChunkStore, ChunkStoreChunkStats, ChunkStoreDiff, ChunkStoreDiffKind,

diff --git a/crates/store/re_chunk_store/src/stats.rs b/crates/store/re_chunk_store/src/stats.rs
@@ -1,7 +1,7 @@
 use std::sync::{atomic::Ordering, Arc};
 
+use re_byte_size::SizeBytes;
 use re_chunk::{Chunk, ComponentName, EntityPath, Timeline};
-use re_types_core::SizeBytes;
 
 use crate::ChunkStore;
 

diff --git a/crates/store/re_chunk_store/src/writes.rs b/crates/store/re_chunk_store/src/writes.rs
@@ -4,8 +4,8 @@ use ahash::HashMap;
 use arrow2::array::{Array as _, ListArray as Arrow2ListArray};
 use itertools::Itertools as _;
 
+use re_byte_size::SizeBytes;
 use re_chunk::{Chunk, EntityPath, RowId};
-use re_types_core::SizeBytes;
 
 use crate::{
     store::ChunkIdSetPerTime, ChunkStore, ChunkStoreChunkStats, ChunkStoreConfig, ChunkStoreDiff,

diff --git a/crates/store/re_entity_db/Cargo.toml b/crates/store/re_entity_db/Cargo.toml
@@ -28,6 +28,7 @@ serde = ["dep:serde", "re_log_types/serde"]
 
 [dependencies]
 re_build_info.workspace = true
+re_byte_size.workspace = true
 re_chunk = { workspace = true, features = ["serde"] }
 re_chunk_store.workspace = true
 re_format.workspace = true

diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs
@@ -761,7 +761,7 @@ impl EntityDb {
     }
 }
 
-impl re_types_core::SizeBytes for EntityDb {
+impl re_byte_size::SizeBytes for EntityDb {
     #[inline]
     fn heap_size_bytes(&self) -> u64 {
         // TODO(emilk): size of entire EntityDb, including secondary indices etc

diff --git a/crates/store/re_log_types/Cargo.toml b/crates/store/re_log_types/Cargo.toml
@@ -43,6 +43,7 @@ serde = [
 
 # Rerun
 re_build_info.workspace = true
+re_byte_size.workspace = true
 re_format.workspace = true
 re_log.workspace = true
 re_protos.workspace = true

diff --git a/crates/store/re_log_types/src/example_components.rs b/crates/store/re_log_types/src/example_components.rs
@@ -2,7 +2,8 @@
 
 use std::sync::Arc;
 
-use re_types_core::{Component, ComponentDescriptor, DeserializationError, Loggable, SizeBytes};
+use re_byte_size::SizeBytes;
+use re_types_core::{Component, ComponentDescriptor, DeserializationError, Loggable};
 
 // ----------------------------------------------------------------------------
 

diff --git a/crates/store/re_log_types/src/hash.rs b/crates/store/re_log_types/src/hash.rs
@@ -7,7 +7,7 @@
 #[derive(Copy, Clone, Eq, PartialOrd, Ord)]
 pub struct Hash64(u64);
 
-impl re_types_core::SizeBytes for Hash64 {
+impl re_byte_size::SizeBytes for Hash64 {
     #[inline]
     fn heap_size_bytes(&self) -> u64 {
         0

diff --git a/crates/store/re_log_types/src/path/entity_path.rs b/crates/store/re_log_types/src/path/entity_path.rs
@@ -3,8 +3,8 @@ use std::sync::Arc;
 use ahash::{HashMap, HashSet};
 use itertools::Itertools as _;
 
+use re_byte_size::SizeBytes;
 use re_string_interner::InternedString;
-use re_types_core::SizeBytes;
 
 use crate::{hash::Hash64, EntityPathPart};
 
@@ -14,7 +14,7 @@ use crate::{hash::Hash64, EntityPathPart};
 #[derive(Copy, Clone, Eq, PartialOrd, Ord)]
 pub struct EntityPathHash(Hash64);
 
-impl re_types_core::SizeBytes for EntityPathHash {
+impl re_byte_size::SizeBytes for EntityPathHash {
     #[inline]
     fn heap_size_bytes(&self) -> u64 {
         self.0.heap_size_bytes()