Merge branch 'main' into grtlr/better-example

.github/workflows/nightly.yml

@@ -15,6 +15,9 @@ permissions:
   contents: "write"
   id-token: "write"
   deployments: "write"
+  # This is needed since the web viewer build has this permission in order to write comments in PRs
+  # (not needed for nightly, but the permission is still active).
+  pull-requests: "write"
 
 jobs:
   checks:

ARCHITECTURE.md

@@ -88,11 +88,11 @@ Of course, this will only take us so far. In the future we plan on caching queri
 Here is an overview of the crates included in the project:
 
 <picture>
-  <img src="https://static.rerun.io/architecture/1cc75ea6e625e77e94996b1c9aef8a0cca08bfcc/full.png" alt="">
-  <source media="(max-width: 480px)" srcset="https://static.rerun.io/architecture/1cc75ea6e625e77e94996b1c9aef8a0cca08bfcc/480w.png">
-  <source media="(max-width: 768px)" srcset="https://static.rerun.io/architecture/1cc75ea6e625e77e94996b1c9aef8a0cca08bfcc/768w.png">
-  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/architecture/1cc75ea6e625e77e94996b1c9aef8a0cca08bfcc/1024w.png">
-  <source media="(max-width: 1200px)" srcset="https://static.rerun.io/architecture/1cc75ea6e625e77e94996b1c9aef8a0cca08bfcc/1200w.png">
+  <img src="https://static.rerun.io/crates/8044a2d9cf4e26687eaaf43352c391cb7aac557b/full.png" alt="">
+  <source media="(max-width: 480px)" srcset="https://static.rerun.io/crates/8044a2d9cf4e26687eaaf43352c391cb7aac557b/480w.png">
+  <source media="(max-width: 768px)" srcset="https://static.rerun.io/crates/8044a2d9cf4e26687eaaf43352c391cb7aac557b/768w.png">
+  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/crates/8044a2d9cf4e26687eaaf43352c391cb7aac557b/1024w.png">
+  <source media="(max-width: 1200px)" srcset="https://static.rerun.io/crates/8044a2d9cf4e26687eaaf43352c391cb7aac557b/1200w.png">
 </picture>
 
 
@@ -168,7 +168,6 @@ Update instructions:
 | re_protos             | Rerun remote store gRPC API types                                        |
 | re_query              | Querying data in the re_chunk_store                                      |
 | re_types              | The built-in Rerun data types, component types, and archetypes.          |
-| re_types_blueprint    | The core traits and types that power Rerun's Blueprint sub-system.       |
 
 
 ### Low-level store

Cargo.lock

@@ -5689,7 +5689,6 @@ dependencies = [
  "re_log_types",
  "re_tracing",
  "re_types",
- "re_types_blueprint",
  "re_types_core",
  "re_ui",
  "re_viewer_context",
@@ -5923,6 +5922,7 @@ dependencies = [
  "re_log_types",
  "re_protos",
  "re_smart_channel",
+ "re_types",
  "thiserror",
  "tokio",
  "tokio-stream",
@@ -6286,7 +6286,6 @@ dependencies = [
  "re_log_types",
  "re_tracing",
  "re_types",
- "re_types_blueprint",
  "re_types_core",
  "re_ui",
  "re_view",
@@ -6414,18 +6413,6 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "re_types_blueprint"
-version = "0.21.0-alpha.1+dev"
-dependencies = [
- "arrow",
- "once_cell",
- "re_arrow2",
- "re_tracing",
- "re_types",
- "re_types_core",
-]
-
 [[package]]
 name = "re_types_builder"
 version = "0.21.0-alpha.1+dev"
@@ -6819,7 +6806,6 @@ dependencies = [
  "re_time_panel",
  "re_tracing",
  "re_types",
- "re_types_blueprint",
  "re_types_core",
  "re_ui",
  "re_video",
@@ -6923,7 +6909,6 @@ dependencies = [
  "re_renderer",
  "re_tracing",
  "re_types",
- "re_types_blueprint",
  "re_ui",
  "re_view",
  "re_viewer_context",
@@ -6948,7 +6933,6 @@ dependencies = [
  "re_log_types",
  "re_tracing",
  "re_types",
- "re_types_blueprint",
  "re_types_core",
  "re_ui",
  "re_viewer_context",

Cargo.toml

@@ -61,7 +61,6 @@ re_log_types = { path = "crates/store/re_log_types", version = "=0.21.0-alpha.1"
 re_query = { path = "crates/store/re_query", version = "=0.21.0-alpha.1", default-features = false }
 re_sdk_comms = { path = "crates/store/re_sdk_comms", version = "=0.21.0-alpha.1", default-features = false }
 re_types = { path = "crates/store/re_types", version = "=0.21.0-alpha.1", default-features = false }
-re_types_blueprint = { path = "crates/store/re_types_blueprint", version = "=0.21.0-alpha.1", default-features = false }
 re_types_core = { path = "crates/store/re_types_core", version = "=0.21.0-alpha.1", default-features = false }
 re_ws_comms = { path = "crates/store/re_ws_comms", version = "=0.21.0-alpha.1", default-features = false }
 

crates/build/re_types_builder/src/codegen/rust/reflection.rs

@@ -23,7 +23,7 @@ pub fn generate_reflection(
     // Put into its own subfolder since codegen is set up in a way that it thinks that everything
     // inside the folder is either generated or an extension to the generated code.
     // This way we don't have to build an exception just for this file.
-    let path = Utf8PathBuf::from("crates/viewer/re_viewer/src/reflection/mod.rs");
+    let path = Utf8PathBuf::from("crates/store/re_types/src/reflection/mod.rs");
 
     let mut imports = BTreeSet::new();
     let component_reflection = generate_component_reflection(
@@ -60,9 +60,9 @@ pub fn generate_reflection(
             SerializationError,
         };
 
-        /// Generates reflection about all known components.
-        ///
-        /// Call only once and reuse the results.
+        #[doc = "Generates reflection about all known components."]
+        #[doc = ""]
+        #[doc = "Call only once and reuse the results."]
         pub fn generate_reflection() -> Result<Reflection, SerializationError> {
             re_tracing::profile_function!();
 
@@ -95,10 +95,11 @@ fn generate_component_reflection(
         .objects_of_kind(ObjectKind::Component)
         .filter(|obj| !obj.is_testing())
     {
+        let crate_name = patched_crate_name(&obj.crate_name());
         if let Some(scope) = obj.scope() {
-            imports.insert(format!("{}::{scope}::components::*", obj.crate_name()));
+            imports.insert(format!("{crate_name}::{scope}::components::*"));
         } else {
-            imports.insert(format!("{}::components::*", obj.crate_name()));
+            imports.insert(format!("{crate_name}::components::*"));
         }
 
         let type_name = format_ident!("{}", obj.name);
@@ -152,9 +153,9 @@ fn generate_component_reflection(
     }
 
     quote! {
-        /// Generates reflection about all known components.
-        ///
-        /// Call only once and reuse the results.
+        #[doc = "Generates reflection about all known components."]
+        #[doc = ""]
+        #[doc = "Call only once and reuse the results."]
         fn generate_component_reflection() -> Result<ComponentReflectionMap, SerializationError> {
             re_tracing::profile_function!();
             let array = [
@@ -246,9 +247,9 @@ fn generate_archetype_reflection(reporter: &Reporter, objects: &Objects) -> Toke
     }
 
     quote! {
-        /// Generates reflection about all known archetypes.
-        ///
-        /// Call only once and reuse the results.
+        #[doc = "Generates reflection about all known archetypes."]
+        #[doc = ""]
+        #[doc = "Call only once and reuse the results."]
         fn generate_archetype_reflection() -> ArchetypeReflectionMap {
             re_tracing::profile_function!();
             let array = [
@@ -258,3 +259,13 @@ fn generate_archetype_reflection(reporter: &Reporter, objects: &Objects) -> Toke
         }
     }
 }
+
+/// Returns `crate_name` as is, unless it's `re_types`, in which case it's replace by `crate`,
+/// because that's where this code lives.
+fn patched_crate_name(crate_name: &str) -> String {
+    if crate_name == "re_types" {
+        "crate".to_owned()
+    } else {
+        crate_name.to_owned()
+    }
+}

crates/store/re_grpc_client/Cargo.toml

@@ -27,6 +27,7 @@ re_log_encoding.workspace = true
 re_log_types.workspace = true
 re_protos.workspace = true
 re_smart_channel.workspace = true
+re_types.workspace = true
 
 thiserror.workspace = true
 tokio-stream.workspace = true

crates/store/re_grpc_client/src/lib.rs

@@ -3,13 +3,19 @@
 mod address;
 
 pub use address::{InvalidRedapAddress, RedapAddress};
+use re_chunk::external::arrow2;
+use re_log_types::external::re_types_core::ComponentDescriptor;
+use re_types::components::RecordingUri;
+use re_types::Component;
 use url::Url;
 
 // ----------------------------------------------------------------------------
 
 use std::error::Error;
 
-use re_chunk::Chunk;
+use arrow2::array::Utf8Array as Arrow2Utf8Array;
+use arrow2::datatypes::Field as Arrow2Field;
+use re_chunk::{Arrow2Array, Chunk, ChunkId, TransportChunk};
 use re_log_encoding::codec::{wire::decode, CodecError};
 use re_log_types::{
     ApplicationId, LogMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time,
@@ -64,6 +70,9 @@ enum StreamError {
 
     #[error(transparent)]
     ChunkError(#[from] re_chunk::ChunkError),
+
+    #[error("Invalid URI: {0}")]
+    InvalidUri(String),
 }
 
 // ----------------------------------------------------------------------------
@@ -179,7 +188,7 @@ async fn stream_recording_async(
     let store_id = StoreId::from_string(StoreKind::Recording, recording_id.clone());
 
     let store_info = StoreInfo {
-        application_id: ApplicationId::from("rerun_data_platform"),
+        application_id: ApplicationId::from("redap_recording"),
         store_id: store_id.clone(),
         cloned_from: None,
         is_official_example: false,
@@ -221,9 +230,6 @@ async fn stream_recording_async(
     Ok(())
 }
 
-/// TODO(zehiko) - this is a copy of `stream_recording_async` with a different gRPC call,
-/// this will go away as we tackle unification of data and metadata streams REDAP #74, hence
-/// avoiding refactoring right now
 async fn stream_catalog_async(
     tx: re_smart_channel::Sender<LogMsg>,
     redap_endpoint: Url,
@@ -273,7 +279,7 @@ async fn stream_catalog_async(
     let store_id = StoreId::from_string(StoreKind::Recording, "catalog".to_owned());
 
     let store_info = StoreInfo {
-        application_id: ApplicationId::from("rerun_data_platform"),
+        application_id: ApplicationId::from("redap_catalog"),
         store_id: store_id.clone(),
         cloned_from: None,
         is_official_example: false,
@@ -295,8 +301,64 @@ async fn stream_catalog_async(
 
     re_log::info!("Starting to read...");
     while let Some(result) = resp.next().await {
-        let tc = result.map_err(TonicStatusError)?;
-        let chunk = Chunk::from_transport(&tc)?;
+        let mut tc = result.map_err(TonicStatusError)?;
+        // received TransportChunk doesn't have ChunkId, hence we need to add it before converting
+        // to Chunk
+        tc.schema.metadata.insert(
+            TransportChunk::CHUNK_METADATA_KEY_ID.to_owned(),
+            ChunkId::new().to_string(),
+        );
+
+        let mut chunk = Chunk::from_transport(&tc)?;
+
+        // enrich catalog data with RecordingUri that's based on the ReDap endpoint (that we know)
+        // and the recording id (that we have in the catalog data)
+        let host = redap_endpoint
+            .host()
+            .ok_or(StreamError::InvalidUri(format!(
+                "couldn't get host from {redap_endpoint}"
+            )))?;
+        let port = redap_endpoint
+            .port()
+            .ok_or(StreamError::InvalidUri(format!(
+                "couldn't get port from {redap_endpoint}"
+            )))?;
+
+        let recording_uri_arrays: Vec<Box<dyn Arrow2Array>> = chunk
+            .iter_component_arrays(&"id".into())
+            .map(|id| {
+                let rec_id = id
+                    .as_any()
+                    .downcast_ref::<Arrow2Utf8Array<i32>>()
+                    .ok_or(StreamError::ChunkError(re_chunk::ChunkError::Malformed {
+                        reason: format!("id must be a utf8 array: {:?}", tc.schema),
+                    }))?
+                    .value(0); // each component batch is of length 1 i.e. single 'id' value
+
+                let recording_uri = format!("rerun://{host}:{port}/recording/{rec_id}");
+
+                let recording_uri_data = Arrow2Utf8Array::<i32>::from([Some(recording_uri)]);
+
+                Ok::<Box<_>, StreamError>(
+                    Box::new(recording_uri_data) as Box<dyn arrow2::array::Array>
+                )
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+
+        let recording_id_arrays = recording_uri_arrays
+            .iter()
+            .map(|e| Some(e.as_ref()))
+            .collect::<Vec<_>>();
+
+        let rec_id_field = Arrow2Field::new("item", arrow2::datatypes::DataType::Utf8, true);
+        #[allow(clippy::unwrap_used)] // we know we've given the right field type
+        let uris = re_chunk::util::arrays_to_list_array(
+            rec_id_field.data_type().clone(),
+            &recording_id_arrays,
+        )
+        .unwrap();
+
+        chunk.add_component(ComponentDescriptor::new(RecordingUri::name()), uris)?;
 
         if tx
             .send(LogMsg::ArrowMsg(store_id.clone(), chunk.to_arrow_msg()?))

crates/store/re_types/definitions/rerun/blueprint/archetypes/container_blueprint.fbs

@@ -4,8 +4,7 @@ namespace rerun.blueprint.archetypes;
 
 /// The description of a container.
 table ContainerBlueprint (
-    "attr.rerun.scope": "blueprint",
-    "attr.rust.override_crate": "re_types_blueprint"
+    "attr.rerun.scope": "blueprint"
 ) {
     // --- Required ---
 

crates/store/re_types/definitions/rerun/blueprint/archetypes/panel_blueprint.fbs

@@ -5,12 +5,13 @@ namespace rerun.blueprint.archetypes;
 /// Shared state for the 3 collapsible panels.
 table PanelBlueprint (
     "attr.rerun.scope": "blueprint",
-    "attr.rust.derive": "Default",
-    "attr.rust.override_crate": "re_types_blueprint"
+    "attr.rust.derive": "Default"
 ) {
     // --- Required ---
 
     // --- Optional ---
+
+    /// Current state of the panels.
     state: rerun.blueprint.components.PanelState ("attr.rerun.component_optional", nullable, order: 1000);
 
     // TODO(jleibs): Add a float to track how expanded the panel is.

crates/store/re_types/definitions/rerun/blueprint/archetypes/viewport_blueprint.fbs

@@ -5,8 +5,7 @@ namespace rerun.blueprint.archetypes;
 /// The top-level description of the viewport.
 table ViewportBlueprint (
     "attr.rerun.scope": "blueprint",
-    "attr.rust.derive": "Default",
-    "attr.rust.override_crate": "re_types_blueprint"
+    "attr.rust.derive": "Default"
 ) {
     // --- Required ---
 

crates/store/re_types/definitions/rerun/blueprint/components/auto_layout.fbs

@@ -6,10 +6,9 @@ namespace rerun.blueprint.components;
 /// Whether the viewport layout is determined automatically.
 struct AutoLayout (
   "attr.arrow.transparent",
-  "attr.rerun.scope": "blueprint",
   "attr.python.aliases": "bool",
+  "attr.rerun.scope": "blueprint",
   "attr.rust.derive": "Copy",
-  "attr.rust.override_crate": "re_types_blueprint",
   "attr.rust.repr": "transparent",
   "attr.rust.tuple_struct"
 ) {

crates/store/re_types/definitions/rerun/blueprint/components/auto_views.fbs

@@ -7,7 +7,6 @@ struct AutoViews (
   "attr.rerun.scope": "blueprint",
   "attr.python.aliases": "bool",
   "attr.rust.derive": "Copy, Default, PartialEq, Eq, PartialOrd, Ord",
-  "attr.rust.override_crate": "re_types_blueprint",
   "attr.rust.repr": "transparent",
   "attr.rust.tuple_struct"
 ) {

crates/store/re_types/definitions/rerun/blueprint/components/container_kind.fbs

@@ -3,8 +3,7 @@ namespace rerun.blueprint.components;
 
 /// The kind of a blueprint container (tabs, grid, …).
 enum ContainerKind: ubyte (
-    "attr.rerun.scope": "blueprint",
-    "attr.rust.override_crate": "re_types_blueprint"
+    "attr.rerun.scope": "blueprint"
 ) {
     /// Invalid value. Won't show up in generated types.
     Invalid = 0,

crates/store/re_types/definitions/rerun/blueprint/components/grid_columns.fbs

@@ -5,8 +5,7 @@ namespace rerun.blueprint.components;
 table GridColumns (
     "attr.rerun.scope": "blueprint",
     "attr.python.aliases": "int",
-    "attr.rust.derive": "Default, PartialEq, Eq, PartialOrd, Ord",
-    "attr.rust.override_crate": "re_types_blueprint"
+    "attr.rust.derive": "Default, PartialEq, Eq, PartialOrd, Ord"
 ) {
     /// The number of columns.
     columns: rerun.datatypes.UInt32 (order: 100);