Merge branch 'main' into fork/tfoldi/feat/mapbox-view

crates/store/re_remote_store_types/proto/rerun/v0/common.proto

@@ -161,3 +161,15 @@ enum SparseFillStrategy {
     NONE = 0;
     LATEST_AT_GLOBAL = 1;
 }
+
+// Error codes for application level errors
+enum ErrorCode {
+    // unused
+    _UNUSED = 0;
+
+    // object store access error
+    OBJECT_STORE_ERROR = 1;
+
+    // metadata database access error
+    METADATA_DB_ERROR = 2;
+}

crates/store/re_remote_store_types/proto/rerun/v0/remote_store.proto

@@ -34,6 +34,16 @@ message RegisterRecordingsResponse {
     repeated RecordingMetadata metadata = 2;
 }
 
+// Server can include details about the error as part of gRPC error (Status)
+message RegistrationError {
+    // error code
+    ErrorCode code = 1;
+    // url of the recording that failed to register
+    string url = 2;
+    // human readable details about the error
+    string message = 3;
+}
+
 // ---------------- GetRecordingMetadata  -----------------
 
 message GetRecordingMetadataRequest {

crates/store/re_remote_store_types/src/lib.rs

@@ -312,6 +312,18 @@ pub mod v0 {
             }
         }
     }
+
+    // ------- Application level errors -------
+    impl std::error::Error for RegistrationError {}
+
+    impl std::fmt::Display for RegistrationError {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            f.write_fmt(format_args!(
+                "Failed to register recording: {}, error code: {}, error message: {}",
+                self.url, self.code, self.message
+            ))
+        }
+    }
 }
 
 #[cfg(test)]

crates/store/re_remote_store_types/src/v0/rerun.remote_store.v0.rs

@@ -209,6 +209,39 @@ impl SparseFillStrategy {
         }
     }
 }
+/// Error codes for application level errors
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum ErrorCode {
+    /// unused
+    Unused = 0,
+    /// object store access error
+    ObjectStoreError = 1,
+    /// metadata database access error
+    MetadataDbError = 2,
+}
+impl ErrorCode {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::Unused => "_UNUSED",
+            Self::ObjectStoreError => "OBJECT_STORE_ERROR",
+            Self::MetadataDbError => "METADATA_DB_ERROR",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "_UNUSED" => Some(Self::Unused),
+            "OBJECT_STORE_ERROR" => Some(Self::ObjectStoreError),
+            "METADATA_DB_ERROR" => Some(Self::MetadataDbError),
+            _ => None,
+        }
+    }
+}
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct RegisterRecordingsRequest {
     #[prost(string, tag = "1")]
@@ -235,6 +268,19 @@ pub struct RegisterRecordingsResponse {
     #[prost(message, repeated, tag = "2")]
     pub metadata: ::prost::alloc::vec::Vec<RecordingMetadata>,
 }
+/// Server can include details about the error as part of gRPC error (Status)
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct RegistrationError {
+    /// error code
+    #[prost(enumeration = "ErrorCode", tag = "1")]
+    pub code: i32,
+    /// url of the recording that failed to register
+    #[prost(string, tag = "2")]
+    pub url: ::prost::alloc::string::String,
+    /// human readable details about the error
+    #[prost(string, tag = "3")]
+    pub message: ::prost::alloc::string::String,
+}
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct GetRecordingMetadataRequest {
     #[prost(message, optional, tag = "1")]

docs/content/concepts/app-model.md

@@ -0,0 +1,189 @@
+---
+title: Application model
+order: 0
+---
+
+The Rerun distribution comes with numerous moving pieces:
+* The **SDKs** (Python, Rust & C++), for logging data and querying it back. These are libraries running directly in the end user's process.
+* The **Native Viewer**: the Rerun GUI application for native platforms (Linux, macOS, Windows).
+* The **TCP server**, which receives data from the **SDKs** and forwards it to the **Native Viewer** and/or **WebSocket Server**. The communication is unidirectional: clients push data into the TCP connection, never the other way around.
+* The **Web Viewer**, which packs the **Native Viewer** into a WASM application that can run on the Web and its derivatives (notebooks, etc).
+* The **Web/HTTP Server**, for serving the web page that hosts the **Web Viewer**.
+* The **WebSocket server**, for serving data to the **Web Viewer**. The communication is unidirectional: the server pushes data to the **Web Viewer**, never the other way around.
+* The **CLI**, which allows you to control all the pieces above as well as manipulate RRD files.
+
+The **Native Viewer** always includes:
+  * A **Chunk Store**: an in-memory database that stores the logged data.
+  * A **Renderer**: a 3D engine that renders the contents of the **Chunk Store**.
+
+
+## What runs where?
+
+This is a lot to take in at first, but as we'll see these different pieces are generally deployed in just a few unique configurations for most common cases.
+
+The first thing to understand is what process do each of these things run in.
+
+The **CLI**, **Native Viewer**, **TCP server**, **Web/HTTP Server** and **WebSocket Server** are all part of the same binary: `rerun`.
+Some of them can be enabled or disabled on demand using the appropriate flags but, no matter what, all these pieces are part of the same binary and execute in the same process.
+Keep in mind that even the **Native Viewer** can be disabled (headless mode).
+
+The **SDKs** are vanilla software libraries and therefore always executes in the same context as the end-user's code.
+
+Finally, the **Web Viewer** is a WASM application and therefore has its own dedicated `.wasm` artifact, and always runs in isolation in the end-user's web browser.
+
+The best way to make sense of it all it to look at some of the most common scenarios when:
+* Logging and visualizing data on native.
+* Logging data on native and visualizing it on the web.
+
+
+## Logging and visualizing data on native
+
+There are two common sub-scenarios when working natively:
+* Data is being logged and visualized at the same time (synchronous workflow).
+* Data is being logged first to some persistent storage, and visualized at a later time (asynchronous workflow).
+
+
+### Synchronous workflow
+
+This is the most common kind of Rerun deployment, and also the simplest: one or more **SDKs**, embedded into the user's process, are logging data directly to a **TCP Server**, which in turns feeds the **Native Viewer**.
+Both the **Native Viewer** and the **TCP Server** are running in the same `rerun` process.
+
+Logging script:
+
+snippet: concepts/app-model/native-sync
+
+Deployment:
+<!-- TODO(#7768): talk about rr.spawn(serve=True) once that's thing -->
+```sh
+# Start the Rerun Native Viewer in the background.
+#
+# This will also start the TCP server on its default port (9876, use `--port`
+# to pick another one).
+#
+# We could also have just used `spawn()` instead of `connect()` in the logging
+# script above, and # we wouldn't have had to start the Native Viewer manually.
+# `spawn()` does exactly this: it fork-execs a Native Viewer in the background
+# using the first `rerun` # binary available # on your $PATH.
+$ rerun &
+
+# Start logging data. It will be pushed to the Native Viewer through the TCP link.
+$ ./logging_script
+```
+
+
+Dataflow:
+
+<picture>
+  <img src="https://static.rerun.io/rerun_native_sync/df05102a1dd04839ffec8442e5e9ffe65e9649db/full.png" alt="">
+  <source media="(max-width: 480px)" srcset="https://static.rerun.io/rerun_native_sync/df05102a1dd04839ffec8442e5e9ffe65e9649db/480w.png">
+  <source media="(max-width: 768px)" srcset="https://static.rerun.io/rerun_native_sync/df05102a1dd04839ffec8442e5e9ffe65e9649db/768w.png">
+  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/rerun_native_sync/df05102a1dd04839ffec8442e5e9ffe65e9649db/1024w.png">
+</picture>
+
+
+Reference:
+* [SDK operating modes: `connect`](../reference/sdk/operating-modes.md#connect)
+* [🐍 Python `connect`](https://ref.rerun.io/docs/python/0.19.0/common/initialization_functions/#rerun.connect)
+* [🦀 Rust `connect`](https://docs.rs/rerun/latest/rerun/struct.RecordingStreamBuilder.html#method.connect)
+* [🌊 C++ `connect`](https://ref.rerun.io/docs/cpp/stable/classrerun_1_1RecordingStream.html#aef3377ffaa2441b906d2bac94dd8fc64)
+
+### Asynchronous workflow
+
+The asynchronous native workflow is similarly simple: one or more **SDKs**, embedded into the user's process, are logging data directly to one or more files.
+The user will then manually start the **Native Viewer** at some later point, in order to visualize these files.
+
+Note: the `rerun` process still embeds both a **Native Viewer** and a **TCP Server**. For each **Native Viewer**, there is **always** an accompanying **TCP Server**, no exception.
+
+Logging script:
+
+snippet: concepts/app-model/native-async
+
+Deployment:
+```sh
+# Log the data into one or more files.
+$ ./logging_script
+
+# Start the Rerun Native Viewer and feed it the RRD file directly.
+#
+# This will also start the TCP server on its default port (9876, use `--port`
+# to pick another one). Although it is not used yet, some client might want
+# to connect in the future.
+$ rerun /tmp/my_recording.rrd
+```
+
+Dataflow:
+
+<picture>
+  <img src="https://static.rerun.io/rerun_native_async/272c9ba7e7afe0ee5491ff1aabc76965588c513f/full.png" alt="">
+  <source media="(max-width: 480px)" srcset="https://static.rerun.io/rerun_native_async/272c9ba7e7afe0ee5491ff1aabc76965588c513f/480w.png">
+  <source media="(max-width: 768px)" srcset="https://static.rerun.io/rerun_native_async/272c9ba7e7afe0ee5491ff1aabc76965588c513f/768w.png">
+  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/rerun_native_async/272c9ba7e7afe0ee5491ff1aabc76965588c513f/1024w.png">
+  <source media="(max-width: 1200px)" srcset="https://static.rerun.io/rerun_native_async/272c9ba7e7afe0ee5491ff1aabc76965588c513f/1200w.png">
+</picture>
+
+
+Reference:
+* [SDK operating modes: `save`](../reference/sdk/operating-modes.md#save)
+* [🐍 Python `save`](https://ref.rerun.io/docs/python/0.19.0/common/initialization_functions/#rerun.save)
+* [🦀 Rust `save`](https://docs.rs/rerun/latest/rerun/struct.RecordingStreamBuilder.html#method.save)
+* [🌊 C++ `save`](https://ref.rerun.io/docs/cpp/stable/classrerun_1_1RecordingStream.html#a555a7940a076c93d951de5b139d14918)
+
+## Logging data on native and visualizing it on the web.
+
+TODO(cmc): incoming.
+
+
+## FAQ
+
+### How can I use multiple **Native Viewers** at the same (i.e. multiple windows)?
+
+Every **Native Viewer** comes with a corresponding **TCP Server** -- always. You cannot start a **Native Viewer** without starting a **TCP server**.
+
+The only way to have more than one Rerun window is to have more than one **TCP server**, by means of the `--port` flag.
+
+E.g.:
+```sh
+# starts a new viewer, listening for TCP connections on :9876
+rerun &
+
+# does nothing, there's already a viewer session running at that address
+rerun &
+
+# does nothing, there's already a viewer session running at that address
+rerun --port 9876 &
+
+# logs the image file to the existing viewer running on :9876
+rerun image.jpg
+
+# logs the image file to the existing viewer running on :9876
+rerun --port 9876 image.jpg
+
+# starts a new viewer, listening for TCP connections on :6789, and logs the image data to it
+rerun --port 6789 image.jpg
+
+# does nothing, there's already a viewer session running at that address
+rerun --port 6789 &
+
+# logs the image file to the existing viewer running on :6789
+rerun --port 6789 image.jpg &
+```
+
+
+### What happens when I use `rr.spawn()` from my SDK of choice?
+
+TODO(cmc): incoming.
+
+
+### What happens when I use `rr.serve()` from my SDK of choice?
+
+TODO(cmc): incoming.
+
+
+### What happens when I use `rerun --serve`?
+
+TODO(cmc): incoming.
+
+
+### Can the **Native Viewer** pull data from a **WebSocket Server**, like the **Web Viewer** does?
+
+TODO(cmc): incoming.

docs/content/concepts/blueprint.md

@@ -1,5 +1,5 @@
 ---
-title: Blueprint
+title: Blueprints
 order: 600
 ---
 

docs/content/howto/embed-rerun-viewer.md

@@ -1,5 +1,5 @@
 ---
 title: Embed a Rerun Viewer
 hidden: true
-redirect: howto/embed-web
+redirect: howto/integrations/embed-web
 ---

docs/content/howto/extend.md

@@ -1,6 +1,6 @@
 ---
 title: Extend Rerun
-order: 1000
+hidden: true
 ---
 
 There are currently two major ways of extending Rerun. You can use Rerun with [your own custom data](logging/custom-data.md), or [extend the Rerun Viewer](visualization/extend-ui.md) (currently Rust only).

docs/content/howto/notebook.md

@@ -1,5 +1,5 @@
 ---
 title: Embed Rerun in notebooks
 hidden: true
-redirect: howto/integrations/embed-notebook
+redirect: howto/integrations/embed-notebooks
 ---

docs/content/howto/using-native-loggers.md

@@ -1,19 +1,5 @@
 ---
 title: Integrate Rerun with native loggers
-order: 700
-description: How to use the Rerun SDK as a native logger for the host language
+hidden: true
+redirect: howto/integrations/integrate-host-loggers
 ---
-
-The Rerun SDK implements the native logging interfaces of its supported host languages, allowing you to transparently stream text logs logged with the native APIs into the Rerun Viewer.
-
-The details of how to achieve that vary language by language, see the snippets below.
-
-snippet: archetypes/text_log_integration
-
-<picture>
-  <img src="https://static.rerun.io/text_log_integration/9737d0c986325802a9885499d6fcc773b1736488/full.png" alt="">
-  <source media="(max-width: 480px)" srcset="https://static.rerun.io/text_log_integration/9737d0c986325802a9885499d6fcc773b1736488/480w.png">
-  <source media="(max-width: 768px)" srcset="https://static.rerun.io/text_log_integration/9737d0c986325802a9885499d6fcc773b1736488/768w.png">
-  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/text_log_integration/9737d0c986325802a9885499d6fcc773b1736488/1024w.png">
-  <source media="(max-width: 1200px)" srcset="https://static.rerun.io/text_log_integration/9737d0c986325802a9885499d6fcc773b1736488/1200w.png">
-</picture>

docs/content/howto/visualization/configure-viewer-through-code.md

@@ -1,5 +1,5 @@
 ---
-title: Configure the Viewer through code
+title: Configure the Viewer through code (Blueprints)
 order: 100
 ---