diff --git a/docs/en/1-Experiments/Notebooks/ReRun_EN.ipynb b/docs/en/1-Experiments/Notebooks/ReRun_EN.ipynb index 0ae488224..ee980d4ae 100644 --- a/docs/en/1-Experiments/Notebooks/ReRun_EN.ipynb +++ b/docs/en/1-Experiments/Notebooks/ReRun_EN.ipynb @@ -29,12 +29,8 @@ "\n", "- Stream multimodal data from your code by logging it with the Rerun SDK\n", "- Visualize and interact with live or recorded streams, whether local or remote\n", - "- Interactively build layouts and customize visualizations\n", - "- Extend Rerun when you need to\n", "\n", - "![](mito-fullscreen.png)\n", - "\n", - "![](mito.png)\n", + "![](rerun.png)\n", "\n", "## Installation:\n", "\n", @@ -54,14 +50,28 @@ "! pip install -U rerun-sdk" ] }, + { + "cell_type": "markdown", + "id": "b421be69-2741-4d5c-a3fa-f60f8d9187aa", + "metadata": { + "tags": [] + }, + "source": [ + "## Getting Started:\n", + "\n", + "The simplest example is to open ReRun using the `spawn()` method:\n", + "\n", + "![Empty Viewer](./img/rerun-empty.png)" + ] + }, { "cell_type": "code", - "execution_count": 11, - "id": "044bc84c", + "execution_count": 1, + "id": "a846e8c0", "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ "\n", @@ -75,56 +85,80 @@ " 9: eframe::run_native\n", "\n", "Troubleshooting Rerun: https://www.rerun.io/docs/getting-started/troubleshooting\n", - "Report bugs: https://github.com/rerun-io/rerun/issues\n", - "\n", - " Welcome to Rerun!\n", - "\n", - " This open source library collects anonymous usage data to\n", - " help the Rerun team improve the library.\n", - "\n", - " Summary:\n", - " - We only collect high level events about the features used within the Rerun Viewer.\n", - " - The actual data you log to Rerun, such as point clouds, images, or text logs,\n", - " will never be collected.\n", - " - We don't log IP addresses.\n", - " - We don't log your user name, file paths, or any personal identifiable data.\n", - " - Usage data we do collect will be sent to and stored on servers within the EU.\n", - "\n", - " For more details and instructions on how to opt out, run the command:\n", - "\n", - " rerun analytics details\n", - "\n", - " As this is this your first session, _no_ usage data has been sent yet,\n", - " giving you an opportunity to opt-out first if you wish.\n", - "\n", - " Happy Rerunning!\n", - "\n" + "Report bugs: https://github.com/rerun-io/rerun/issues\n" ] } ], "source": [ - "! rerun" + "import rerun as rr\n", + "\n", + "rr.init()\n", + "rr.spawn()" ] }, { "cell_type": "markdown", - "id": "b421be69-2741-4d5c-a3fa-f60f8d9187aa", - "metadata": { - "tags": [] - }, + "id": "b84e8b9d", + "metadata": {}, "source": [ - "## Getting Started:\n", + "But we can also view something simple in 3 space by generating some points and plotting them." + ] + }, + { + "cell_type": "markdown", + "id": "56bb4e64", + "metadata": {}, + "source": [ + "![Cube](./img/rerun-cube.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7127bbc", + "metadata": {}, + "outputs": [], + "source": [ + "import rerun as rr\n", + "import numpy as np\n", + "\n", + "rr.init(\"rerun_example_my_data\", spawn=True)\n", "\n", - "Begin your ReRun journey by running the following code snippet:" + "SIZE = 10\n", + "\n", + "pos_grid = np.meshgrid(*[np.linspace(-10, 10, SIZE)]*3)\n", + "positions = np.vstack([d.reshape(-1) for d in pos_grid]).T\n", + "\n", + "col_grid = np.meshgrid(*[np.linspace(0, 255, SIZE)]*3)\n", + "colors = np.vstack([c.reshape(-1) for c in col_grid]).astype(np.uint8).T\n", + "\n", + "rr.log(\n", + " \"my_points\",\n", + " rr.Points3D(positions, colors=colors, radii=0.5)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "759d2169", + "metadata": {}, + "source": [ + "But that's not that impressive. ReRun shines when we view multi-model time-series data:" ] }, { "cell_type": "code", - "execution_count": 12, - "id": "a2fa0eae-fceb-49a9-9c0f-4daf35d200f7", - "metadata": { - "tags": [] - }, + "execution_count": null, + "id": "59cfd2b2", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e9bd9a87", + "metadata": {}, "outputs": [ { "name": "stderr", @@ -141,76 +175,371 @@ " 9: eframe::run_native\n", "\n", "Troubleshooting Rerun: https://www.rerun.io/docs/getting-started/troubleshooting\n", - "Report bugs: https://github.com/rerun-io/rerun/issues\n" + "Report bugs: https://github.com/rerun-io/rerun/issues\n", + "[2023-12-11T20:56:26Z WARN re_sdk_comms::buffered_client] Failed to send message after 3 attempts: Failed to send to Rerun server at 127.0.0.1:9876: Broken pipe (os error 32)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2023-12-11T20:56:29Z WARN re_sdk_comms::buffered_client] Dropping messages because tcp client has timed out.\n", + "[2023-12-11T20:56:29Z WARN re_sdk_comms::buffered_client] Dropping messages because tcp client has timed out.\n" ] } ], "source": [ - "import rerun as rr # NOTE: `rerun`, not `rerun-sdk`!\n", + "from __future__ import annotations\n", + "\n", + "import argparse\n", + "from math import tau\n", + "\n", "import numpy as np\n", + "import rerun as rr # pip install rerun-sdk\n", + "from rerun.utilities import bounce_lerp, build_color_spiral\n", "\n", - "rr.init(\"rerun_example_my_data\", spawn=True)\n", + "DESCRIPTION = \"\"\"\n", + "# DNA\n", + "This is a minimal example that logs synthetic 3D data in the shape of a double helix. The underlying data is generated\n", + "using numpy and visualized using Rerun.\n", "\n", - "positions = np.zeros((10, 3))\n", - "positions[:,0] = np.linspace(-10,10,10)\n", + "## How it was made\n", + "The full source code for this example is available\n", + "[on GitHub](https://github.com/rerun-io/rerun/blob/latest/examples/python/dna/main.py).\n", "\n", - "colors = np.zeros((10,3), dtype=np.uint8)\n", - "colors[:,0] = np.linspace(0,255,10)\n", + "### Colored 3D points\n", + "The colored 3D points were added to the scene by logging the\n", + "[rr.Points3D archetype](https://www.rerun.io/docs/reference/types/archetypes/points3d) to the\n", + "[helix/structure/left](recording://helix/structure/left) and [helix/structure/right](recording://helix/structure/right)\n", + "entities.\n", "\n", - "rr.log(\n", - " \"my_points\",\n", - " rr.Points3D(positions, colors=colors, radii=0.5)\n", - ")" + "### 3D line strips\n", + "The 3D line strips connecting the 3D point pairs are logged as an\n", + "[rr.LineStrips3D archetype](https://www.rerun.io/docs/reference/types/archetypes/line_strips3d) to the\n", + "[helix/structure/scaffolding entity](recording://helix/structure/scaffolding).\n", + "\n", + "### Rotation\n", + "The whole structure is rotated over time by logging a\n", + "[rr.Transform3D archetype](https://www.rerun.io/docs/reference/types/archetypes/transform3d) to the\n", + "[helix/structure entity](recording://helix/structure.Transform3D) that changes over time. This transform determines the rotation of\n", + "the [structure entity](recording://helix/structure) relative to the [helix](recording://helix) entity. Since all other\n", + "entities are children of [helix/structure](recording://helix/structure) they will also rotate based on this transform.\n", + "\n", + "You can visualize this rotation by selecting the two entities on the left-hand side and activating `Show transform` in\n", + "the Blueprint settings on the right-hand side. You will see one static frame (i.e., the frame of\n", + "[helix](recording://helix)) and the rotating frame (i.e., the frame of [structure](recording://helix/structure)).\n", + "\"\"\".strip()\n", + "\n", + "rr.spawn()\n", + "\n", + "rr.log(\"description\", rr.TextDocument(DESCRIPTION, media_type=rr.MediaType.MARKDOWN), timeless=True)\n", + "\n", + "rr.set_time_seconds(\"stable_time\", 0)\n", + "\n", + "NUM_POINTS = 100\n", + "\n", + "# points and colors are both np.array((NUM_POINTS, 3))\n", + "points1, colors1 = build_color_spiral(NUM_POINTS)\n", + "points2, colors2 = build_color_spiral(NUM_POINTS, angular_offset=tau * 0.5)\n", + "rr.log(\"helix/structure/left\", rr.Points3D(points1, colors=colors1, radii=0.08))\n", + "rr.log(\"helix/structure/right\", rr.Points3D(points2, colors=colors2, radii=0.08))\n", + "\n", + "rr.log(\"helix/structure/scaffolding\", rr.LineStrips3D(np.stack((points1, points2), axis=1), colors=[128, 128, 128]))\n", + "\n", + "time_offsets = np.random.rand(NUM_POINTS)\n", + "for i in range(400):\n", + " time = i * 0.01\n", + " rr.set_time_seconds(\"stable_time\", time)\n", + "\n", + " times = np.repeat(time, NUM_POINTS) + time_offsets\n", + " beads = [bounce_lerp(points1[n], points2[n], times[n]) for n in range(NUM_POINTS)]\n", + " colors = [[int(bounce_lerp(80, 230, times[n] * 2))] for n in range(NUM_POINTS)]\n", + " rr.log(\n", + " \"helix/structure/scaffolding/beads\", rr.Points3D(beads, radii=0.06, colors=np.repeat(colors, 3, axis=-1))\n", + " )\n", + "\n", + " rr.log(\n", + " \"helix/structure\",\n", + " rr.Transform3D(rotation=rr.RotationAxisAngle(axis=[0, 0, 1], radians=time / 4.0 * tau)),\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "0cd849b8", + "metadata": {}, + "source": [ + "![DNA](./img/rerun-dna.png)" + ] + }, + { + "cell_type": "markdown", + "id": "b51051ab", + "metadata": {}, + "source": [ + "![DNA](./img/rerun-dna.png)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "a7127bbc", + "execution_count": 4, + "id": "de21e1f2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "thread 'ThreadId(1)' panicked at 'Failed to initialize any backend! Wayland status: XdgRuntimeDirNotSet X11 status: XOpenDisplayFailed'\n", + "winit-0.28.7/src/platform_impl/linux/mod.rs:757\n", + "stack backtrace:\n", + " 6: core::panicking::panic_fmt\n", + " at core/src/panicking.rs:67:14\n", + " 7: eframe::native::run::create_event_loop\n", + " 8: eframe::native::run::with_event_loop\n", + " 9: eframe::run_native\n", + "\n", + "Troubleshooting Rerun: https://www.rerun.io/docs/getting-started/troubleshooting\n", + "Report bugs: https://github.com/rerun-io/rerun/issues\n", + "[2023-12-11T21:09:39Z WARN re_sdk_comms::buffered_client] Failed to send message after 3 attempts: Failed to send to Rerun server at 127.0.0.1:9876: Broken pipe (os error 32)\n" + ] + } + ], "source": [ - "import rerun as rr\n", + "#!/usr/bin/env python3\n", + "\"\"\"\n", + "Demonstrates how to log simple plots with the Rerun SDK.\n", + "\n", + "\"\"\"\n", + "from __future__ import annotations\n", + "\n", + "import argparse\n", + "import random\n", + "from math import cos, sin, tau\n", + "\n", "import numpy as np\n", + "import rerun as rr # pip install rerun-sdk\n", "\n", - "rr.init(\"rerun_example_my_data\", spawn=True)\n", + "DESCRIPTION = \"\"\"\n", + "# Plots\n", + "This example shows various plot types that you can create using Rerun. Common usecases for such plots would be logging\n", + "losses or metrics over time, histograms, or general function plots.\n", "\n", - "SIZE = 10\n", + "## How it was made\n", + "The full source code for this example is available [on GitHub](https://github.com/rerun-io/rerun/blob/latest/examples/python/plots/main.py).\n", "\n", - "pos_grid = np.meshgrid(*[np.linspace(-10, 10, SIZE)]*3)\n", - "positions = np.vstack([d.reshape(-1) for d in pos_grid]).T\n", + "### Bar charts\n", + "The [bar chart](recording://bar_chart) is created by logging the [rr.BarChart archetype](https://www.rerun.io/docs/reference/types/archetypes/bar_chart).\n", "\n", - "col_grid = np.meshgrid(*[np.linspace(0, 255, SIZE)]*3)\n", - "colors = np.vstack([c.reshape(-1) for c in col_grid]).astype(np.uint8).T\n", + "### Time series\n", + "All other plots are created using the\n", + "[rr.TimeSeriesScalar archetype](https://www.rerun.io/docs/reference/types/archetypes/bar_chart)\n", + "with different settings. Each plot is created by logging scalars at different time steps (i.e., the x-axis).\n", "\n", - "rr.log(\n", - " \"my_points\",\n", - " rr.Points3D(positions, colors=colors, radii=0.5)\n", - ")" + "For the [parabola](recording://curves/parabola) the radius and color is changed over time.\n", + "\n", + "[sin](recording://trig/sin) and [cos](recording://trig/cos) are logged with the same parent entity (i.e.,\n", + "`trig/{cos,sin}`) which will put them in the same view by default.\n", + "\n", + "For the [classification samples](recording://classification/samples) `rr.TimeSeriesScalar(..., scatter=True)` is used to\n", + "create separate points that do not connect over time. Note, that in the other plots the logged scalars are connected\n", + "over time by lines.\n", + "\"\"\".strip()\n", + "\n", + "rr.spawn()\n", + "\n", + "def clamp(n, smallest, largest): # type: ignore[no-untyped-def]\n", + " return max(smallest, min(n, largest))\n", + "\n", + "\n", + "def log_bar_chart() -> None:\n", + " rr.set_time_sequence(\"frame_nr\", 0)\n", + " # Log a gauss bell as a bar chart\n", + " mean = 0\n", + " std = 1\n", + " variance = np.square(std)\n", + " x = np.arange(-5, 5, 0.1)\n", + " y = np.exp(-np.square(x - mean) / 2 * variance) / (np.sqrt(2 * np.pi * variance))\n", + " rr.log(\"bar_chart\", rr.BarChart(y))\n", + "\n", + "\n", + "def log_parabola() -> None:\n", + " # Log a parabola as a time series\n", + " for t in range(0, 1000, 10):\n", + " rr.set_time_sequence(\"frame_nr\", t)\n", + "\n", + " f_of_t = (t * 0.01 - 5) ** 3 + 1\n", + " radius = clamp(abs(f_of_t) * 0.1, 0.5, 10.0)\n", + " color = [255, 255, 0]\n", + " if f_of_t < -10.0:\n", + " color = [255, 0, 0]\n", + " elif f_of_t > 10.0:\n", + " color = [0, 255, 0]\n", + "\n", + " rr.log(\n", + " \"curves/parabola\",\n", + " rr.TimeSeriesScalar(\n", + " f_of_t,\n", + " label=\"f(t) = (0.01t - 3)³ + 1\",\n", + " radius=radius,\n", + " color=color,\n", + " ),\n", + " )\n", + "\n", + "\n", + "def log_trig() -> None:\n", + " # Log a time series\n", + " for t in range(0, int(tau * 2 * 100.0)):\n", + " rr.set_time_sequence(\"frame_nr\", t)\n", + "\n", + " sin_of_t = sin(float(t) / 100.0)\n", + " rr.log(\"trig/sin\", rr.TimeSeriesScalar(sin_of_t, label=\"sin(0.01t)\", color=[255, 0, 0]))\n", + "\n", + " cos_of_t = cos(float(t) / 100.0)\n", + " rr.log(\"trig/cos\", rr.TimeSeriesScalar(cos_of_t, label=\"cos(0.01t)\", color=[0, 255, 0]))\n", + "\n", + "\n", + "def log_classification() -> None:\n", + " # Log a time series\n", + " for t in range(0, 1000, 2):\n", + " rr.set_time_sequence(\"frame_nr\", t)\n", + "\n", + " f_of_t = (2 * 0.01 * t) + 2\n", + " color = [255, 255, 0]\n", + " rr.log(\"classification/line\", rr.TimeSeriesScalar(f_of_t, color=color, radius=3.0))\n", + "\n", + " g_of_t = f_of_t + random.uniform(-5.0, 5.0)\n", + " if g_of_t < f_of_t - 1.5:\n", + " color = [255, 0, 0]\n", + " elif g_of_t > f_of_t + 1.5:\n", + " color = [0, 255, 0]\n", + " else:\n", + " color = [255, 255, 255]\n", + " radius = abs(g_of_t - f_of_t)\n", + " rr.log(\"classification/samples\", rr.TimeSeriesScalar(g_of_t, color=color, scattered=True, radius=radius))\n", + "\n", + "\n", + "\n", + "rr.log(\"description\", rr.TextDocument(DESCRIPTION, media_type=rr.MediaType.MARKDOWN), timeless=True)\n", + "log_bar_chart()\n", + "log_parabola()\n", + "log_trig()\n", + "log_classification()" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "759d2169", + "cell_type": "markdown", + "id": "07f8187c", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "![DNA](./img/rerun-multi-1.png)" + ] }, { "cell_type": "markdown", - "id": "d9bd7e11", + "id": "290dec98", "metadata": {}, "source": [ - "See the official ReRun [Python Quickstart](https://www.rerun.io/docs/getting-started/python) for more information about ReRun and what it can do for you." + "The next example is more serious and will require us to run the code from a script." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "11fccc2c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting segment_anything\n", + " Downloading segment_anything-1.0-py3-none-any.whl (36 kB)\n", + "Collecting opencv_python\n", + " Downloading opencv_python-4.8.1.78-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\n", + "Requirement already satisfied: torch in /home/codespace/.local/lib/python3.10/site-packages (2.1.1)\n", + "Collecting torchvision\n", + " Downloading torchvision-0.16.1-cp310-cp310-manylinux1_x86_64.whl.metadata (6.6 kB)\n", + "Collecting tqdm\n", + " Downloading tqdm-4.66.1-py3-none-any.whl.metadata (57 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.6/57.6 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy>=1.21.2 in /home/codespace/.local/lib/python3.10/site-packages (from opencv_python) (1.26.2)\n", + "Requirement already satisfied: filelock in /home/codespace/.local/lib/python3.10/site-packages (from torch) (3.13.1)\n", + "Requirement already satisfied: typing-extensions in /home/codespace/.local/lib/python3.10/site-packages (from torch) (4.8.0)\n", + "Requirement already satisfied: sympy in /home/codespace/.local/lib/python3.10/site-packages (from torch) (1.12)\n", + "Requirement already satisfied: networkx in /home/codespace/.local/lib/python3.10/site-packages (from torch) (3.2.1)\n", + "Requirement already satisfied: jinja2 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (3.1.2)\n", + "Requirement already satisfied: fsspec in /home/codespace/.local/lib/python3.10/site-packages (from torch) (2023.10.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (8.9.2.26)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (12.1.3.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (11.0.2.54)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (10.3.2.106)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (11.4.5.107)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (12.1.0.106)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.18.1 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (2.18.1)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: triton==2.1.0 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (2.1.0)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/codespace/.local/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.3.101)\n", + "Requirement already satisfied: requests in /home/codespace/.local/lib/python3.10/site-packages (from torchvision) (2.31.0)\n", + "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /home/codespace/.local/lib/python3.10/site-packages (from torchvision) (10.1.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /home/codespace/.local/lib/python3.10/site-packages (from jinja2->torch) (2.1.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/codespace/.local/lib/python3.10/site-packages (from requests->torchvision) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/codespace/.local/lib/python3.10/site-packages (from requests->torchvision) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from requests->torchvision) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/codespace/.local/lib/python3.10/site-packages (from requests->torchvision) (2023.7.22)\n", + "Requirement already satisfied: mpmath>=0.19 in /home/codespace/.local/lib/python3.10/site-packages (from sympy->torch) (1.3.0)\n", + "Downloading opencv_python-4.8.1.78-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (61.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.7/61.7 MB\u001b[0m \u001b[31m21.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hDownloading torchvision-0.16.1-cp310-cp310-manylinux1_x86_64.whl (6.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m56.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hDownloading tqdm-4.66.1-py3-none-any.whl (78 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.3/78.3 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: segment_anything, tqdm, opencv_python, torchvision\n", + "Successfully installed opencv_python-4.8.1.78 segment_anything-1.0 torchvision-0.16.1 tqdm-4.66.1\n" + ] + } + ], + "source": [ + "! pip install segment_anything opencv_python torch torchvision tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "0c62304c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Traceback (most recent call last):\n", + " File \"/workspaces/aaw/docs/en/1-Experiments/Notebooks/scripts/rerun-segment_anything_model.py\", line 28, in \n", + " import cv2\n", + " File \"/usr/local/python/3.10.13/lib/python3.10/site-packages/cv2/__init__.py\", line 181, in \n", + " bootstrap()\n", + " File \"/usr/local/python/3.10.13/lib/python3.10/site-packages/cv2/__init__.py\", line 153, in bootstrap\n", + " native_module = importlib.import_module(\"cv2\")\n", + " File \"/usr/local/python/3.10.13/lib/python3.10/importlib/__init__.py\", line 126, in import_module\n", + " return _bootstrap._gcd_import(name[level:], package, level)\n", + "ImportError: libGL.so.1: cannot open shared object file: No such file or directory\n" + ] + } + ], + "source": [ + "! python scripts/rerun-segment_anything_model.py" ] }, { "cell_type": "markdown", - "id": "4c99099e", + "id": "d9bd7e11", "metadata": {}, "source": [ - "Mito Sheet empowers you to leverage your Excel proficiency within the JupyterLab environment. Seamlessly analyze datasets, perform computations, and visualize results – all within the familiar interface you know and love. Embrace the power of Mito Sheet for a more integrated and efficient data analysis experience." + "See the official ReRun [Python Quickstart](https://www.rerun.io/docs/getting-started/python) for more information about ReRun and what it can do for you." ] } ], diff --git a/docs/en/1-Experiments/Notebooks/img/rerun-cube.png b/docs/en/1-Experiments/Notebooks/img/rerun-cube.png new file mode 100644 index 000000000..28d5d591e Binary files /dev/null and b/docs/en/1-Experiments/Notebooks/img/rerun-cube.png differ diff --git a/docs/en/1-Experiments/Notebooks/img/rerun-dna.png b/docs/en/1-Experiments/Notebooks/img/rerun-dna.png new file mode 100644 index 000000000..21bb51eb1 Binary files /dev/null and b/docs/en/1-Experiments/Notebooks/img/rerun-dna.png differ diff --git a/docs/en/1-Experiments/Notebooks/img/rerun-empty.png b/docs/en/1-Experiments/Notebooks/img/rerun-empty.png new file mode 100644 index 000000000..c8f122994 Binary files /dev/null and b/docs/en/1-Experiments/Notebooks/img/rerun-empty.png differ diff --git a/docs/en/1-Experiments/Notebooks/img/rerun-multi-1.png b/docs/en/1-Experiments/Notebooks/img/rerun-multi-1.png new file mode 100644 index 000000000..8fa533b39 Binary files /dev/null and b/docs/en/1-Experiments/Notebooks/img/rerun-multi-1.png differ diff --git a/docs/en/1-Experiments/Notebooks/scripts/rerun-segment_anything_model.py b/docs/en/1-Experiments/Notebooks/scripts/rerun-segment_anything_model.py new file mode 100644 index 000000000..5124182d4 --- /dev/null +++ b/docs/en/1-Experiments/Notebooks/scripts/rerun-segment_anything_model.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +""" +Example of using Rerun to log and visualize the output of segment-anything. + +See: [segment_anything](https://segment-anything.com/). + +Can be used to test mask-generation on one or more images. Images can be local file-paths +or remote urls. + +Exa: +``` +# Run on a remote image: +python main.py https://raw.githubusercontent.com/facebookresearch/segment-anything/main/notebooks/images/dog.jpg + +# Use cuda and a different model on a local image +python main.py --device cuda --model vit_h /path/to/my_image.jpg +``` +""" +from __future__ import annotations + +import argparse +import logging +import os +from pathlib import Path +from typing import Final +from urllib.parse import urlparse + +import cv2 +import numpy as np +import requests +import rerun as rr # pip install rerun-sdk +import torch +import torchvision +from cv2 import Mat +from segment_anything import SamAutomaticMaskGenerator, sam_model_registry +from segment_anything.modeling import Sam +from tqdm import tqdm + +MODEL_DIR: Final = Path(os.path.dirname(__file__)) / "model" +MODEL_URLS: Final = { + "vit_h": "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth", + "vit_l": "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_l_0b3195.pth", + "vit_b": "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth", +} + + +def download_with_progress(url: str, dest: Path) -> None: + """Download file with tqdm progress bar.""" + chunk_size = 1024 * 1024 + resp = requests.get(url, stream=True) + total_size = int(resp.headers.get("content-length", 0)) + with open(dest, "wb") as dest_file: + with tqdm( + desc="Downloading model", total=total_size, unit="iB", unit_scale=True, unit_divisor=1024 + ) as progress: + for data in resp.iter_content(chunk_size): + dest_file.write(data) + progress.update(len(data)) + + +def get_downloaded_model_path(model_name: str) -> Path: + """Fetch the segment-anything model to a local cache directory.""" + model_url = MODEL_URLS[model_name] + + model_location = MODEL_DIR / model_url.split("/")[-1] + if not model_location.exists(): + os.makedirs(MODEL_DIR, exist_ok=True) + download_with_progress(model_url, model_location) + + return model_location + + +def create_sam(model: str, device: str) -> Sam: + """Load the segment-anything model, fetching the model-file as necessary.""" + model_path = get_downloaded_model_path(model) + + logging.info(f"PyTorch version: {torch.__version__}") + logging.info(f"Torchvision version: {torchvision.__version__}") + logging.info(f"CUDA is available: {torch.cuda.is_available()}") + + logging.info(f"Building sam from: {model_path}") + sam = sam_model_registry[model](checkpoint=model_path) + return sam.to(device=device) + + +def run_segmentation(mask_generator: SamAutomaticMaskGenerator, image: Mat) -> None: + """Run segmentation on a single image.""" + rr.log("image", rr.Image(image)) + + logging.info("Finding masks") + masks = mask_generator.generate(image) + + logging.info(f"Found {len(masks)} masks") + + # Log all the masks stacked together as a tensor + # TODO(jleibs): Tensors with class-ids and annotation-coloring would make this much slicker + mask_tensor = ( + np.dstack([np.zeros((image.shape[0], image.shape[1]))] + [m["segmentation"] for m in masks]).astype("uint8") + * 128 + ) + rr.log("mask_tensor", rr.Tensor(mask_tensor)) + + # Note: for stacking, it is important to sort these masks by area from largest to smallest + # this is because the masks are overlapping and we want smaller masks to + # be drawn on top of larger masks. + # TODO(jleibs): we could instead draw each mask as a separate image layer, but the current layer-stacking + # does not produce great results. + masks_with_ids = list(enumerate(masks, start=1)) + masks_with_ids.sort(key=(lambda x: x[1]["area"]), reverse=True) # type: ignore[no-any-return] + + # Layer all of the masks together, using the id as class-id in the segmentation + segmentation_img = np.zeros((image.shape[0], image.shape[1])) + for id, m in masks_with_ids: + segmentation_img[m["segmentation"]] = id + + rr.log("image/masks", rr.SegmentationImage(segmentation_img.astype(np.uint8))) + + mask_bbox = np.array([m["bbox"] for _, m in masks_with_ids]) + rr.log( + "image/boxes", + rr.Boxes2D(array=mask_bbox, array_format=rr.Box2DFormat.XYWH, class_ids=[id for id, _ in masks_with_ids]), + ) + + +def is_url(path: str) -> bool: + """Check if a path is a url or a local file.""" + try: + result = urlparse(path) + return all([result.scheme, result.netloc]) + except ValueError: + return False + + +def load_image(image_uri: str) -> Mat: + """Conditionally download an image from URL or load it from disk.""" + logging.info(f"Loading: {image_uri}") + if is_url(image_uri): + response = requests.get(image_uri) + response.raise_for_status() + image_data = np.asarray(bytearray(response.content), dtype="uint8") + image = cv2.imdecode(image_data, cv2.IMREAD_COLOR) + else: + image = cv2.imread(image_uri, cv2.IMREAD_COLOR) + + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + return image + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Run the Facebook Research Segment Anything example.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--model", + action="store", + default="vit_b", + choices=MODEL_URLS.keys(), + help="Which model to use." "(See: https://github.com/facebookresearch/segment-anything#model-checkpoints)", + ) + parser.add_argument( + "--device", + action="store", + default="cpu", + help="Which torch device to use, e.g. cpu or cuda. " + "(See: https://pytorch.org/docs/stable/tensor_attributes.html#torch.device)", + ) + parser.add_argument( + "--points-per-batch", + action="store", + default=32, + type=int, + help="Points per batch. More points will run faster, but too many will exhaust GPU memory.", + ) + parser.add_argument("images", metavar="N", type=str, nargs="*", help="A list of images to process.") + + rr.script_add_args(parser) + args = parser.parse_args() + + rr.script_setup(args, "rerun_example_segment_anything_model") + logging.getLogger().addHandler(rr.LoggingHandler("logs")) + logging.getLogger().setLevel(logging.INFO) + + sam = create_sam(args.model, args.device) + + mask_config = {"points_per_batch": args.points_per_batch} + mask_generator = SamAutomaticMaskGenerator(sam, **mask_config) + + if len(args.images) == 0: + logging.info("No image provided. Using default.") + args.images = [ + "https://raw.githubusercontent.com/facebookresearch/segment-anything/main/notebooks/images/truck.jpg" + ] + + for n, image_uri in enumerate(args.images): + rr.set_time_sequence("image", n) + image = load_image(image_uri) + run_segmentation(mask_generator, image) + + rr.script_teardown(args) + + +if __name__ == "__main__": + main()