From 3ec21c8e3032cbe04f7623955cbfa2efd6310560 Mon Sep 17 00:00:00 2001 From: Filip Jeretina <59307111+zrezke@users.noreply.github.com> Date: Thu, 22 Jun 2023 12:51:47 +0200 Subject: [PATCH] Release 0.1.0 (#21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * v0.0.1 of DepthaiViewer, WIP (#1) * node graph scaffolding and a bit of depthai integration * remove import * left, right + depth * fix fps sliders for mono cams * partial refactor, partially added support for device selection. * pc support + currently selected device fixes * reafactor subscriptions to api, doesn't work too well, trouble with syncing the ui with the backend * Partially migrated to websockets for the api * finish moving to websockets for the config api * fix issues after merge, retry websocket connections * fix ws thread not exiting * sleep for 1 sec before trying to reconnect * Ai support + imu support + rotate camera + bug fixes (#9) * ai support + update pipeline on device select * quick fix depth * set subs when selecting device * rename to Depthai Viewer, use changed() for config ui, disable ui when device not selected * added age gender detection * added mobilenet support * pointcloud support * Toggle subs from visible (#4) * very bad implementation of toggling subscriptions based on space view visibility TODO: less cloning and stuff * better implementation and actually seems to work * fix windows not resizing properly when size gets smaller * fix issues with 3d when resizing scene bbox, only force 2d to always have the size of self.scene_bbox * Nuked Subscriptions struct, just use Vec, much nicer, much better aswell, also fixed subscriptions for channels that need some time to actually show up, it all feels a bit hacky tho * Toggle subs from visible (#5) * very bad implementation of toggling subscriptions based on space view visibility TODO: less cloning and stuff * better implementation and actually seems to work * fix windows not resizing properly when size gets smaller * fix issues with 3d when resizing scene bbox, only force 2d to always have the size of self.scene_bbox * Nuked Subscriptions struct, just use Vec, much nicer, much better aswell, also fixed subscriptions for channels that need some time to actually show up, it all feels a bit hacky tho * fix incorrect despawn of entity paths * remove todo comment * Fix ws not sending messages, fix stutter every 2s (#6) * very bad implementation of toggling subscriptions based on space view visibility TODO: less cloning and stuff * better implementation and actually seems to work * fix windows not resizing properly when size gets smaller * fix issues with 3d when resizing scene bbox, only force 2d to always have the size of self.scene_bbox * Nuked Subscriptions struct, just use Vec, much nicer, much better aswell, also fixed subscriptions for channels that need some time to actually show up, it all feels a bit hacky tho * fix incorrect despawn of entity paths * remove todo comment * error handling * fix websocket not sending pipeline, hotfix more or less might keep * change back to unbounded * Imu support (#8) * add imu logging, exclude it from regular space views * fix imu logging * POC imu plotting * Imu accelerometer and gyroscope plotting * add orientation to imu log, make device id a String * fix imu charts layout * imu log add magnetometer * fix right panel ui as much as it makes sense to fix rn * sadly idk how to create Option::None in pyarrow for the magnetometer field * imu logging fixed, magnetometer value is now None when not logged * fix plot scrolling * Merge upstream (#10) * `arrow2_convert` primitive (de)serialization benchmarks (#1742) * arrow2_convert primitive benchmarks * addressing PR comments * Fix logged obb being displayed with half of the requested size (#1749) * benchmarks for common vector ops across `smallvec`/`tinyvec`/std (#1747) * benchmarks for common vector ops * handle N=1 * Tracked 3D cameras lead now to on-hover rays in other space views that show the same camera but don't track it. (#1751) In the same way as a 2D scene causes a on-hover ray in all space views that contain the space camera at which the 2D view "sits". * Improve dealing with raw buffers for texture read/write (#1744) * Replace TextureRowDataInfo with the more versatile Texture2DBufferInfo * comment & naming fixes * `arrow2` erased refcounted clones benchmarks (#1745) * arrow2 erased refcounted clone benchmarks * lint * addressing PR comments * dude * `arrow2` estimated_bytes_size benchmarks (#1743) * arrow2 estimated_bytes_size benchmarks * cleanup * Fix crash when trying to do picking on depth clouds * Readback depth from GPU picking (#1752) * gpu picking in the viewer picks up depth now * WebGL workarounds * Add new ARKitScenes example (#1538) Co-authored-by: Nikolaus West Co-authored-by: Emil Ernerfeldt * Fix log_obb usage (#1761) * Make sure all log_obb uses uses half_size correctly * Remove outdated link from README * Fix docstring of save * Force named arguments of log_scalar * Add link to --memory-limit docs * update ros example * Python SDK: document that we also accept colors in 0-1 floats (#1740) * Python SDK: document that we also accept colors in 0-1 floats * Assume float colors to be in gamma-space, and document that * Update arkitscenes example * Fix bug * typo * py-format * Collapse space-view by default if there is only one child (#1762) * Always create the log_time timeline (#1763) * Columnar timepoints in data tables and during transport (#1767) * columnar timepoints * self review * Fix undo/redo selection shortcut/action changing selection history without changing selection (#1765) * Fix undo/redo selection shortcut/action changing selection history without changing selection Fixes #1172 * typo fix * Don't initialize an SDK session if we are only going to be launching the app (#1768) * Allow torch tensors for log_rigid3 (#1769) * Option to show scene bounding box (#1770) * Include depth clouds in bounding box calculation * Don't wrap text when showing bbox in ui * Handle projective transforms * Nicer selection view: don't wrap second column too early * Add checkbox to show the scene bounding box * Fix a whole lot of crashes, all at once (#1780) * Add typing_extensions to requirements-doc.txt (#1786) * auto_color class-ids if they are present (#1783) * auto_color class-ids if they are present * Update log line in segmentation demo * Avoid tuple structs * Don't run 3rd party bench suites on CI (#1787) * dont run 3rd party bench suites on CI * typo * and other annoyances * Use copilot markers in PR template (#1784) * Use copilot markers in PR template * remove poem Co-authored-by: Clement Rey --------- Co-authored-by: Clement Rey * re_format: barebone support for custom formatting (#1776) * implement barebone support for custom formatting and apply to Tuid * unwrap * rather than [] * use re_tuid * Always send recording_id as part of LogMsg (#1778) * Always send recording_id as part of LogMsg * Rename build_chunk_from_components -> build_data_table_from_components * Don't make RecordingInfo optional * Always default the recording id * Log an error if we hit the initialization issue * Refactor: Add new helper crate `re_log_encoding` (#1772) * CI: Check `rerun` with --no-default features and/or with --features sdk * Create a new helper crate re_transport containing stream_rrd_from_http * Fix warnings * Move file sink to re_transport * wasm compilation fix * Move LogMsg encoding/decoding into re_transport * Fix typo * Fix web build * Fix tests * Remove a lot of unused dependencies with `cargo machete` * Build fix * Clarify * Rename the crate to re_log_encoding * better docstring Co-authored-by: Jeremy Leibs * better readme Co-authored-by: Jeremy Leibs --------- Co-authored-by: Jeremy Leibs * New example code for facebook research segment anything (#1788) * New example code for facebook research segment anything * Add segmentation workaround for users still on 0.4.0 * Images should use class-id as label * Add an alternative tensor-based view * Implement `re_tuid::Tuid::random()` on web (#1796) * Implement `re_tuid::Tuid::random()` on web * Fix bad error message * ci: fix benchmarks (#1799) * workflow: just run --all * datastore: skip bucket permutations etc on CI * i give up, just replace re_log_types by re_log_encoding * Add `minimal_options` example (`RerunArgs`) (#1773) * Allows connecting to remote server through rerun's RerunArgs. Co-authored-by: Clement Rey * Add `pacman` support to `setup_web.sh` (#1797) Co-authored-by: Clement Rey * fix ci (cargo-deny): cargo update -p crossbeam-channel (#1806) * Compile with `panic = "abort"` (#1813) * Compile with `panic = "abort"` This PR sets `panic = "abort"` for both debug and release builds. This cuts down the `rerun` binary size in release builds from 29.9 MB to 22.7 MB - a 25% reduction! ## Details The default panic behavior in Rust is to unwind the stack. This leads to a lot of extra code bloat, and some missed opportunities for optimization. The benefit is that one can let a thread die without crashing the whole application, and one can use `std::panic::catch_unwind` as a kind of try-catch block. We don't make use of these features at all (at least not intentionally), and so are paying a cost for something we don't need. I would also argue that a panic SHOULD lead to a hard crash unless you are building an Erlang-like robust actor system where you use defensive programming to protect against programmer errors (all panics are programmer errors - user errors should use `Result`). * Quiet clippy * Add `rerun --strict`: crash if any warning or error is logged (#1812) * Add `rerun --strict`: crash if any warning or error is logged Part of https://github.com/rerun-io/rerun/issues/1483 * Can't doc-test private functions * Refactor: Remove `TensorTrait` (#1819) * Refactor: Remove `TensorTrait` We don't need it anymore * End-to-end testing of python logging -> store ingestion (#1817) * Sort the arguments to `rerun` * Pass on `LogMsg::Goodbye` just like any other message * Add `rerun --test-receive` * `just py-build --quiet` is now possible * Add scripts/run_python_e2e_test.py * replace `cargo r -p rerun` with `python3 -m rerun` * lint and explain choice of examples * Add to CI * check returncode * Fix e2e test on CI: Don't try to re-build rerun-sdk (#1821) * Use gpu picking for points, streamline/share picking code some more (#1814) * use gpu picking for picking points * gpu based picking no longer works like a fallback but integrates with other picking sources * fix incorrect cursor rounding for picking * refactor picking context to be a pub struct with exposed state * unify ui picking method for 2d & 3d space views * less indentation for picking method * picking rect size is dynamically chosen * fix accidental z scaling in projection correction for picking & make cropped_projection_from_projection easier to read * CI: install pip requirements for Python e2e test * Process 2d points always in batches (#1820) * Fix CI syntax error * New option to disable persistent storage (#1825) * New option to disable persistent storage * New API to reset_time (#1826) * Datastore revamp 1: new indexing model & core datastructures (#1727) * Datastore revamp 2: serialization & formatting (#1735) * Datastore revamp 3: efficient incremental stats (#1739) * Datastore revamp 4: sunset `MsgId` (#1785) * Datastore revamp 5: (#1791) * Datastore revamp 6: sunset `LogMsg` storage + save store to disk (#1795) * Datastore revamp 7: garbage collection (#1801) * Don't assert if inserting a rowid with a matching timepoint does not create a conflict (#1832) * CI: Try installing the correct wheel on CI * CI: try again with the CI * re_query: up to date with latest data types and structures (#1828) * No more raw arrays for primary components * Don't need to carry around component names no more * Cluster keys are now raw-array-less and _not_ optional anymore * that is done indeed * helpers * datastore: incremental metadata registry stats (#1833) * add profile scopes for stats * implement incremental metadata registry stats * lint * future proofing comment * RFC: datastore state of the union & end-to-end batching (#1610) * add batching rfc * Update design/batching.md Co-authored-by: Emil Ernerfeldt * Update design/batching.md Co-authored-by: Emil Ernerfeldt * Update design/batching.md Co-authored-by: Emil Ernerfeldt * Update design/batching.md Co-authored-by: Emil Ernerfeldt * qa component instances vs. instance keys * no sticky * qa are there any special components & non-integer instance keys * give some clue about cell incompatibility * temporary constructs * zstd is already setup * more planning * date and links --------- Co-authored-by: Emil Ernerfeldt * Python CI: use bash as shell * Fix too many points crash (#1822) * Simplify point cloud builder and fix crash on too many points Fixes #1779 * faster point cloud population by not chaining iterators with default values * Use GPU picking for line(like) primitives, fix `interactive` flags (#1829) * line strip builder no longer has user data, exposes picking id instead (not implemented yet) * handle interactive object property when evaluating picking code * take line strip builder directly when building up line draw data * finish implementing picking for lines * remove unused iter_strips_with_vertices * Simplify picking handling now that there are a lot less types. Labels & textured rects are always picked now, fixes #1021 * CI: try to fix mac wheel build * Reduce memory used by staging belts on Web (#1836) In particular this prevents crashing with out of memory on a run-away belt memory usage caused by failure to unmap buffers. A bit concerningly, the fix uses our knowledge of how `wgpu::Device::poll` is broken in the current wgpu version. I took the opportunity to sharpens the definition of `HardwareTier` a bit. * CI: only test the x86_64 wheel on macos * Always flush when we remove a sink (#1830) Whenever we disconnect (or implicitly disconnect by swapping a sink) we should flush the pending messages. Additionally disconnect and flush calls both require releasing the GIL (for the same reason as shutdown previously). * GPU colormapping, first step (#1835) * Add TextureManager2D::get_or_create_with * Small code cleanup * Add code to upload a Tensor to a GPU texture * Add helper method Tensor::image_height_width_depth * Minor code cleanup (multiplicative_tint) * Hook up color textures via the new path * Refactor: introduce ColormappedTexture * Start working on an uint sampler * merge fix * Dumb colormapping of depth textures! * Use turbo for depth maps (and single-channel images :grimace:) * Use grayscale for luminance * ColorMap -> Colormap * Apply annotation context colormaps * Support sint textures too * cleanup * merge fix * Fix RGB images * More cleanup * Better error-handlign and nicer error message * Clean up the SAMPLE_TYPE with constants * Nicer shader interface * Better error handling * Remove dead code * Self-review cleanup * Fix bug in shader when sampling sint textures * Use textureSampleLevel * Apply a gamma to the image (unused as of now) * image_height_width_channels * fix various review comments * Optimize narrow_f64_to_f32s: avoid one allocation * Test and handle all tensor dtypes as images (#1840) * Support i64 and u64 tensors * api_demo: log all image types * Don't even print out the contents of a tensor * Handle unfilterable float textures * fix typo * py-format * Simplify is_float_filterable * Add a helper function pad_and_narrow_and_cast * Reuse existing image * Exclude image_tensors demo from default api_demo * Still run all api demos in e2e test * pyformat * Install the rerun-sdk in CI using --no-index and split out linux wheel build to run first. (#1838) * Install the rerun-sdk by the expected version * Fix comment * typo * Use --no-index when installing the rerun wheel * Use the cargo_version, not the new_version * Split dependency install into its own step * Don't use force-reinstall * Refactor setting of expected_version variable. * Use bash when setting env * Always run the linux job first and use its rrds for the other wheels * GPU tensor colormapping (#1841) * Refactor: introduce struct SliceSelection * Refactor: use SliceSelection inside of ViewTensorState * MVP of tensor colormapping on GPU * Remove old ui code * Support 64-bit tensors by narrowing to f32 * Allow more colormap options * Clippy * Report range errors instead of ignoring them * Sort colormaps * Shorten function name * Create module gpu_bridge * Move some code around * Simnplify API * Create ViewBuilder::new * Fix missing colon in lint.py * fix typos and formatting * Disable texture filtering options for tensors for now * Update docstrings * Add profile scopes * ViewBuilder cleanup * Make ViewBuilder::setup non-Option * Remove Result from thing that cannot fail * Fix colormap numbering * review cleanup * pass in debug_name * Unify the `range` function * typo * Show previews of colormaps when selecting them (#1846) * Make infallible version of get_or_create_texture * Show colormap previews in UI * Spelling * Implement billinear filtering of textures (#1850) * Implement opt-in billinear filtering of textures * bilinear * MVP Support for inline-rendering of Rerun within jupyter notebooks (#1798) (#1834) (#1844) * Introduce the ability to push an rrd binary via iframe.contentWindow.postMessage * New API to output the current buffered messages as a cell in jupyter * Example notebook with the cube demo * Track that we need to send another recording msg after draining the backlog * Dynamically resolve the app location based on git commit. Allow override to use self-hosted assets * Add some crude timeout logic in case the iframe fails to load * Don't persist app state in notebooks * Introduce new MemoryRecording for use with Jupyter notebooks (#1834) * Refactor the relationship between the assorted web / websocket servers (#1844) * Rename RemoteViewerServer to WebViewerSink * CLI arguments for specifying ports * Proper typing for the ports * Disable wheel tests for x86_64-apple-darwin (#1853) * Fix typos in notebook readme (#1852) * Fix the python build when running without web_viewer enabled (#1856) * Error instead of expect inside msg_encode. (#1857) * Restore: New API to reset_time (#1826) (#1854) * Revert "Implement billinear filtering of textures (#1850)" (#1859) This reverts commit d33dab6e7a33f82ab2513058d0f85744e3ce6ef4. * Add Restart command and keyboard shortcut for moving time to start of timeline (#1802) * Add Restart button to timeline UI. This sets the timeline back to zero. * Adds Restart Command & Timeline Command * Adds Ctrl-Shift-Space keyboard modifier * Remove restart button from timeline UI. * Use cmd(Key::LeftArrow) as key combo for restart timeline. * Add TimeControl::restart to restart the current timeline. * Use this from the kb_shortcut function * fix some code nits --------- Co-authored-by: Emil Ernerfeldt * Fix shutdown race condition in `re_sdk_comms` client (#1861) * Wait for encoder to shut down before shutting down the other threads * Remove unused dependencies (#1863) * Gpu picking for depth clouds (#1849) * wip * allow for hovering depth clouds via gpu picking * Use `[x, y]: [u32; 2]` as argument --------- Co-authored-by: Emil Ernerfeldt * Remove manual depth projection from car and nyud examples (#1869) * Remove manual depth projection from car and nyud examples * revert change to cube.ipynb * revert changes to cube.ipynb * third times a charm for cube.ipynb * Improve end-to-end testing slightly (#1862) * CI: Run e2e tests with RUST_LOG=debug * Move installing of pip packaged from CI to e2e script * Re-enable bilinear interpolation again (#1860) * Revert "Revert "Implement billinear filtering of textures (#1850)" (#1859)" This reverts commit 625d2bdd241c09ff9d0ae394ba91565fa48455ec. * Split rectangle.wgsl into fragme/vertex parts to work around naga bug * Use GPU colormapping when showing images in the GUI (#1865) * Cleanup: move Default close to the struct definition * Simplify code: use if-let-else-return * Simplify code: no need for Arc * Add EntityDataUi so that the Tensor ui function knows entity path * Better naming: selection -> item * Simplify code: no optional tensor stats * Less use of anyhow * Use GPU colormapping when showing tensors in GUI * Link to issue * Optimize pad_to_four_elements for debug builds * Refactor: simpler arguments to show_zoomed_image_region_area_outline * Fix missing meter argument * Refactor: break up long function * Less use of Arc * Pipe annotation context to the hover preview * Simplify `AnnotationMap::find` * Use new GPU colormapper for the hover-zoom-in tooltip * Refactor * Add helper function for turning a Tensor into an image::DynamicImage * Fix warning on web builds * Add helper function `Tensor::could_be_dynamic_image` * Implement click-to-copy and click-to-save for tensors without egui * Convert histogram to the new system * Remove the TensorImageCache * Fix TODO formatting * bug fixes and cleanups * Rename some stuff * Build-fix * Simplify some code * Turn off benchmakrs comment on each PR (#1872) * Refactor: remove `GpuTexture2DHandle::invalid` (#1866) * Refactor TexturedRect * Remove GpuTexture2DHandle::invalid * `GpuTexture2DHandle` is always valid * spacing * Update enumflags2 to non-yanked version (#1874) * Update enumflags2 to non-yanked version ``` ❯ cargo update -p enumflags2 Updating crates.io index Updating enumflags2 v0.7.5 -> v0.7.7 Updating enumflags2_derive v0.7.4 -> v0.7.7 Updating proc-macro2 v1.0.47 -> v1.0.56 Updating quote v1.0.21 -> v1.0.26 Adding syn v2.0.15 ``` Unfortunately this adds the syn v2 dependency for some platforms * Updating dependencies is a valid label * cargo deny: check more platforms * fix stuff broken from merging upstream --------- Co-authored-by: Clement Rey Co-authored-by: benjamin de charmoy Co-authored-by: Andreas Reich Co-authored-by: Emil Ernerfeldt Co-authored-by: Pablo Vela Co-authored-by: Nikolaus West Co-authored-by: Jeremy Leibs Co-authored-by: h3mosphere <129932586+h3mosphere@users.noreply.github.com> Co-authored-by: Urho Laukkarinen * expose depth config, get available sensor resolutions for the selected device * move removing entities to a place where the removal will always be tried, not just when expanding space view header * added depth alignment * added stream enabled buttons * Depth cloud textures are now cached frame-to-frame (#1913) * Depth cloud textures are now cached frame-to-frame Simplified logic a bit by enforcing F32 texture conversion (there was a u16 path for native only) * doc fix * naming consistency, format check, remove unnecessary scaling * improve depth cloud texture check * fixes after merging * Smooth out scroll wheel input for camera zooming (#1920) * Always spawn instead of fork in multiprocessing example (#1922) * Add `--num-frames` arg to canny (webcam) example (#1923) * fix formatting issues * rerun format fix * fix spelling * lint check fixes * mypy * some more lint fixes * some more fixes for python lint checks * Collect extra egui features into the main Cargo.toml (#1926) * just rs-run-all * `just py-run-all-{native|web|rrd}` (#1927) * make all python examples handle unknown arguments gracefully * just py-run-all-{native|web|rrd} * bump version * comment out clang * Join threads at end of multi-threading example (#1934) * Add argument parsing to the rerun_demo (#1925) * More robust wait for exit condition during .serve() (#1939) * More robust wait for exit condition during .serve() * lint * Use zipfile python library instead of `unzip` command in arkitscene (#1936) * Use zipfile python library instead of `unzip` command in arkitscene Windows doesn't have unzip! * Nit: import sort order --------- Co-authored-by: Nikolaus West * Fix annotation images sometimes drawn in the background. (#1933) This caused fairly ugly rendering whenever that happened. Also cleaned up redundant image/textured_rect defintions in spatial scene buildup * Fix backslashes in arkitscene rigid transformation path (#1938) * Fix backslashes in arkitscene rigid transformation path Should be fixed properly by https://github.com/rerun-io/rerun/issues/1937 * Use PosixPath instead of .replace("\\", "/") --------- Co-authored-by: Nikolaus West * Fix hover/select highlights when picking single points in a scene with multiple point clouds (#1942) Batch vertex offset for single highlights wasn't correctly computed. Different parts of the code made different assumptions what offsets referred to * Fix hovering depth clouds (#1943) We didn't add to `scene.primitives.image`. Instead of adding to this list, it is instead now no longer needed for picking since we can very easily query for tensor again. * change python workflow for testing purposes, remove windows and macos wheels * add back one macos so the matrix is valid * 2.5GB before GC kick in on web (#1944) * change name to depthai-viewer * change pip install/uninstalls from rerun-sdk to depthai-viewer * change all occurances of rerun-sdk to depthai-viewer * change windows runner to windows-latest for now when using my personal gh * Release `0.5.0` (#1919) * changelog * 0.5.0-alpha.0 * more changelog * re_format: fix implicit recursive feature flags * publish_crates.sh: fix crate ordering * Join threads at end of multi-threading example (#1934) * Add argument parsing to the rerun_demo (#1925) * More robust wait for exit condition during .serve() (#1939) * More robust wait for exit condition during .serve() * lint * Use zipfile python library instead of `unzip` command in arkitscene (#1936) * Use zipfile python library instead of `unzip` command in arkitscene Windows doesn't have unzip! * Nit: import sort order --------- Co-authored-by: Nikolaus West * Fix annotation images sometimes drawn in the background. (#1933) This caused fairly ugly rendering whenever that happened. Also cleaned up redundant image/textured_rect defintions in spatial scene buildup * Fix backslashes in arkitscene rigid transformation path (#1938) * Fix backslashes in arkitscene rigid transformation path Should be fixed properly by https://github.com/rerun-io/rerun/issues/1937 * Use PosixPath instead of .replace("\\", "/") --------- Co-authored-by: Nikolaus West * changelog * Fix hover/select highlights when picking single points in a scene with multiple point clouds (#1942) Batch vertex offset for single highlights wasn't correctly computed. Different parts of the code made different assumptions what offsets referred to * changelog * Fix hovering depth clouds (#1943) We didn't add to `scene.primitives.image`. Instead of adding to this list, it is instead now no longer needed for picking since we can very easily query for tensor again. * changelog * 2.5GB before GC kick in on web (#1944) * changelog * 0.5.0 --------- Co-authored-by: Jeremy Leibs Co-authored-by: Andreas Reich Co-authored-by: Nikolaus West * Fix imu plots scrolling past their container * fix bottom/top panel sizing after showing the spinner on config setting * Lint error names in `map_err` (#1948) * Lint: Properly name errors in `map_err` * Use correct names for errors * fix config and stats tabs not being able to be viewed at the same time, renamed stats to IMU * New dispatch-only workflow for running the lint-job (#1950) * Fix secret in dispatch_lint.yml * Only maintain a single manual-dispatch job for testing workflows * apply button, have to make it look a bit nicer, but will do that when I fix scrolling in device configuration panel * Bump hyper version due to RUSTSEC-2023-0034 (#1951) * Add other build parameterizations to manual_dispatch.yml * Use proper if gates on the manual_dispatch.yml jobs * Add ability to save cache to manual_disaptch.yml * Standard case of inputs * Add manual step for packaging to 'manual_dispatch.yml' * add back panels when the underlying subscription reappears * Fix crash for missing class ids causing zero sized texture (#1947) * Fix crash for missing class ids causing zero sized texture Two things fixed actually: * texture manager now checks for zero sized texture, this ripples out in a lot more error handling * class id texture texture handles not having any classes gracefully * Use Display for all errors * typo * Better naming of error * Better docs and names * Fix off-by-one error * some use of `context`, change which error is implicitly converted on texture manager2d --------- Co-authored-by: Emil Ernerfeldt * fixes LR stream subscriptions (maybe breaks panels reappearing after sub is gone) (#15) * make config ui scrollable, fix padding * Move clippy_wasm/clippy.toml to under scripts (#1949) * Move clippy_wasm/clippy.toml to under scripts This is just to clean up the root a bit, and to move it closer to where it is actually used. * Fix comment typo --------- Co-authored-by: Andreas Reich * change crate version to 0.6.0-alpha.0 (#1952) * New workflow_dispatch for building wheels for a PR * initial light mode, luxonis depthai viewer * Rename build_wheels_for_pr.yml -> manual_build_wheels_for_pr.yml * Fix run-wasm crash on trying to wait for server (#1959) This ruined my dev experience for re_renderer examples a bit. Not sure what made the previous hack stop working, might be a timing issue. It ended up crashing the `cargo_run_wasm` web server * Update egui to latest and wgpu to 0.16 (#1958) * update to wgpu 0.16 and egui using this version * shader fixup for type aliases and rectangle shader * shader signed/unsigned shenanigans * more signed/unsigned issues * fix texture component count * fix picking layer depth readback crash on web * patch wgpu * better texture size estimate * fix patches * Handle leaking of prerelease into alpha version (#1953) * Make device config panel remember it's height throughout loading * hide time panel, make ai model dropdown wider * Fix incorrect memory usage stats for destroyed on-creation-mapped buffers (#1963) We actually don't have anywhere where we discard this kind of buffer yet, but if we would the stats would be wrong (noticed while doing quick & dirty experiments on the staging belt) * New manual workflow for running benches * Introduce new reusable workflow jobs and cleanup manual trigger (#1954) There are 8 reusable workflow "components" that we can use to build different scenarios: reusable_checks.yml - These are all the checks that run to ensure the code is formatted, reusable_bench.yml - This job runs the benchmarks to check for performance regressions. reusable_deploy_docs- This job deploys the python and rust documentation to https://ref.rerun.io reusable_build_and_test_wheels.yml - This job builds the wheels, runs the end-to-end test, and produces a sample RRD. The artifacts are accessible via GitHub artifacts, but not otherwise uploaded anywhere. reusable_upload_wheels.yml- This job uploads the wheels to google cloud reusable_build_web.yml - This job builds the wasm artifacts for the web. reusable_upload_web.yml - This job uploads the web assets to google cloud. By default this only uploads to: app.rerun.io/commit// reusable_pr_summary.yml - This job updates the PR summary with the results of the CI run. Example summary can be found at: https://storage.googleapis.com/rerun-builds/pull_request/1954/index.html This also introduces a manual_dispatch.yml helper as a convenience for testing these workflows and their different parameterizations. * New manual workflow for adhoc web builds * Use new CI workflows for pull-request and merge to main (#1955) on_pull_request.yml includes the following pieces: - reusable_checks.yml -- Run all of the lints, code-formatting, tests, etc. - reusable_build_and_test_wheels.yml -- Configured in a "minimal" mode with SDK includes end-to-end test and produces an rrd. - reusable_build_web.yml -- Verifies we can build the wasm - reusable_upload_web.yml -- Uploads the RRD and Wasm to app.rerun.io to confirm the demo works as well as support notebook testing. - reusable_pr_summary.yml -- Create a manifest page with a link to the on_push_main.yml includes the following pieces: - reusable_checks.yml -- Run all of the lints, code-formatting, tests, etc. - reusable_bench.yml -- Run the benchmarks - reusable_build_and_test_wheels.yml -- Builds wheels for all platforms - reusable_upload_wheel.yml -- Uploads the all the wheels to gcloud - reusable_build_web.yml -- Builds the wasm bundle - reusable_upload_web.yml -- Uploads the RRD and Wasm to app.rerun.io - reusable_pip_index.yml -- Generates a pip index page which can be used to install packages with, e.g. * Fix name of on_push_main.yml * Fix usage of long commit in generate_prerelease_pip_index.py * Try making pull-request workflows non-concurrent (#1970) * Try making pull-request workflows non-concurrent * Concurrency groups for push_main as well * Each sub-workflow needs its own name or they fight * Another attempt to make jobs non-concurrent on a per-PR basis (#1974) * Another attempt to make jobs non-concurrent on a per-PR basis * Move concurrency into the reusable job * Jobs with duplicated instances still need separate concurrency keys based on platform * Round to nearest color_index when doing color mapping (#1969) * Full (experimental) WebGPU support (#1965) * always build with unstable web sys apis * Make shader Tint friendly * expose webgl feature flag on re_renderer & re_viewer * fix bug link on negative hexadecimal * hardware tier is now created from wgpu adapter * sort out build flags for webgpu & document building webviewer * introduce shader text replacement workarounds to workaround current chrome issue * latest egui master * typo fix * doc fixes, use if cfg! instead of attribute cfg * move backend to rerun * If there's a `{{ pr-build-summary }}` in the PR description, update it. (#1971) * If there's a `{{ pr-build-summary }}` in the PR description, update it. * Add comment to the PR template * Add pull-requests permission to pr_summary job * Run the cube notebook on PR (#1972) * Run the cube notebook on PR * Add notebooks to the build summary * Use the new concurrency model * reformat py files * reformat * fix pylint errors * Add ability to manually run a web build to upload to an adhoc name (#1966) * Add ability to manually run a web build to upload to an adhoc name * Pass through ADHOC_NAME * Add a concurrency criteria for the new adhoc job * Make input description more explicit * change entity paths * merged wip albedo colormap into latest depth_cloud * remove pointclouds generated in sdk * fix compiler error, trying to compile frame.close for wasm * Default to albedo texture for depth cloud, added support for mono albedo textures * restart backend on failure, added oak_cam.device.close seems to really close the cam * py lint fix * don't run notebooks * remove run-notebook dependency --------- Co-authored-by: Clement Rey Co-authored-by: benjamin de charmoy Co-authored-by: Andreas Reich Co-authored-by: Emil Ernerfeldt Co-authored-by: Pablo Vela Co-authored-by: Nikolaus West Co-authored-by: Jeremy Leibs Co-authored-by: h3mosphere <129932586+h3mosphere@users.noreply.github.com> Co-authored-by: Urho Laukkarinen Co-authored-by: Nikolaus West * Fix rerun lints * rename binary, always start with memory-limit, pin depthai dependencies, fix errors after changing dependency versions * remove compiler warnings * fix memory leak when app is in background * Custom viewport UI (#4) * custom blueprint panel, show logical space views for depthai-viewer users, added settings clog on top of space view tab to configure what is visible * small fixes, clear entity_paths every time to avoid displaying an unavailable entity_path in space view options ui * custom left panel to add or remove space view instances, created a new default viewport layout. Improved behaviour when a panel re appears after user selected to hide it, then if stream stops and starts again the panel will be spawned back in correctly. * improve auto layout to not split when only 3D or 2D view is available * MJPEG encode image frames if connected to a PoE device. Only add magnetometer to imu sensors list if the device has a BNO IMU. Lower the memory limit to 100MB * Styling (#6) * initial styling impl * make buttons that should be small, small * Runtime depth config and fix device selection ui * comment * Proper runtime depth config updates * switch to Yolo v8 * add comma to label for non open zoo models * split 2d + 3d cam view vertically instead of horizontally * Tabify all panels (except for blueprint) (#7) * initial styling impl * make buttons that should be small, small * Runtime depth config and fix device selection ui * comment * Proper runtime depth config updates * switch to Yolo v8 * add comma to label for non open zoo models * split 2d + 3d cam view vertically instead of horizontally * Make the UI more configurable by converting the right panel into tabs. TODO: UX while laying out the panels. When a new space view appears only update the viewport layout, try to keep the user configured fixed function panels as they were. Just handle it in a way that is intuitive * remove bottom panel, switch to RAW imu sensors * XLink statistics initial implementation * initial xlink throughput statistics impllementation, have to glow it up a bit and maybe clean up the code * Xlink and rerun rename (#9) * Rename rerun py library to depthai_viewer * bug fixes and started working on a smart auto layout that operates on an existing tree, to preserve ui as much as possible, while also creating good layouts * auto layout * Fix maximize not working and add maximize for Stats tab * delete profiling stuff that shouldn't have been commited * mostly fix smart layout, TODO: detect when you can group mono and color 3d + 2d views into a 4 way split * add docstring for update_tree * WIP auto layout can_create_mono_quad checker, not at all finished yet * pass lint checks and bugfixes * forgot to sort imports * Fix mypy lint (TODO: Proper typing, especially in the comms from back to store to ws) other types are pretty solid * pylints and fix Queue typehinting * fix doc build * try to pass pylints with py.typed * ignore misc mypy errors * forgot to run black formatter * switch back to old ci * change version to 0.0.1 * format pyproject.toml * add back check_version to version_util.py * switch to normal runners * try cache-apt-pkgs-action@1.2.4 * sync mono camera settings * fix web build * Update scripts/version_util.py * Some fixes (#11) * v0.0.1 of DepthaiViewer, WIP (#1) * node graph scaffolding and a bit of depthai integration * remove import * left, right + depth * fix fps sliders for mono cams * partial refactor, partially added support for device selection. * pc support + currently selected device fixes * reafactor subscriptions to api, doesn't work too well, trouble with syncing the ui with the backend * Partially migrated to websockets for the api * finish moving to websockets for the config api * fix issues after merge, retry websocket connections * fix ws thread not exiting * sleep for 1 sec before trying to reconnect * Ai support + imu support + rotate camera + bug fixes (#9) * ai support + update pipeline on device select * quick fix depth * set subs when selecting device * rename to Depthai Viewer, use changed() for config ui, disable ui when device not selected * added age gender detection * added mobilenet support * pointcloud support * Toggle subs from visible (#4) * very bad implementation of toggling subscriptions based on space view visibility TODO: less cloning and stuff * better implementation and actually seems to work * fix windows not resizing properly when size gets smaller * fix issues with 3d when resizing scene bbox, only force 2d to always have the size of self.scene_bbox * Nuked Subscriptions struct, just use Vec, much nicer, much better aswell, also fixed subscriptions for channels that need some time to actually show up, it all feels a bit hacky tho * Toggle subs from visible (#5) * very bad implementation of toggling subscriptions based on space view visibility TODO: less cloning and stuff * better implementation and actually seems to work * fix windows not resizing properly when size gets smaller * fix issues with 3d when resizing scene bbox, only force 2d to always have the size of self.scene_bbox * Nuked Subscriptions struct, just use Vec, much nicer, much better aswell, also fixed subscriptions for channels that need some time to actually show up, it all feels a bit hacky tho * fix incorrect despawn of entity paths * remove todo comment * Fix ws not sending messages, fix stutter every 2s (#6) * very bad implementation of toggling subscriptions based on space view visibility TODO: less cloning and stuff * better implementation and actually seems to work * fix windows not resizing properly when size gets smaller * fix issues with 3d when resizing scene bbox, only force 2d to always have the size of self.scene_bbox * Nuked Subscriptions struct, just use Vec, much nicer, much better aswell, also fixed subscriptions for channels that need some time to actually show up, it all feels a bit hacky tho * fix incorrect despawn of entity paths * remove todo comment * error handling * fix websocket not sending pipeline, hotfix more or less might keep * change back to unbounded * Imu support (#8) * add imu logging, exclude it from regular space views * fix imu logging * POC imu plotting * Imu accelerometer and gyroscope plotting * add orientation to imu log, make device id a String * fix imu charts layout * imu log add magnetometer * fix right panel ui as much as it makes sense to fix rn * sadly idk how to create Option::None in pyarrow for the magnetometer field * imu logging fixed, magnetometer value is now None when not logged * fix plot scrolling * Merge upstream (#10) * `arrow2_convert` primitive (de)serialization benchmarks (#1742) * arrow2_convert primitive benchmarks * addressing PR comments * Fix logged obb being displayed with half of the requested size (#1749) * benchmarks for common vector ops across `smallvec`/`tinyvec`/std (#1747) * benchmarks for common vector ops * handle N=1 * Tracked 3D cameras lead now to on-hover rays in other space views that show the same camera but don't track it. (#1751) In the same way as a 2D scene causes a on-hover ray in all space views that contain the space camera at which the 2D view "sits". * Improve dealing with raw buffers for texture read/write (#1744) * Replace TextureRowDataInfo with the more versatile Texture2DBufferInfo * comment & naming fixes * `arrow2` erased refcounted clones benchmarks (#1745) * arrow2 erased refcounted clone benchmarks * lint * addressing PR comments * dude * `arrow2` estimated_bytes_size benchmarks (#1743) * arrow2 estimated_bytes_size benchmarks * cleanup * Fix crash when trying to do picking on depth clouds * Readback depth from GPU picking (#1752) * gpu picking in the viewer picks up depth now * WebGL workarounds * Add new ARKitScenes example (#1538) Co-authored-by: Nikolaus West Co-authored-by: Emil Ernerfeldt * Fix log_obb usage (#1761) * Make sure all log_obb uses uses half_size correctly * Remove outdated link from README * Fix docstring of save * Force named arguments of log_scalar * Add link to --memory-limit docs * update ros example * Python SDK: document that we also accept colors in 0-1 floats (#1740) * Python SDK: document that we also accept colors in 0-1 floats * Assume float colors to be in gamma-space, and document that * Update arkitscenes example * Fix bug * typo * py-format * Collapse space-view by default if there is only one child (#1762) * Always create the log_time timeline (#1763) * Columnar timepoints in data tables and during transport (#1767) * columnar timepoints * self review * Fix undo/redo selection shortcut/action changing selection history without changing selection (#1765) * Fix undo/redo selection shortcut/action changing selection history without changing selection Fixes #1172 * typo fix * Don't initialize an SDK session if we are only going to be launching the app (#1768) * Allow torch tensors for log_rigid3 (#1769) * Option to show scene bounding box (#1770) * Include depth clouds in bounding box calculation * Don't wrap text when showing bbox in ui * Handle projective transforms * Nicer selection view: don't wrap second column too early * Add checkbox to show the scene bounding box * Fix a whole lot of crashes, all at once (#1780) * Add typing_extensions to requirements-doc.txt (#1786) * auto_color class-ids if they are present (#1783) * auto_color class-ids if they are present * Update log line in segmentation demo * Avoid tuple structs * Don't run 3rd party bench suites on CI (#1787) * dont run 3rd party bench suites on CI * typo * and other annoyances * Use copilot markers in PR template (#1784) * Use copilot markers in PR template * remove poem Co-authored-by: Clement Rey --------- Co-authored-by: Clement Rey * re_format: barebone support for custom formatting (#1776) * implement barebone support for custom formatting and apply to Tuid * unwrap * rather than [] * use re_tuid * Always send recording_id as part of LogMsg (#1778) * Always send recording_id as part of LogMsg * Rename build_chunk_from_components -> build_data_table_from_components * Don't make RecordingInfo optional * Always default the recording id * Log an error if we hit the initialization issue * Refactor: Add new helper crate `re_log_encoding` (#1772) * CI: Check `rerun` with --no-default features and/or with --features sdk * Create a new helper crate re_transport containing stream_rrd_from_http * Fix warnings * Move file sink to re_transport * wasm compilation fix * Move LogMsg encoding/decoding into re_transport * Fix typo * Fix web build * Fix tests * Remove a lot of unused dependencies with `cargo machete` * Build fix * Clarify * Rename the crate to re_log_encoding * better docstring Co-authored-by: Jeremy Leibs * better readme Co-authored-by: Jeremy Leibs --------- Co-authored-by: Jeremy Leibs * New example code for facebook research segment anything (#1788) * New example code for facebook research segment anything * Add segmentation workaround for users still on 0.4.0 * Images should use class-id as label * Add an alternative tensor-based view * Implement `re_tuid::Tuid::random()` on web (#1796) * Implement `re_tuid::Tuid::random()` on web * Fix bad error message * ci: fix benchmarks (#1799) * workflow: just run --all * datastore: skip bucket permutations etc on CI * i give up, just replace re_log_types by re_log_encoding * Add `minimal_options` example (`RerunArgs`) (#1773) * Allows connecting to remote server through rerun's RerunArgs. Co-authored-by: Clement Rey * Add `pacman` support to `setup_web.sh` (#1797) Co-authored-by: Clement Rey * fix ci (cargo-deny): cargo update -p crossbeam-channel (#1806) * Compile with `panic = "abort"` (#1813) * Compile with `panic = "abort"` This PR sets `panic = "abort"` for both debug and release builds. This cuts down the `rerun` binary size in release builds from 29.9 MB to 22.7 MB - a 25% reduction! ## Details The default panic behavior in Rust is to unwind the stack. This leads to a lot of extra code bloat, and some missed opportunities for optimization. The benefit is that one can let a thread die without crashing the whole application, and one can use `std::panic::catch_unwind` as a kind of try-catch block. We don't make use of these features at all (at least not intentionally), and so are paying a cost for something we don't need. I would also argue that a panic SHOULD lead to a hard crash unless you are building an Erlang-like robust actor system where you use defensive programming to protect against programmer errors (all panics are programmer errors - user errors should use `Result`). * Quiet clippy * Add `rerun --strict`: crash if any warning or error is logged (#1812) * Add `rerun --strict`: crash if any warning or error is logged Part of https://github.com/rerun-io/rerun/issues/1483 * Can't doc-test private functions * Refactor: Remove `TensorTrait` (#1819) * Refactor: Remove `TensorTrait` We don't need it anymore * End-to-end testing of python logging -> store ingestion (#1817) * Sort the arguments to `rerun` * Pass on `LogMsg::Goodbye` just like any other message * Add `rerun --test-receive` * `just py-build --quiet` is now possible * Add scripts/run_python_e2e_test.py * replace `cargo r -p rerun` with `python3 -m rerun` * lint and explain choice of examples * Add to CI * check returncode * Fix e2e test on CI: Don't try to re-build rerun-sdk (#1821) * Use gpu picking for points, streamline/share picking code some more (#1814) * use gpu picking for picking points * gpu based picking no longer works like a fallback but integrates with other picking sources * fix incorrect cursor rounding for picking * refactor picking context to be a pub struct with exposed state * unify ui picking method for 2d & 3d space views * less indentation for picking method * picking rect size is dynamically chosen * fix accidental z scaling in projection correction for picking & make cropped_projection_from_projection easier to read * CI: install pip requirements for Python e2e test * Process 2d points always in batches (#1820) * Fix CI syntax error * New option to disable persistent storage (#1825) * New option to disable persistent storage * New API to reset_time (#1826) * Datastore revamp 1: new indexing model & core datastructures (#1727) * Datastore revamp 2: serialization & formatting (#1735) * Datastore revamp 3: efficient incremental stats (#1739) * Datastore revamp 4: sunset `MsgId` (#1785) * Datastore revamp 5: (#1791) * Datastore revamp 6: sunset `LogMsg` storage + save store to disk (#1795) * Datastore revamp 7: garbage collection (#1801) * Don't assert if inserting a rowid with a matching timepoint does not create a conflict (#1832) * CI: Try installing the correct wheel on CI * CI: try again with the CI * re_query: up to date with latest data types and structures (#1828) * No more raw arrays for primary components * Don't need to carry around component names no more * Cluster keys are now raw-array-less and _not_ optional anymore * that is done indeed * helpers * datastore: incremental metadata registry stats (#1833) * add profile scopes for stats * implement incremental metadata registry stats * lint * future proofing comment * RFC: datastore state of the union & end-to-end batching (#1610) * add batching rfc * Update design/batching.md Co-authored-by: Emil Ernerfeldt * Update design/batching.md Co-authored-by: Emil Ernerfeldt * Update design/batching.md Co-authored-by: Emil Ernerfeldt * Update design/batching.md Co-authored-by: Emil Ernerfeldt * qa component instances vs. instance keys * no sticky * qa are there any special components & non-integer instance keys * give some clue about cell incompatibility * temporary constructs * zstd is already setup * more planning * date and links --------- Co-authored-by: Emil Ernerfeldt * Python CI: use bash as shell * Fix too many points crash (#1822) * Simplify point cloud builder and fix crash on too many points Fixes #1779 * faster point cloud population by not chaining iterators with default values * Use GPU picking for line(like) primitives, fix `interactive` flags (#1829) * line strip builder no longer has user data, exposes picking id instead (not implemented yet) * handle interactive object property when evaluating picking code * take line strip builder directly when building up line draw data * finish implementing picking for lines * remove unused iter_strips_with_vertices * Simplify picking handling now that there are a lot less types. Labels & textured rects are always picked now, fixes #1021 * CI: try to fix mac wheel build * Reduce memory used by staging belts on Web (#1836) In particular this prevents crashing with out of memory on a run-away belt memory usage caused by failure to unmap buffers. A bit concerningly, the fix uses our knowledge of how `wgpu::Device::poll` is broken in the current wgpu version. I took the opportunity to sharpens the definition of `HardwareTier` a bit. * CI: only test the x86_64 wheel on macos * Always flush when we remove a sink (#1830) Whenever we disconnect (or implicitly disconnect by swapping a sink) we should flush the pending messages. Additionally disconnect and flush calls both require releasing the GIL (for the same reason as shutdown previously). * GPU colormapping, first step (#1835) * Add TextureManager2D::get_or_create_with * Small code cleanup * Add code to upload a Tensor to a GPU texture * Add helper method Tensor::image_height_width_depth * Minor code cleanup (multiplicative_tint) * Hook up color textures via the new path * Refactor: introduce ColormappedTexture * Start working on an uint sampler * merge fix * Dumb colormapping of depth textures! * Use turbo for depth maps (and single-channel images :grimace:) * Use grayscale for luminance * ColorMap -> Colormap * Apply annotation context colormaps * Support sint textures too * cleanup * merge fix * Fix RGB images * More cleanup * Better error-handlign and nicer error message * Clean up the SAMPLE_TYPE with constants * Nicer shader interface * Better error handling * Remove dead code * Self-review cleanup * Fix bug in shader when sampling sint textures * Use textureSampleLevel * Apply a gamma to the image (unused as of now) * image_height_width_channels * fix various review comments * Optimize narrow_f64_to_f32s: avoid one allocation * Test and handle all tensor dtypes as images (#1840) * Support i64 and u64 tensors * api_demo: log all image types * Don't even print out the contents of a tensor * Handle unfilterable float textures * fix typo * py-format * Simplify is_float_filterable * Add a helper function pad_and_narrow_and_cast * Reuse existing image * Exclude image_tensors demo from default api_demo * Still run all api demos in e2e test * pyformat * Install the rerun-sdk in CI using --no-index and split out linux wheel build to run first. (#1838) * Install the rerun-sdk by the expected version * Fix comment * typo * Use --no-index when installing the rerun wheel * Use the cargo_version, not the new_version * Split dependency install into its own step * Don't use force-reinstall * Refactor setting of expected_version variable. * Use bash when setting env * Always run the linux job first and use its rrds for the other wheels * GPU tensor colormapping (#1841) * Refactor: introduce struct SliceSelection * Refactor: use SliceSelection inside of ViewTensorState * MVP of tensor colormapping on GPU * Remove old ui code * Support 64-bit tensors by narrowing to f32 * Allow more colormap options * Clippy * Report range errors instead of ignoring them * Sort colormaps * Shorten function name * Create module gpu_bridge * Move some code around * Simnplify API * Create ViewBuilder::new * Fix missing colon in lint.py * fix typos and formatting * Disable texture filtering options for tensors for now * Update docstrings * Add profile scopes * ViewBuilder cleanup * Make ViewBuilder::setup non-Option * Remove Result from thing that cannot fail * Fix colormap numbering * review cleanup * pass in debug_name * Unify the `range` function * typo * Show previews of colormaps when selecting them (#1846) * Make infallible version of get_or_create_texture * Show colormap previews in UI * Spelling * Implement billinear filtering of textures (#1850) * Implement opt-in billinear filtering of textures * bilinear * MVP Support for inline-rendering of Rerun within jupyter notebooks (#1798) (#1834) (#1844) * Introduce the ability to push an rrd binary via iframe.contentWindow.postMessage * New API to output the current buffered messages as a cell in jupyter * Example notebook with the cube demo * Track that we need to send another recording msg after draining the backlog * Dynamically resolve the app location based on git commit. Allow override to use self-hosted assets * Add some crude timeout logic in case the iframe fails to load * Don't persist app state in notebooks * Introduce new MemoryRecording for use with Jupyter notebooks (#1834) * Refactor the relationship between the assorted web / websocket servers (#1844) * Rename RemoteViewerServer to WebViewerSink * CLI arguments for specifying ports * Proper typing for the ports * Disable wheel tests for x86_64-apple-darwin (#1853) * Fix typos in notebook readme (#1852) * Fix the python build when running without web_viewer enabled (#1856) * Error instead of expect inside msg_encode. (#1857) * Restore: New API to reset_time (#1826) (#1854) * Revert "Implement billinear filtering of textures (#1850)" (#1859) This reverts commit d33dab6e7a33f82ab2513058d0f85744e3ce6ef4. * Add Restart command and keyboard shortcut for moving time to start of timeline (#1802) * Add Restart button to timeline UI. This sets the timeline back to zero. * Adds Restart Command & Timeline Command * Adds Ctrl-Shift-Space keyboard modifier * Remove restart button from timeline UI. * Use cmd(Key::LeftArrow) as key combo for restart timeline. * Add TimeControl::restart to restart the current timeline. * Use this from the kb_shortcut function * fix some code nits --------- Co-authored-by: Emil Ernerfeldt * Fix shutdown race condition in `re_sdk_comms` client (#1861) * Wait for encoder to shut down before shutting down the other threads * Remove unused dependencies (#1863) * Gpu picking for depth clouds (#1849) * wip * allow for hovering depth clouds via gpu picking * Use `[x, y]: [u32; 2]` as argument --------- Co-authored-by: Emil Ernerfeldt * Remove manual depth projection from car and nyud examples (#1869) * Remove manual depth projection from car and nyud examples * revert change to cube.ipynb * revert changes to cube.ipynb * third times a charm for cube.ipynb * Improve end-to-end testing slightly (#1862) * CI: Run e2e tests with RUST_LOG=debug * Move installing of pip packaged from CI to e2e script * Re-enable bilinear interpolation again (#1860) * Revert "Revert "Implement billinear filtering of textures (#1850)" (#1859)" This reverts commit 625d2bdd241c09ff9d0ae394ba91565fa48455ec. * Split rectangle.wgsl into fragme/vertex parts to work around naga bug * Use GPU colormapping when showing images in the GUI (#1865) * Cleanup: move Default close to the struct definition * Simplify code: use if-let-else-return * Simplify code: no need for Arc * Add EntityDataUi so that the Tensor ui function knows entity path * Better naming: selection -> item * Simplify code: no optional tensor stats * Less use of anyhow * Use GPU colormapping when showing tensors in GUI * Link to issue * Optimize pad_to_four_elements for debug builds * Refactor: simpler arguments to show_zoomed_image_region_area_outline * Fix missing meter argument * Refactor: break up long function * Less use of Arc * Pipe annotation context to the hover preview * Simplify `AnnotationMap::find` * Use new GPU colormapper for the hover-zoom-in tooltip * Refactor * Add helper function for turning a Tensor into an image::DynamicImage * Fix warning on web builds * Add helper function `Tensor::could_be_dynamic_image` * Implement click-to-copy and click-to-save for tensors without egui * Convert histogram to the new system * Remove the TensorImageCache * Fix TODO formatting * bug fixes and cleanups * Rename some stuff * Build-fix * Simplify some code * Turn off benchmakrs comment on each PR (#1872) * Refactor: remove `GpuTexture2DHandle::invalid` (#1866) * Refactor TexturedRect * Remove GpuTexture2DHandle::invalid * `GpuTexture2DHandle` is always valid * spacing * Update enumflags2 to non-yanked version (#1874) * Update enumflags2 to non-yanked version ``` ❯ cargo update -p enumflags2 Updating crates.io index Updating enumflags2 v0.7.5 -> v0.7.7 Updating enumflags2_derive v0.7.4 -> v0.7.7 Updating proc-macro2 v1.0.47 -> v1.0.56 Updating quote v1.0.21 -> v1.0.26 Adding syn v2.0.15 ``` Unfortunately this adds the syn v2 dependency for some platforms * Updating dependencies is a valid label * cargo deny: check more platforms * fix stuff broken from merging upstream --------- Co-authored-by: Clement Rey Co-authored-by: benjamin de charmoy Co-authored-by: Andreas Reich Co-authored-by: Emil Ernerfeldt Co-authored-by: Pablo Vela Co-authored-by: Nikolaus West Co-authored-by: Jeremy Leibs Co-authored-by: h3mosphere <129932586+h3mosphere@users.noreply.github.com> Co-authored-by: Urho Laukkarinen * expose depth config, get available sensor resolutions for the selected device * move removing entities to a place where the removal will always be tried, not just when expanding space view header * added depth alignment * added stream enabled buttons * Depth cloud textures are now cached frame-to-frame (#1913) * Depth cloud textures are now cached frame-to-frame Simplified logic a bit by enforcing F32 texture conversion (there was a u16 path for native only) * doc fix * naming consistency, format check, remove unnecessary scaling * improve depth cloud texture check * fixes after merging * Smooth out scroll wheel input for camera zooming (#1920) * Always spawn instead of fork in multiprocessing example (#1922) * Add `--num-frames` arg to canny (webcam) example (#1923) * fix formatting issues * rerun format fix * fix spelling * lint check fixes * mypy * some more lint fixes * some more fixes for python lint checks * Collect extra egui features into the main Cargo.toml (#1926) * just rs-run-all * `just py-run-all-{native|web|rrd}` (#1927) * make all python examples handle unknown arguments gracefully * just py-run-all-{native|web|rrd} * bump version * comment out clang * Join threads at end of multi-threading example (#1934) * Add argument parsing to the rerun_demo (#1925) * More robust wait for exit condition during .serve() (#1939) * More robust wait for exit condition during .serve() * lint * Use zipfile python library instead of `unzip` command in arkitscene (#1936) * Use zipfile python library instead of `unzip` command in arkitscene Windows doesn't have unzip! * Nit: import sort order --------- Co-authored-by: Nikolaus West * Fix annotation images sometimes drawn in the background. (#1933) This caused fairly ugly rendering whenever that happened. Also cleaned up redundant image/textured_rect defintions in spatial scene buildup * Fix backslashes in arkitscene rigid transformation path (#1938) * Fix backslashes in arkitscene rigid transformation path Should be fixed properly by https://github.com/rerun-io/rerun/issues/1937 * Use PosixPath instead of .replace("\\", "/") --------- Co-authored-by: Nikolaus West * Fix hover/select highlights when picking single points in a scene with multiple point clouds (#1942) Batch vertex offset for single highlights wasn't correctly computed. Different parts of the code made different assumptions what offsets referred to * Fix hovering depth clouds (#1943) We didn't add to `scene.primitives.image`. Instead of adding to this list, it is instead now no longer needed for picking since we can very easily query for tensor again. * change python workflow for testing purposes, remove windows and macos wheels * add back one macos so the matrix is valid * 2.5GB before GC kick in on web (#1944) * change name to depthai-viewer * change pip install/uninstalls from rerun-sdk to depthai-viewer * change all occurances of rerun-sdk to depthai-viewer * change windows runner to windows-latest for now when using my personal gh * Release `0.5.0` (#1919) * changelog * 0.5.0-alpha.0 * more changelog * re_format: fix implicit recursive feature flags * publish_crates.sh: fix crate ordering * Join threads at end of multi-threading example (#1934) * Add argument parsing to the rerun_demo (#1925) * More robust wait for exit condition during .serve() (#1939) * More robust wait for exit condition during .serve() * lint * Use zipfile python library instead of `unzip` command in arkitscene (#1936) * Use zipfile python library instead of `unzip` command in arkitscene Windows doesn't have unzip! * Nit: import sort order --------- Co-authored-by: Nikolaus West * Fix annotation images sometimes drawn in the background. (#1933) This caused fairly ugly rendering whenever that happened. Also cleaned up redundant image/textured_rect defintions in spatial scene buildup * Fix backslashes in arkitscene rigid transformation path (#1938) * Fix backslashes in arkitscene rigid transformation path Should be fixed properly by https://github.com/rerun-io/rerun/issues/1937 * Use PosixPath instead of .replace("\\", "/") --------- Co-authored-by: Nikolaus West * changelog * Fix hover/select highlights when picking single points in a scene with multiple point clouds (#1942) Batch vertex offset for single highlights wasn't correctly computed. Different parts of the code made different assumptions what offsets referred to * changelog * Fix hovering depth clouds (#1943) We didn't add to `scene.primitives.image`. Instead of adding to this list, it is instead now no longer needed for picking since we can very easily query for tensor again. * changelog * 2.5GB before GC kick in on web (#1944) * changelog * 0.5.0 --------- Co-authored-by: Jeremy Leibs Co-authored-by: Andreas Reich Co-authored-by: Nikolaus West * Fix imu plots scrolling past their container * fix bottom/top panel sizing after showing the spinner on config setting * Lint error names in `map_err` (#1948) * Lint: Properly name errors in `map_err` * Use correct names for errors * fix config and stats tabs not being able to be viewed at the same time, renamed stats to IMU * New dispatch-only workflow for running the lint-job (#1950) * Fix secret in dispatch_lint.yml * Only maintain a single manual-dispatch job for testing workflows * apply button, have to make it look a bit nicer, but will do that when I fix scrolling in device configuration panel * Bump hyper version due to RUSTSEC-2023-0034 (#1951) * Add other build parameterizations to manual_dispatch.yml * Use proper if gates on the manual_dispatch.yml jobs * Add ability to save cache to manual_disaptch.yml * Standard case of inputs * Add manual step for packaging to 'manual_dispatch.yml' * add back panels when the underlying subscription reappears * Fix crash for missing class ids causing zero sized texture (#1947) * Fix crash for missing class ids causing zero sized texture Two things fixed actually: * texture manager now checks for zero sized texture, this ripples out in a lot more error handling * class id texture texture handles not having any classes gracefully * Use Display for all errors * typo * Better naming of error * Better docs and names * Fix off-by-one error * some use of `context`, change which error is implicitly converted on texture manager2d --------- Co-authored-by: Emil Ernerfeldt * fixes LR stream subscriptions (maybe breaks panels reappearing after sub is gone) (#15) * make config ui scrollable, fix padding * Move clippy_wasm/clippy.toml to under scripts (#1949) * Move clippy_wasm/clippy.toml to under scripts This is just to clean up the root a bit, and to move it closer to where it is actually used. * Fix comment typo --------- Co-authored-by: Andreas Reich * change crate version to 0.6.0-alpha.0 (#1952) * New workflow_dispatch for building wheels for a PR * initial light mode, luxonis depthai viewer * Rename build_wheels_for_pr.yml -> manual_build_wheels_for_pr.yml * Fix run-wasm crash on trying to wait for server (#1959) This ruined my dev experience for re_renderer examples a bit. Not sure what made the previous hack stop working, might be a timing issue. It ended up crashing the `cargo_run_wasm` web server * Update egui to latest and wgpu to 0.16 (#1958) * update to wgpu 0.16 and egui using this version * shader fixup for type aliases and rectangle shader * shader signed/unsigned shenanigans * more signed/unsigned issues * fix texture component count * fix picking layer depth readback crash on web * patch wgpu * better texture size estimate * fix patches * Handle leaking of prerelease into alpha version (#1953) * Make device config panel remember it's height throughout loading * hide time panel, make ai model dropdown wider * Fix incorrect memory usage stats for destroyed on-creation-mapped buffers (#1963) We actually don't have anywhere where we discard this kind of buffer yet, but if we would the stats would be wrong (noticed while doing quick & dirty experiments on the staging belt) * New manual workflow for running benches * Introduce new reusable workflow jobs and cleanup manual trigger (#1954) There are 8 reusable workflow "components" that we can use to build different scenarios: reusable_checks.yml - These are all the checks that run to ensure the code is formatted, reusable_bench.yml - This job runs the benchmarks to check for performance regressions. reusable_deploy_docs- This job deploys the python and rust documentation to https://ref.rerun.io reusable_build_and_test_wheels.yml - This job builds the wheels, runs the end-to-end test, and produces a sample RRD. The artifacts are accessible via GitHub artifacts, but not otherwise uploaded anywhere. reusable_upload_wheels.yml- This job uploads the wheels to google cloud reusable_build_web.yml - This job builds the wasm artifacts for the web. reusable_upload_web.yml - This job uploads the web assets to google cloud. By default this only uploads to: app.rerun.io/commit// reusable_pr_summary.yml - This job updates the PR summary with the results of the CI run. Example summary can be found at: https://storage.googleapis.com/rerun-builds/pull_request/1954/index.html This also introduces a manual_dispatch.yml helper as a convenience for testing these workflows and their different parameterizations. * New manual workflow for adhoc web builds * Use new CI workflows for pull-request and merge to main (#1955) on_pull_request.yml includes the following pieces: - reusable_checks.yml -- Run all of the lints, code-formatting, tests, etc. - reusable_build_and_test_wheels.yml -- Configured in a "minimal" mode with SDK includes end-to-end test and produces an rrd. - reusable_build_web.yml -- Verifies we can build the wasm - reusable_upload_web.yml -- Uploads the RRD and Wasm to app.rerun.io to confirm the demo works as well as support notebook testing. - reusable_pr_summary.yml -- Create a manifest page with a link to the on_push_main.yml includes the following pieces: - reusable_checks.yml -- Run all of the lints, code-formatting, tests, etc. - reusable_bench.yml -- Run the benchmarks - reusable_build_and_test_wheels.yml -- Builds wheels for all platforms - reusable_upload_wheel.yml -- Uploads the all the wheels to gcloud - reusable_build_web.yml -- Builds the wasm bundle - reusable_upload_web.yml -- Uploads the RRD and Wasm to app.rerun.io - reusable_pip_index.yml -- Generates a pip index page which can be used to install packages with, e.g. * Fix name of on_push_main.yml * Fix usage of long commit in generate_prerelease_pip_index.py * Try making pull-request workflows non-concurrent (#1970) * Try making pull-request workflows non-concurrent * Concurrency groups for push_main as well * Each sub-workflow needs its own name or they fight * Another attempt to make jobs non-concurrent on a per-PR basis (#1974) * Another attempt to make jobs non-concurrent on a per-PR basis * Move concurrency into the reusable job * Jobs with duplicated instances still need separate concurrency keys based on platform * Round to nearest color_index when doing color mapping (#1969) * Full (experimental) WebGPU support (#1965) * always build with unstable web sys apis * Make shader Tint friendly * expose webgl feature flag on re_renderer & re_viewer * fix bug link on negative hexadecimal * hardware tier is now created from wgpu adapter * sort out build flags for webgpu & document building webviewer * introduce shader text replacement workarounds to workaround current chrome issue * latest egui master * typo fix * doc fixes, use if cfg! instead of attribute cfg * move backend to rerun * If there's a `{{ pr-build-summary }}` in the PR description, update it. (#1971) * If there's a `{{ pr-build-summary }}` in the PR description, update it. * Add comment to the PR template * Add pull-requests permission to pr_summary job * Run the cube notebook on PR (#1972) * Run the cube notebook on PR * Add notebooks to the build summary * Use the new concurrency model * reformat py files * reformat * fix pylint errors * Add ability to manually run a web build to upload to an adhoc name (#1966) * Add ability to manually run a web build to upload to an adhoc name * Pass through ADHOC_NAME * Add a concurrency criteria for the new adhoc job * Make input description more explicit * change entity paths * merged wip albedo colormap into latest depth_cloud * remove pointclouds generated in sdk * fix compiler error, trying to compile frame.close for wasm * Default to albedo texture for depth cloud, added support for mono albedo textures * restart backend on failure, added oak_cam.device.close seems to really close the cam * py lint fix * don't run notebooks * remove run-notebook dependency --------- Co-authored-by: Clement Rey Co-authored-by: benjamin de charmoy Co-authored-by: Andreas Reich Co-authored-by: Emil Ernerfeldt Co-authored-by: Pablo Vela Co-authored-by: Nikolaus West Co-authored-by: Jeremy Leibs Co-authored-by: h3mosphere <129932586+h3mosphere@users.noreply.github.com> Co-authored-by: Urho Laukkarinen Co-authored-by: Nikolaus West * Fix rerun lints * rename binary, always start with memory-limit, pin depthai dependencies, fix errors after changing dependency versions * remove compiler warnings * fix memory leak when app is in background * Custom viewport UI (#4) * custom blueprint panel, show logical space views for depthai-viewer users, added settings clog on top of space view tab to configure what is visible * small fixes, clear entity_paths every time to avoid displaying an unavailable entity_path in space view options ui * custom left panel to add or remove space view instances, created a new default viewport layout. Improved behaviour when a panel re appears after user selected to hide it, then if stream stops and starts again the panel will be spawned back in correctly. * improve auto layout to not split when only 3D or 2D view is available * MJPEG encode image frames if connected to a PoE device. Only add magnetometer to imu sensors list if the device has a BNO IMU. Lower the memory limit to 100MB * Styling (#6) * initial styling impl * make buttons that should be small, small * Runtime depth config and fix device selection ui * comment * Proper runtime depth config updates * switch to Yolo v8 * add comma to label for non open zoo models * split 2d + 3d cam view vertically instead of horizontally * Tabify all panels (except for blueprint) (#7) * initial styling impl * make buttons that should be small, small * Runtime depth config and fix device selection ui * comment * Proper runtime depth config updates * switch to Yolo v8 * add comma to label for non open zoo models * split 2d + 3d cam view vertically instead of horizontally * Make the UI more configurable by converting the right panel into tabs. TODO: UX while laying out the panels. When a new space view appears only update the viewport layout, try to keep the user configured fixed function panels as they were. Just handle it in a way that is intuitive * remove bottom panel, switch to RAW imu sensors * XLink statistics initial implementation * initial xlink throughput statistics impllementation, have to glow it up a bit and maybe clean up the code * Xlink and rerun rename (#9) * Rename rerun py library to depthai_viewer * bug fixes and started working on a smart auto layout that operates on an existing tree, to preserve ui as much as possible, while also creating good layouts * auto layout * Fix maximize not working and add maximize for Stats tab * delete profiling stuff that shouldn't have been commited * mostly fix smart layout, TODO: detect when you can group mono and color 3d + 2d views into a 4 way split * add docstring for update_tree * WIP auto layout can_create_mono_quad checker, not at all finished yet * pass lint checks and bugfixes * forgot to sort imports * Fix mypy lint (TODO: Proper typing, especially in the comms from back to store to ws) other types are pretty solid * pylints and fix Queue typehinting * fix doc build * try to pass pylints with py.typed * ignore misc mypy errors * forgot to run black formatter * switch back to old ci * sync mono camera settings * fix web build * Update scripts/version_util.py --------- Co-authored-by: Clement Rey Co-authored-by: benjamin de charmoy Co-authored-by: Andreas Reich Co-authored-by: Emil Ernerfeldt Co-authored-by: Pablo Vela Co-authored-by: Nikolaus West Co-authored-by: Jeremy Leibs Co-authored-by: h3mosphere <129932586+h3mosphere@users.noreply.github.com> Co-authored-by: Urho Laukkarinen Co-authored-by: Nikolaus West * install depthai_viewer instead of rerun-sdk for pytest * Updated design and improvements * get rid of rerun_sdk and rerun_bindings, now just depthai_viewer and depthai_viewer_bindings get installed, so you don't clash with an existing rerun install * hopefuly fix ci * hopefuly fix ci * a bunch of fixes and improvements * disable smart layout updater for now * bugfixes * Update README.md * Patches * WIP moving away from relying on board sockets. This will enable us to support all devices! * fix merge error * LR fixes, and other stuff * a few fixes, viewer seems pretty stable on any device * auto layout * Make blueprint work * bigfixes for adding and removing space views, ai model board socket change bugfix. * bump version, fixes, Install deps in venv and use the deps from venv instead of global. * Fix for windows (and linux) * fixes * Changelog * ci checks * clear by cutoff only when not targeting wasm * Use sysconfig instead of site for getting site-packages, change app icon * No more (ignored) exceptions on startup * Fix pylints, bump depthai_sdk to latest develop * disable depth if no legit stereo pairs are available * Bump version * remove unused variable * Fix duplicate resolutions, improve first startup virtual environment creation * Fix xlink plots a bit * disable playback commands * focus on camera by default * Changelog * bump version * Improved first time startup stability * Prioritize camera images over depth image when creating albedo texture * Changed logos and get depthai_sdk from the artifactory * changelog * forgot to run pylints * Improved tensor logging performance * Don't show 2D image if the 3d view has a pointcloud * fix memory leak, loose a bit of performance because of tobytes * NV12 support! (#18) * WIP: Support sending encoded images to reduce data transfer sizes. * Support logging of NV12 encoded images, support for mapping depth_cloud onto NV12 image, todo: Fix tooltip tensor view... * Almost correct display of NV12 in every view, tooltip is still incorrect * Fixed picking! Tensor::get_nv12_pixel is wrong - todo along with correct NV12 -> RGB decoding coefficients in the shader * Fully working NV12 decoding in the shader! * Backend refactor * Better layout * Nicer behavior on device select * Fix Depth and Image spawning in seperate tabs * refactor, use sdk create_queue to get better performance * add sdk dependency * moved back to callbacks * sync cameras and depth, decode jpeg on the backend using turbojpeg (still slow) * prerelease build to test on multiple platforms * rerun lints * mypy lints * mypy lints * imports * match alpha and beta version tags * remove openh264 for now * - Fixed mono depth cloud albedo textures sampling. - improved viewport behaviour. - No more detections in depth 3d views where it just looks weird * fixes * ISP scaled resolutions, better UX, handled depthai errors * Delayed drag values, start filling in the readme * Nicer auto layout. Split the depth image and (aligned) image into two seperate space views that are side by side * Remove point depth picking outline for now * Warnings for USB2 and PoE modes * Removed turbo jpeg dependency, fill in changelog * mypy lint --------- Co-authored-by: Clement Rey Co-authored-by: benjamin de charmoy Co-authored-by: Andreas Reich Co-authored-by: Emil Ernerfeldt Co-authored-by: Pablo Vela Co-authored-by: Nikolaus West Co-authored-by: Jeremy Leibs Co-authored-by: h3mosphere <129932586+h3mosphere@users.noreply.github.com> Co-authored-by: Urho Laukkarinen Co-authored-by: Nikolaus West --- CHANGELOG.md | 11 + Cargo.lock | 71 +-- Cargo.toml | 54 +- README.md | 86 +-- crates/re_renderer/shader/depth_cloud.wgsl | 41 +- .../re_renderer/src/renderer/depth_cloud.rs | 119 ++--- crates/re_ui/Cargo.toml | 2 + crates/re_ui/src/lib.rs | 83 ++- crates/re_viewer/src/depthai/depthai.rs | 179 ++++--- crates/re_viewer/src/depthai/ws.rs | 105 ++-- crates/re_viewer/src/ui/auto_layout.rs | 105 +++- crates/re_viewer/src/ui/data_blueprint.rs | 5 + .../re_viewer/src/ui/device_settings_panel.rs | 496 ++++++++++-------- crates/re_viewer/src/ui/selection_panel.rs | 3 +- crates/re_viewer/src/ui/space_view.rs | 6 + .../re_viewer/src/ui/space_view_heuristics.rs | 131 ++++- crates/re_viewer/src/ui/view_spatial/ui.rs | 5 + crates/re_viewer/src/ui/viewport.rs | 154 ++++-- .../depthai_viewer/_backend/config_api.py | 6 +- rerun_py/depthai_viewer/_backend/device.py | 194 ++++++- .../_backend/device_configuration.py | 53 +- rerun_py/depthai_viewer/_backend/main.py | 23 +- rerun_py/depthai_viewer/_backend/messages.py | 22 +- .../depthai_viewer/_backend/packet_handler.py | 38 +- rerun_py/depthai_viewer/_backend/store.py | 11 + rerun_py/depthai_viewer/requirements.txt | 1 - 26 files changed, 1299 insertions(+), 705 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56cdd155abba..7b82d74d3550 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Depthai Viewer changelog +## 0.1.0 +Depthai Viewer official Beta release on pypi! + +- Performance improvements +- Better Auto Layouts +- Tweaks to UI for a better UX + +## 0.0.8-alpha.0 + +- Pre-release + ## 0.0.7 - Install depthai_sdk from artifactory diff --git a/Cargo.lock b/Cargo.lock index 32f70da3af73..d5a7410a78b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,7 +124,7 @@ checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "api_demo" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "anyhow", "clap 4.1.4", @@ -1277,7 +1277,7 @@ dependencies = [ [[package]] name = "depthai-viewer" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "anyhow", "backtrace", @@ -1392,7 +1392,7 @@ dependencies = [ [[package]] name = "dna" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "depthai-viewer", "itertools", @@ -2832,7 +2832,7 @@ dependencies = [ [[package]] name = "minimal" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "depthai-viewer", ] @@ -2845,7 +2845,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "minimal_options" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "anyhow", "clap 4.1.4", @@ -3263,7 +3263,7 @@ dependencies = [ [[package]] name = "objectron" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "anyhow", "clap 4.1.4", @@ -3881,7 +3881,7 @@ checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9" [[package]] name = "raw_mesh" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "anyhow", "bytes", @@ -3921,7 +3921,7 @@ dependencies = [ [[package]] name = "re_analytics" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "anyhow", "crossbeam", @@ -3942,7 +3942,7 @@ dependencies = [ [[package]] name = "re_arrow_store" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "ahash 0.8.2", "anyhow", @@ -3969,7 +3969,7 @@ dependencies = [ [[package]] name = "re_build_build_info" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "anyhow", "time 0.3.20", @@ -3977,18 +3977,18 @@ dependencies = [ [[package]] name = "re_build_info" -version = "0.0.8-alpha.0" +version = "0.1.0" [[package]] name = "re_build_web_viewer" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "cargo_metadata", ] [[package]] name = "re_data_store" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "ahash 0.8.2", "criterion", @@ -4011,14 +4011,14 @@ dependencies = [ [[package]] name = "re_error" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "anyhow", ] [[package]] name = "re_format" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "arrow2", "arrow2_convert", @@ -4028,7 +4028,7 @@ dependencies = [ [[package]] name = "re_int_histogram" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "criterion", "insta", @@ -4039,7 +4039,7 @@ dependencies = [ [[package]] name = "re_log" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "env_logger", "js-sys", @@ -4052,7 +4052,7 @@ dependencies = [ [[package]] name = "re_log_encoding" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "criterion", "ehttp", @@ -4077,7 +4077,7 @@ dependencies = [ [[package]] name = "re_log_types" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "ahash 0.8.2", "array-init", @@ -4116,7 +4116,7 @@ dependencies = [ [[package]] name = "re_memory" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "ahash 0.8.2", "backtrace", @@ -4136,7 +4136,7 @@ dependencies = [ [[package]] name = "re_query" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "arrow2", "criterion", @@ -4154,7 +4154,7 @@ dependencies = [ [[package]] name = "re_renderer" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "ahash 0.8.2", "anyhow", @@ -4207,7 +4207,7 @@ dependencies = [ [[package]] name = "re_sdk" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "arrow2_convert", "document-features", @@ -4227,7 +4227,7 @@ dependencies = [ [[package]] name = "re_sdk_comms" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "ahash 0.8.2", "anyhow", @@ -4243,7 +4243,7 @@ dependencies = [ [[package]] name = "re_smart_channel" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "crossbeam", "instant", @@ -4251,7 +4251,7 @@ dependencies = [ [[package]] name = "re_string_interner" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "ahash 0.8.2", "nohash-hasher", @@ -4262,7 +4262,7 @@ dependencies = [ [[package]] name = "re_tensor_ops" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "ahash 0.8.2", "ndarray", @@ -4272,7 +4272,7 @@ dependencies = [ [[package]] name = "re_tuid" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "arrow2", "arrow2_convert", @@ -4286,13 +4286,14 @@ dependencies = [ [[package]] name = "re_ui" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "eframe", "egui", "egui_dock", "egui_extras", "image", + "instant", "parking_lot 0.12.1", "re_log", "serde", @@ -4304,7 +4305,7 @@ dependencies = [ [[package]] name = "re_viewer" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "ahash 0.8.2", "anyhow", @@ -4375,7 +4376,7 @@ dependencies = [ [[package]] name = "re_web_viewer_server" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "cargo_metadata", "ctrlc", @@ -4392,7 +4393,7 @@ dependencies = [ [[package]] name = "re_ws_comms" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "anyhow", "bincode", @@ -4464,7 +4465,7 @@ checksum = "216080ab382b992234dda86873c18d4c48358f5cfcb70fd693d7f6f2131b628b" [[package]] name = "rerun_py" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "arrow2", "depthai-viewer", @@ -4568,7 +4569,7 @@ dependencies = [ [[package]] name = "run_wasm" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "cargo-run-wasm", "pico-args", @@ -5122,7 +5123,7 @@ dependencies = [ [[package]] name = "test_image_memory" -version = "0.0.8-alpha.0" +version = "0.1.0" dependencies = [ "depthai-viewer", "mimalloc", diff --git a/Cargo.toml b/Cargo.toml index 5e1442a43535..46bd58949df0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,39 +16,39 @@ include = ["../../LICENSE-APACHE", "../../LICENSE-MIT", "**/*.rs", "Cargo.toml"] license = "MIT OR Apache-2.0" repository = "https://github.com/rerun-io/rerun" rust-version = "1.67" -version = "0.0.8-alpha.0" +version = "0.1.0" [workspace.dependencies] # When using alpha-release, always use exact version, e.g. `version = "=0.x.y-alpha.z" # This is because we treat alpha-releases as incompatible, but semver doesn't. # In particular: if we compile rerun 0.3.0-alpha.0 we only want it to use # re_log_types 0.3.0-alpha.0, NOT 0.3.0-alpha.4 even though it is newer and semver-compatible. -re_sdk_comms = { path = "crates/re_sdk_comms", version = "0.0.8-alpha.0" } -re_analytics = { path = "crates/re_analytics", version = "0.0.8-alpha.0" } -re_arrow_store = { path = "crates/re_arrow_store", version = "0.0.8-alpha.0" } -re_build_build_info = { path = "crates/re_build_build_info", version = "0.0.8-alpha.0" } -re_build_info = { path = "crates/re_build_info", version = "0.0.8-alpha.0" } -re_build_web_viewer = { path = "crates/re_build_web_viewer", version = "0.0.8-alpha.0" } -re_data_store = { path = "crates/re_data_store", version = "0.0.8-alpha.0" } -re_error = { path = "crates/re_error", version = "0.0.8-alpha.0" } -re_format = { path = "crates/re_format", version = "0.0.8-alpha.0" } -re_int_histogram = { path = "crates/re_int_histogram", version = "0.0.8-alpha.0" } -re_log = { path = "crates/re_log", version = "0.0.8-alpha.0" } -re_log_encoding = { path = "crates/re_log_encoding", version = "0.0.8-alpha.0" } -re_log_types = { path = "crates/re_log_types", version = "0.0.8-alpha.0" } -re_memory = { path = "crates/re_memory", version = "0.0.8-alpha.0" } -re_query = { path = "crates/re_query", version = "0.0.8-alpha.0" } -re_renderer = { path = "crates/re_renderer", version = "0.0.8-alpha.0", default-features = false } -re_sdk = { path = "crates/re_sdk", version = "0.0.8-alpha.0" } -re_smart_channel = { path = "crates/re_smart_channel", version = "0.0.8-alpha.0" } -re_string_interner = { path = "crates/re_string_interner", version = "0.0.8-alpha.0" } -re_tensor_ops = { path = "crates/re_tensor_ops", version = "0.0.8-alpha.0" } -re_tuid = { path = "crates/re_tuid", version = "0.0.8-alpha.0" } -re_ui = { path = "crates/re_ui", version = "0.0.8-alpha.0" } -re_viewer = { path = "crates/re_viewer", version = "0.0.8-alpha.0", default-features = false } -re_web_viewer_server = { path = "crates/re_web_viewer_server", version = "0.0.8-alpha.0" } -re_ws_comms = { path = "crates/re_ws_comms", version = "0.0.8-alpha.0" } -depthai-viewer = { path = "crates/rerun", version = "0.0.8-alpha.0" } +re_sdk_comms = { path = "crates/re_sdk_comms", version = "0.1.0" } +re_analytics = { path = "crates/re_analytics", version = "0.1.0" } +re_arrow_store = { path = "crates/re_arrow_store", version = "0.1.0" } +re_build_build_info = { path = "crates/re_build_build_info", version = "0.1.0" } +re_build_info = { path = "crates/re_build_info", version = "0.1.0" } +re_build_web_viewer = { path = "crates/re_build_web_viewer", version = "0.1.0" } +re_data_store = { path = "crates/re_data_store", version = "0.1.0" } +re_error = { path = "crates/re_error", version = "0.1.0" } +re_format = { path = "crates/re_format", version = "0.1.0" } +re_int_histogram = { path = "crates/re_int_histogram", version = "0.1.0" } +re_log = { path = "crates/re_log", version = "0.1.0" } +re_log_encoding = { path = "crates/re_log_encoding", version = "0.1.0" } +re_log_types = { path = "crates/re_log_types", version = "0.1.0" } +re_memory = { path = "crates/re_memory", version = "0.1.0" } +re_query = { path = "crates/re_query", version = "0.1.0" } +re_renderer = { path = "crates/re_renderer", version = "0.1.0", default-features = false } +re_sdk = { path = "crates/re_sdk", version = "0.1.0" } +re_smart_channel = { path = "crates/re_smart_channel", version = "0.1.0" } +re_string_interner = { path = "crates/re_string_interner", version = "0.1.0" } +re_tensor_ops = { path = "crates/re_tensor_ops", version = "0.1.0" } +re_tuid = { path = "crates/re_tuid", version = "0.1.0" } +re_ui = { path = "crates/re_ui", version = "0.1.0" } +re_viewer = { path = "crates/re_viewer", version = "0.1.0", default-features = false } +re_web_viewer_server = { path = "crates/re_web_viewer_server", version = "0.1.0" } +re_ws_comms = { path = "crates/re_ws_comms", version = "0.1.0" } +depthai-viewer = { path = "crates/rerun", version = "0.1.0" } ahash = "0.8" anyhow = "1.0" diff --git a/README.md b/README.md index 69788eb69ebd..cc91f9f16113 100644 --- a/README.md +++ b/README.md @@ -6,26 +6,13 @@

PyPi - crates.io + MIT Apache - Rerun Discord +

-# Rerun: Visualization infrastructure for computer vision. - -Use one of our logging APIs (Python or Rust) to log rich data, such as images and point clouds, to the Rerun Viewer, where it is visualized live or after the fact. - -```py -import rerun as rr - -viewer.init("my_app", spawn = True) # Spawn a Rerun Viewer and stream log events to it - -viewer.log_image("rgb_image", image) -viewer.log_points("points", positions) -viewer.log_rect("car", bbox) -… -``` +# Depthai Viewer: The visualization tool for DepthAi

Rerun Viewer @@ -33,61 +20,38 @@ viewer.log_rect("car", bbox) ## Getting started -- **Python**: `pip install depthai-viewer` -- **Rust**: `cargo add rerun` -- **C / C++**: Coming soon +### Prerequisites -### Rerun Viewer binary +- A working version of Python>=3.8 +- Libjpeg turbo: + - MacOS: `brew install jpeg-turbo` + - Ubuntu: `sudo apt install libturbojpeg` + - Windows: [libjpeg-turbo official installer](https://sourceforge.net/projects/libjpeg-turbo/files/) -Both the Python and Rust library can start the Rerun Viewer, but to stream log data over the network or load our `.rrd` data files you also need the `rerun` binary. +### Install -It can be installed with `pip install depthai-viewer` or with `cargo install rerun`. +```sh +# ---------- Linux / MacOS ---------- +python3 -m pip install depthai-viewer +# ------------- Windows ------------- +python -m pip install depthai-viewer +``` +### Run +```sh +python3 -m depthai_viewer +# -------- OR --------- +depthai-viewer +``` -You should now be able to run `rerun --help` in any terminal. -### Documentation +### Documentation +Depthai Viewer can be used as a visualization tool, just like [rerun](https://rerun.io). It uses largely the same python logging api, so you can reefer to the relevant rerun documentation: - 📚 [High-level docs](http://rerun.io/docs) - ⚙️ [Examples](examples) - 🐍 [Python API docs](https://ref.rerun.io/docs/python) -- 🦀 [Rust API docs](https://docs.rs/rerun/) - ⁉️ [Troubleshooting](https://www.rerun.io/docs/getting-started/troubleshooting) ## Status -We are in early beta. -There are many features we want to add, and the API is still evolving. -_Expect breaking changes!_ - -Some shortcomings: - -- Big points clouds (1M+) are slow ([#1136](https://github.com/rerun-io/rerun/issues/1136)) -- The data you want to visualize must fit in RAM. - - See for how to bound memory use - - We plan on having a disk-based data store some time in the future -- The Rust library takes a long time to compile - - We have way too many big dependencies, and we are planning on improving the situation ([#1316](https://github.com/rerun-io/rerun/pull/1316)) - -## Business model - -Rerun uses an open-core model. Everything in this repository will stay open source and free (both as in beer and as in freedom). -In the future, Rerun will offer a commercial product that builds on top of the core free project. - -The Rerun open source project targets the needs of individual developers. -The commercial product targets the needs specific to teams that build and run computer vision and robotics products. - -# Development - -- [`ARCHITECTURE.md`](ARCHITECTURE.md) -- [`BUILD.md`](BUILD.md) -- [`rerun_py/README.md`](rerun_py/README.md) - build instructions for Python SDK -- [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md) -- [`CODE_STYLE.md`](CODE_STYLE.md) -- [`CONTRIBUTING.md`](CONTRIBUTING.md) -- [`RELEASES.md`](RELEASES.md) - -## Installing a pre-release Python SDK - -1. Download the correct `.whl` from [GitHub Releases](https://github.com/rerun-io/rerun/releases) -2. Run `pip install rerun_sdk<...>.whl` (replace `<...>` with the actual filename) -3. Test it: `rerun --version` +We are in early beta, however any DepthAi capable device should work with the viewer. If it doesn't, feel free to open an [issue](https://github.com/luxonis/depthai-viewer/issues). diff --git a/crates/re_renderer/shader/depth_cloud.wgsl b/crates/re_renderer/shader/depth_cloud.wgsl index fbd49c85bb4f..407197a6f353 100644 --- a/crates/re_renderer/shader/depth_cloud.wgsl +++ b/crates/re_renderer/shader/depth_cloud.wgsl @@ -146,19 +146,39 @@ fn compute_point_data(quad_idx: u32) -> PointData { if depth_cloud_info.albedo_sample_type == SAMPLE_TYPE_NV12 { color = decode_nv12(albedo_texture_uint, Vec2(f32(texcoords.x), f32(texcoords.y)) / Vec2(f32(wh.x), f32(wh.x))); } else { // TODO(filip): Support all sample types like in rectangle_fs.wgsl - if depth_cloud_info.depth_sample_type == SAMPLE_TYPE_FLOAT_FILTER { + if depth_cloud_info.albedo_sample_type == SAMPLE_TYPE_FLOAT_FILTER { color = textureSampleLevel( - depth_texture_float, + albedo_texture_float_filterable, trilinear_sampler, - Vec2(texcoords) / Vec2(textureDimensions(depth_texture_float)), + Vec2(texcoords) / Vec2(wh), 0.0 ); - } else { + color = Vec4(Vec3(color.r), 1.0); + } else if depth_cloud_info.albedo_sample_type == SAMPLE_TYPE_FLOAT_NOFILTER { + let coord = Vec2(texcoords) / Vec2(textureDimensions(depth_texture_float)); + let v00 = textureLoad(albedo_texture_float_nofilter, IVec2(coord) + IVec2(0, 0), 0); + let v01 = textureLoad(albedo_texture_float_nofilter, IVec2(coord) + IVec2(0, 1), 0); + let v10 = textureLoad(albedo_texture_float_nofilter, IVec2(coord) + IVec2(1, 0), 0); + let v11 = textureLoad(albedo_texture_float_nofilter, IVec2(coord) + IVec2(1, 1), 0); + let top = mix(v00, v10, fract(coord.x)); + let bottom = mix(v01, v11, fract(coord.x)); + color = mix(top, bottom, fract(coord.y)); + } else if depth_cloud_info.albedo_sample_type == SAMPLE_TYPE_UINT_NOFILTER { + color = Vec4(textureLoad( + albedo_texture_uint, + texcoords, + 0 + )) / 255.0; + color = Vec4(linear_from_srgb(Vec3(color.r)), 1.0); + } else if depth_cloud_info.albedo_sample_type == SAMPLE_TYPE_SINT_NOFILTER { color = Vec4(textureLoad( - depth_texture_uint, + albedo_texture_sint, texcoords, 0 )) / 255.0; + color = Vec4(linear_from_srgb(Vec3(color.r)), 1.0); + } else { + color = ERROR_RGBA; } } } else { @@ -236,12 +256,13 @@ fn fs_main_picking_layer(in: VertexOut) -> @location(0) UVec4 { @fragment fn fs_main_outline_mask(in: VertexOut) -> @location(0) UVec2 { + discard; // TODO(filip): This outline looks really bad... It would be neat to have it tho - implement better // Output is an integer target, can't use coverage therefore. // But we still want to discard fragments where coverage is low. // Since the outline extends a bit, a very low cut off tends to look better. - let coverage = sphere_quad_coverage(in.pos_in_world, in.point_radius, in.point_pos_in_world); - if coverage < 1.0 { - discard; - } - return depth_cloud_info.outline_mask_id; + // let coverage = sphere_quad_coverage(in.pos_in_world, in.point_radius, in.point_pos_in_world); + // if coverage < 1.0 { + // discard; + // } + // return depth_cloud_info.outline_mask_id; } diff --git a/crates/re_renderer/src/renderer/depth_cloud.rs b/crates/re_renderer/src/renderer/depth_cloud.rs index 3d2bce3114f6..f84b92b4b049 100644 --- a/crates/re_renderer/src/renderer/depth_cloud.rs +++ b/crates/re_renderer/src/renderer/depth_cloud.rs @@ -18,7 +18,8 @@ use crate::{ allocator::create_and_fill_uniform_buffer_batch, draw_phases::{ DrawPhase, OutlineMaskProcessor }, include_shader_module, - resource_managers::{ GpuTexture2D, ResourceManagerError }, + resource_managers::{GpuTexture2D, ResourceManagerError}, + texture_info, view_builder::ViewBuilder, wgpu_resources::{ BindGroupDesc, @@ -36,27 +37,20 @@ use crate::{ OutlineMaskPreference, PickingLayerObjectId, PickingLayerProcessor, - texture_info, }; use super::{ - DrawData, - FileResolver, - FileSystem, - RenderContext, - Renderer, - SharedRendererData, - WgpuResourcePools, - ColormappedTexture, + ColormappedTexture, DrawData, FileResolver, FileSystem, RenderContext, Renderer, + SharedRendererData, WgpuResourcePools, }; // --- mod gpu_data { use crate::{ - wgpu_buffer_types::{ self, U32RowPadded }, - PickingLayerObjectId, renderer::TextureEncoding, texture_info, + wgpu_buffer_types::{self, U32RowPadded}, + PickingLayerObjectId, }; // Keep in sync with mirror in depth_cloud.wgsl @@ -111,7 +105,7 @@ mod gpu_data { pub fn from_depth_cloud( radius_boost_in_ui_points: f32, depth_cloud: &super::DepthCloud, - device_features: wgpu::Features + device_features: wgpu::Features, ) -> Self { let super::DepthCloud { world_from_obj, @@ -130,43 +124,33 @@ mod gpu_data { let albedo_sample_type = match albedo_texture { Some(colormapped_texture) => { match colormapped_texture.texture.format().sample_type(None) { - Some(wgpu::TextureSampleType::Float { .. }) => - match colormapped_texture.encoding { - Some(TextureEncoding::Nv12) => SAMPLE_TYPE_NV12, - _ => { - if - texture_info::is_float_filterable( - colormapped_texture.texture.format(), - device_features - ) - { - SAMPLE_TYPE_FLOAT_FILTER - } else { - SAMPLE_TYPE_FLOAT_NOFILTER - } - } - } - Some(wgpu::TextureSampleType::Uint) => { - match colormapped_texture.encoding { - Some(TextureEncoding::Nv12) => SAMPLE_TYPE_NV12, - _ => SAMPLE_TYPE_UINT_NOFILTER, + Some(wgpu::TextureSampleType::Float { .. }) => { + if texture_info::is_float_filterable( + colormapped_texture.texture.format(), + device_features, + ) { + SAMPLE_TYPE_FLOAT_FILTER + } else { + SAMPLE_TYPE_FLOAT_NOFILTER } } + Some(wgpu::TextureSampleType::Uint) => match colormapped_texture.encoding { + Some(TextureEncoding::Nv12) => SAMPLE_TYPE_NV12, + _ => SAMPLE_TYPE_UINT_NOFILTER, + }, Some(wgpu::TextureSampleType::Sint) => SAMPLE_TYPE_SINT_NOFILTER, _ => 0, } } - _ => { 0 } + _ => 0, }; let depth_sample_type = match depth_texture.texture.format().sample_type(None) { Some(wgpu::TextureSampleType::Float { .. }) => { - if - texture_info::is_float_filterable( - depth_texture.texture.format(), - device_features - ) - { + if texture_info::is_float_filterable( + depth_texture.texture.format(), + device_features, + ) { SAMPLE_TYPE_FLOAT_FILTER } else { SAMPLE_TYPE_FLOAT_NOFILTER @@ -291,9 +275,11 @@ pub enum DepthCloudDrawDataError { "Depth texture format was {0:?}, only formats with sample type float are supported" )] InvalidDepthTextureFormat(wgpu::TextureFormat), - #[error("Invalid albedo texture format {0:?}")] InvalidAlbedoTextureFormat(wgpu::TextureFormat), + #[error("Invalid albedo texture format {0:?}")] + InvalidAlbedoTextureFormat(wgpu::TextureFormat), - #[error(transparent)] ResourceManagerError(#[from] ResourceManagerError), + #[error(transparent)] + ResourceManagerError(#[from] ResourceManagerError), } impl DepthCloudDrawData { @@ -324,24 +310,20 @@ impl DepthCloudDrawData { let depth_cloud_ubo_binding_outlines = create_and_fill_uniform_buffer_batch( ctx, "depth_cloud_ubos".into(), - depth_clouds - .iter() - .map(|dc| { - gpu_data::DepthCloudInfoUBO::from_depth_cloud( - *radius_boost_in_ui_points_for_outlines, - dc, - ctx.device.features() - ) - }) + depth_clouds.iter().map(|dc| { + gpu_data::DepthCloudInfoUBO::from_depth_cloud( + *radius_boost_in_ui_points_for_outlines, + dc, + ctx.device.features(), + ) + }), ); let depth_cloud_ubo_binding_opaque = create_and_fill_uniform_buffer_batch( ctx, "depth_cloud_ubos".into(), - depth_clouds - .iter() - .map(|dc| - gpu_data::DepthCloudInfoUBO::from_depth_cloud(0.0, dc, ctx.device.features()) - ) + depth_clouds.iter().map(|dc| { + gpu_data::DepthCloudInfoUBO::from_depth_cloud(0.0, dc, ctx.device.features()) + }), ); let mut instances = Vec::with_capacity(depth_clouds.len()); @@ -370,20 +352,21 @@ impl DepthCloudDrawData { depth_texture_uint = depth_texture.handle; } _ => { - return Err( - DepthCloudDrawDataError::InvalidDepthTextureFormat(depth_texture_format) - ); + return Err(DepthCloudDrawDataError::InvalidDepthTextureFormat( + depth_texture_format, + )); } } - if - let Some(albedo_texture) = depth_cloud.albedo_texture - .as_ref() - .and_then(|t| Some(&t.texture)) + if let Some(albedo_texture) = depth_cloud + .albedo_texture + .as_ref() + .and_then(|t| Some(&t.texture)) { let texture_format = albedo_texture.creation_desc.format; match texture_format.sample_type(None) { Some(wgpu::TextureSampleType::Float { .. }) => { - if texture_info::is_float_filterable(texture_format, ctx.device.features()) { + if texture_info::is_float_filterable(texture_format, ctx.device.features()) + { albedo_texture_float_filterable = albedo_texture.handle; } else { albedo_texture_float_nofilter = albedo_texture.handle; @@ -396,9 +379,9 @@ impl DepthCloudDrawData { albedo_texture_uint = albedo_texture.handle; } _ => { - return Err( - DepthCloudDrawDataError::InvalidAlbedoTextureFormat(texture_format) - ); + return Err(DepthCloudDrawDataError::InvalidAlbedoTextureFormat( + texture_format, + )); } } } @@ -538,7 +521,7 @@ impl Renderer for DepthCloudRenderer { multisampled: false, }, count: None, - } + }, ], }) ); diff --git a/crates/re_ui/Cargo.toml b/crates/re_ui/Cargo.toml index b9a59e8f9c0e..e60721c903a5 100644 --- a/crates/re_ui/Cargo.toml +++ b/crates/re_ui/Cargo.toml @@ -39,6 +39,8 @@ serde_json = "1" strum = { version = "0.24", features = ["derive"] } strum_macros = "0.24" sublime_fuzzy = "0.7" +instant = { version = "0.1", features = ["wasm-bindgen"] } + ## Optional dependencies: eframe = { workspace = true, optional = true, default-features = false } egui_dock = { workspace = true, optional = true, features = ["serde"] } diff --git a/crates/re_ui/src/lib.rs b/crates/re_ui/src/lib.rs index 69324b333f85..876ca880c338 100644 --- a/crates/re_ui/src/lib.rs +++ b/crates/re_ui/src/lib.rs @@ -13,10 +13,12 @@ pub use command::Command; pub use command_palette::CommandPalette; pub use design_tokens::DesignTokens; pub use icons::Icon; +use instant::Instant; pub use static_image_cache::StaticImageCache; +use std::cell::RefCell; use std::ops::RangeInclusive; +use std::rc::Rc; pub use toggle_switch::toggle_switch; - // --------------------------------------------------------------------------- /// If true, we fill the entire window, except for the close/maximize/minimize buttons in the top-left. @@ -49,7 +51,7 @@ use std::sync::Arc; use parking_lot::Mutex; -use egui::{pos2, Align2, Color32, Mesh, NumExt, Rect, Shape, Vec2}; +use egui::{epaint::ahash::HashMap, pos2, Align2, Color32, Mesh, NumExt, Rect, Shape, Vec2}; #[derive(Clone, Copy, Debug, PartialEq)] pub enum ScrollAreaDirection { @@ -58,6 +60,13 @@ pub enum ScrollAreaDirection { Both, } +#[derive(Clone, Debug, PartialEq)] +struct DelayedDragState { + pub last_update: Instant, + pub delay_ms: f32, + pub value: i64, +} + #[derive(Clone)] pub struct ReUi { pub egui_ctx: egui::Context, @@ -66,6 +75,9 @@ pub struct ReUi { pub design_tokens: DesignTokens, pub static_image_cache: Arc>, + + /// Inner state of dragvalues where we want to delay setting the value by a bit + delayed_drag_values: Rc>>, } impl ReUi { @@ -75,6 +87,7 @@ impl ReUi { egui_ctx: egui_ctx.clone(), design_tokens: DesignTokens::load_and_apply(egui_ctx), static_image_cache: Arc::new(Mutex::new(StaticImageCache::default())), + delayed_drag_values: Default::default(), } } @@ -152,6 +165,7 @@ impl ReUi { label: &str, selected_text: String, left_to_right: bool, + wrap: bool, menu_contents: impl FnOnce(&mut egui::Ui) -> R, ) { let align = egui::Align::Center; @@ -165,17 +179,29 @@ impl ReUi { if left_to_right { ui.label(egui::RichText::new(label).color(self.design_tokens.gray_900)); } - ui.add_sized( - [Self::box_width(), Self::box_height() + 1.0], - |ui: &mut egui::Ui| { + if wrap { + ui.add_sized( + [Self::box_width(), Self::box_height() + 1.0], + |ui: &mut egui::Ui| { + egui::ComboBox::from_id_source(label) + .selected_text(selected_text) + .width(Self::box_width()) + .wrap(true) + .show_ui(ui, menu_contents) + .response + }, + ); + } else { + ui.add(|ui: &mut egui::Ui| { egui::ComboBox::from_id_source(label) .selected_text(selected_text) .width(Self::box_width()) .wrap(true) .show_ui(ui, menu_contents) .response - }, - ); + }); + } + if !left_to_right { ui.label(egui::RichText::new(label).color(self.design_tokens.gray_900)); } @@ -233,9 +259,20 @@ impl ReUi { }); } + /// Creates a labeled drag value box, where the label is to the left and the drag box is to the right. + /// + /// Arguments: + /// * `ui`: The egui ui + /// * `id`: A unique id, used to store the state of the drag value box + /// * `delay`: Optional delay in milliseconds before the value get's updated + /// * `label`: The label to the left of the drag value box + /// * `value`: Where the value is stored + /// * `range`: The range of the value pub fn labeled_dragvalue( &self, ui: &mut egui::Ui, + id: egui::Id, + delay: Option, // Delay in milliseconds before the value get's updated label: &str, value: &mut Num, range: RangeInclusive, @@ -243,15 +280,29 @@ impl ReUi { where Num: egui::emath::Numeric, { - ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { - let response = ui.add_sized( - [Self::box_width(), Self::box_height()], - egui::DragValue::new(value).clamp_range(range), - ); - ui.label(egui::RichText::new(label).color(self.design_tokens.gray_900)); - response - }) - .inner + let mut borrow_map = self.delayed_drag_values.borrow_mut(); + let state = borrow_map.entry(id).or_insert(DelayedDragState { + delay_ms: delay.unwrap_or(0.0), + last_update: instant::Instant::now(), + value: value.to_f64() as i64, // TODO(filip): The hell, I'm pretty sure there is a macro that implements to_i64 for Numeric + }); + let response = ui + .with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + let response = ui.add_sized( + [Self::box_width(), Self::box_height()], + egui::DragValue::new(&mut state.value).clamp_range(range), + ); + ui.label(egui::RichText::new(label).color(self.design_tokens.gray_900)); + response + }) + .inner; + if response.changed() { + state.last_update = instant::Instant::now(); + } + if state.delay_ms < state.last_update.elapsed().as_millis() as f32 { + *value = Num::from_f64(state.value as f64); + } + response } pub fn labeled_toggle_switch(&self, ui: &mut egui::Ui, label: &str, value: &mut bool) { diff --git a/crates/re_viewer/src/depthai/depthai.rs b/crates/re_viewer/src/depthai/depthai.rs index ed3b2b0482f5..a005973369b1 100644 --- a/crates/re_viewer/src/depthai/depthai.rs +++ b/crates/re_viewer/src/depthai/depthai.rs @@ -70,7 +70,7 @@ pub enum CameraBoardSocket { CAM_H, } -impl CameraBoardSocket { +impl CameraBoardSocket { pub fn from(socket: String) -> Option { match socket.as_str() { "AUTO" => Some(CameraBoardSocket::AUTO), @@ -120,36 +120,37 @@ pub enum ImuKind { #[allow(non_camel_case_types)] pub enum CameraSensorResolution { THE_400_P, + THE_480_P, THE_720_P, THE_800_P, + THE_5_MP, THE_1440X1080, THE_1080_P, THE_1200_P, - THE_5_MP, THE_4_K, - THE_12_MP, THE_4000X3000, + THE_12_MP, THE_13_MP, - THE_48_MP, + THE_5312X6000, } // fmt::Display is used in UI while fmt::Debug is used with the depthai backend api impl fmt::Display for CameraSensorResolution { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::THE_1080_P => write!(f, "1080p"), - Self::THE_4_K => write!(f, "4k"), + Self::THE_400_P => write!(f, "400p"), + Self::THE_480_P => write!(f, "480p"), Self::THE_720_P => write!(f, "720p"), Self::THE_800_P => write!(f, "800p"), - Self::THE_1200_P => write!(f, "1200p"), Self::THE_5_MP => write!(f, "5MP"), + Self::THE_1440X1080 => write!(f, "1440x1080"), + Self::THE_1080_P => write!(f, "1080p"), + Self::THE_1200_P => write!(f, "1200p"), + Self::THE_4_K => write!(f, "4k"), + Self::THE_4000X3000 => write!(f, "4000x3000"), Self::THE_12_MP => write!(f, "12MP"), Self::THE_13_MP => write!(f, "13MP"), - Self::THE_4000X3000 => write!(f, "4000x3000"), - Self::THE_48_MP => write!(f, "48MP"), - Self::THE_1440X1080 => write!(f, "1440x1080"), - Self::THE_400_P => write!(f, "400p"), - Self::THE_720_P => write!(f, "720p"), + Self::THE_5312X6000 => write!(f, "5312x6000"), } } } @@ -193,6 +194,7 @@ pub struct DeviceProperties { pub imu: Option, pub stereo_pairs: Vec<(CameraBoardSocket, CameraBoardSocket)>, pub default_stereo_pair: Option<(CameraBoardSocket, CameraBoardSocket)>, + pub info: DeviceInfo, } impl DeviceProperties { @@ -273,11 +275,11 @@ impl DepthConfig { } pub fn only_runtime_configs_differ(&self, other: &DepthConfig) -> bool { - self.lr_check == other.lr_check && - self.align == other.align && - self.extended_disparity == other.extended_disparity && - self.subpixel_disparity == other.subpixel_disparity && - self != other + self.lr_check == other.lr_check + && self.align == other.align + && self.extended_disparity == other.extended_disparity + && self.subpixel_disparity == other.subpixel_disparity + && self != other } } @@ -295,13 +297,12 @@ impl From<&DeviceProperties> for Option { let stereo_pair = cam_with_stereo_pair.stereo_pairs[0]; config.stereo_pair = (cam_with_stereo_pair.board_socket, stereo_pair); } - config.align = if - let Some(color_cam) = props.cameras.iter().find(|cam| cam.is_color_camera()) - { - color_cam.board_socket - } else { - config.stereo_pair.0 - }; + config.align = + if let Some(color_cam) = props.cameras.iter().find(|cam| cam.is_color_camera()) { + color_cam.board_socket + } else { + config.stereo_pair.0 + }; Some(config) } } @@ -332,18 +333,17 @@ impl From<&DeviceProperties> for DeviceConfig { let mut config = Self::default(); let has_color_cam = props.cameras.iter().any(|cam| cam.is_color_camera()); - config.cameras = props.cameras + config.cameras = props + .cameras .iter() .map(|cam| CameraConfig { name: cam.name.clone(), - fps: 30, // TODO(filip): Do performance improvements to allow higher fps - resolution: *cam.resolutions + fps: 30, + resolution: *cam + .resolutions .iter() - .filter(|res| { - res != &&CameraSensorResolution::THE_4_K && - res != &&CameraSensorResolution::THE_12_MP - }) - .last() + .as_slice() + .first() .unwrap_or(&CameraSensorResolution::THE_800_P), board_socket: cam.board_socket, stream_enabled: if has_color_cam { @@ -397,10 +397,10 @@ impl PartialEq for DeviceConfig { (Some(a), Some(b)) => a == b, _ => true, // If one is None, it's only different if depth_enabled is different }; - self.cameras == other.cameras && - depth_eq && - self.depth_enabled == other.depth_enabled && - self.ai_model == other.ai_model + self.cameras == other.cameras + && depth_eq + && self.depth_enabled == other.depth_enabled + && self.ai_model == other.ai_model } } @@ -422,10 +422,7 @@ impl fmt::Debug for DeviceConfig { write!( f, "Device config: cams: {:?}, depth: {:?}, ai_model: {:?}, depth_enabled: {:?}", - self.cameras, - self.depth, - self.ai_model, - self.depth_enabled + self.cameras, self.depth, self.ai_model, self.depth_enabled ) } } @@ -449,7 +446,7 @@ pub enum ErrorAction { FullReset, } -// TODO(filip): Move to a more appropriate place, refactor depthai.rs in general +// ---------------- TODO(filip): Move to a more appropriate place, refactor depthai.rs in general ---------------- #[derive(serde::Deserialize, serde::Serialize, Clone, PartialEq, fmt::Debug)] pub struct Error { pub action: ErrorAction, @@ -470,6 +467,13 @@ pub struct Info { pub message: String, } +#[derive(serde::Deserialize, serde::Serialize, Clone, PartialEq, fmt::Debug, Default)] +pub struct Warning { + pub message: String, +} + +// --------------------------------------------------------------------------------------------------------------- + #[derive(serde::Deserialize, serde::Serialize, Clone, fmt::Debug)] pub struct AiModel { pub path: String, @@ -479,7 +483,7 @@ pub struct AiModel { impl Default for AiModel { fn default() -> Self { - default_neural_networks()[1].clone() + default_neural_networks()[2].clone() } } @@ -511,10 +515,44 @@ impl PartialEq for AiModel { } } +#[derive(serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, fmt::Debug)] +pub enum XlinkConnection { + Usb, + PoE, +} + +impl fmt::Display for XlinkConnection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + XlinkConnection::Usb => write!(f, "USB"), + XlinkConnection::PoE => write!(f, "PoE"), + } + } +} + +impl Default for XlinkConnection { + fn default() -> Self { + XlinkConnection::Usb + } +} + +#[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, fmt::Debug, Default)] +pub struct DeviceInfo { + pub mxid: DeviceId, + pub connection: XlinkConnection, + pub name: String, +} + +impl DeviceInfo { + pub fn display_text(&self) -> String { + format!("{}: {} ({})", self.connection, self.name, self.mxid) + } +} + #[derive(serde::Serialize, serde::Deserialize)] pub struct State { #[serde(skip)] - devices_available: Option>, + devices_available: Option>, #[serde(skip)] pub selected_device: DeviceProperties, #[serde(skip)] @@ -564,7 +602,7 @@ fn default_neural_networks() -> Vec { path: String::from("age-gender-recognition-retail-0013"), display_name: String::from("Age gender recognition"), camera: CameraBoardSocket::CAM_A, - } + }, ] } @@ -587,15 +625,7 @@ impl Default for State { #[repr(u8)] #[derive( - serde::Serialize, - serde::Deserialize, - Copy, - Clone, - PartialEq, - Eq, - fmt::Debug, - Hash, - EnumIter + serde::Serialize, serde::Deserialize, Copy, Clone, PartialEq, Eq, fmt::Debug, Hash, EnumIter, )] pub enum ChannelId { ColorImage, @@ -609,21 +639,25 @@ pub enum ChannelId { impl State { pub fn only_runtime_configs_changed( old_config: &DeviceConfig, - new_config: &DeviceConfig + new_config: &DeviceConfig, ) -> bool { - let any_runtime_conf_changed = - old_config.depth.is_some() && - new_config.depth.is_some() && - old_config.depth.unwrap().only_runtime_configs_differ(&new_config.depth.unwrap()); // || others to be added - any_runtime_conf_changed && - old_config.cameras == new_config.cameras && - old_config.ai_model == new_config.ai_model + let any_runtime_conf_changed = old_config.depth.is_some() + && new_config.depth.is_some() + && old_config + .depth + .unwrap() + .only_runtime_configs_differ(&new_config.depth.unwrap()); // || others to be added + any_runtime_conf_changed + && old_config.cameras == new_config.cameras + && old_config.ai_model == new_config.ai_model } pub fn set_subscriptions(&mut self, subscriptions: &Vec) { - if - self.subscriptions.len() == subscriptions.len() && - self.subscriptions.iter().all(|channel_id| subscriptions.contains(channel_id)) + if self.subscriptions.len() == subscriptions.len() + && self + .subscriptions + .iter() + .all(|channel_id| subscriptions.contains(channel_id)) { return; } @@ -632,7 +666,7 @@ impl State { } /// Returns available devices - pub fn get_devices(&mut self) -> Vec { + pub fn get_devices(&mut self) -> Vec { // Return stored available devices or fetch them from the api (they get fetched every 30s via poller) if let Some(devices) = self.devices_available.clone() { return devices; @@ -680,10 +714,8 @@ impl State { if config.depth.is_some() { subs.push(ChannelId::DepthImage); } - if - let Some(color_camera) = &config.cameras - .iter() - .find(|cam| cam.is_color_camera()) + if let Some(color_camera) = + &config.cameras.iter().find(|cam| cam.is_color_camera()) { if color_camera.stream_enabled { subs.push(ChannelId::ColorImage); @@ -694,7 +726,8 @@ impl State { subs.push(ChannelId::LeftMono); } } - if let Some(right_cam) = &config.cameras.iter().find(|cam| cam.name == "right") { + if let Some(right_cam) = &config.cameras.iter().find(|cam| cam.name == "right") + { if right_cam.stream_enabled { subs.push(ChannelId::RightMono); } @@ -737,6 +770,12 @@ impl State { } re_log::info!("{}", info.message); } + WsMessageData::Warning(warning) => { + if warning.message.is_empty() { + return; + } + re_log::warn!("{}", warning.message); + } } } diff --git a/crates/re_viewer/src/depthai/ws.rs b/crates/re_viewer/src/depthai/ws.rs index 3a81824e7c91..15d64dba209a 100644 --- a/crates/re_viewer/src/depthai/ws.rs +++ b/crates/re_viewer/src/depthai/ws.rs @@ -1,6 +1,6 @@ use crossbeam_channel; -use ewebsock::{ WsEvent, WsMessage }; -use serde::{ Deserialize, Serialize }; +use ewebsock::{WsEvent, WsMessage}; +use serde::{Deserialize, Serialize}; use std::fmt; use std::ops::ControlFlow; use std::process::exit; @@ -13,7 +13,7 @@ async fn spawn_ws_client( recv_tx: crossbeam_channel::Sender, send_rx: crossbeam_channel::Receiver, shutdown: Arc, - connected: Arc + connected: Arc, ) { let (error_tx, error_rx) = crossbeam_channel::unbounded(); // Retry connection until successful @@ -21,37 +21,35 @@ async fn spawn_ws_client( let recv_tx = recv_tx.clone(); let error_tx = error_tx.clone(); let connected = connected.clone(); - if - let Ok(sender) = ewebsock - ::ws_connect( - String::from("ws://localhost:9001"), - Box::new(move |event| { - match event { - WsEvent::Opened => { - re_log::info!("Websocket opened"); - connected.store(true, std::sync::atomic::Ordering::SeqCst); - ControlFlow::Continue(()) - } - WsEvent::Message(message) => { - // re_log::debug!("Websocket message"); - recv_tx.send(message); - ControlFlow::Continue(()) - } - WsEvent::Error(e) => { - // re_log::info!("Websocket Error: {:?}", e); - connected.store(false, std::sync::atomic::Ordering::SeqCst); - error_tx.send(e); - ControlFlow::Break(()) - } - WsEvent::Closed => { - // re_log::info!("Websocket Closed"); - error_tx.send(String::from("Websocket Closed")); - ControlFlow::Break(()) - } - } - }) - ) - .as_mut() + if let Ok(sender) = ewebsock::ws_connect( + String::from("ws://localhost:9001"), + Box::new(move |event| { + match event { + WsEvent::Opened => { + re_log::info!("Websocket opened"); + connected.store(true, std::sync::atomic::Ordering::SeqCst); + ControlFlow::Continue(()) + } + WsEvent::Message(message) => { + // re_log::debug!("Websocket message"); + recv_tx.send(message); + ControlFlow::Continue(()) + } + WsEvent::Error(e) => { + // re_log::info!("Websocket Error: {:?}", e); + connected.store(false, std::sync::atomic::Ordering::SeqCst); + error_tx.send(e); + ControlFlow::Break(()) + } + WsEvent::Closed => { + // re_log::info!("Websocket Closed"); + error_tx.send(String::from("Websocket Closed")); + ControlFlow::Break(()) + } + } + }), + ) + .as_mut() { while error_rx.is_empty() { if shutdown.load(std::sync::atomic::Ordering::SeqCst) { @@ -82,11 +80,12 @@ type RuntimeOnly = bool; #[derive(Serialize, Deserialize, fmt::Debug)] pub enum WsMessageData { Subscriptions(Vec), - Devices(Vec), + Devices(Vec), DeviceProperties(depthai::DeviceProperties), Pipeline((depthai::DeviceConfig, RuntimeOnly)), Error(depthai::Error), Info(depthai::Info), + Warning(depthai::Warning), } #[derive(Deserialize, Serialize, fmt::Debug)] @@ -97,6 +96,7 @@ pub enum WsMessageType { Pipeline, Error, Info, + Warning, } impl Default for WsMessageType { @@ -114,7 +114,10 @@ pub struct BackWsMessage { } impl<'de> Deserialize<'de> for BackWsMessage { - fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de> { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { #[derive(Deserialize)] pub struct Message { #[serde(rename = "type")] @@ -125,10 +128,9 @@ impl<'de> Deserialize<'de> for BackWsMessage { let message = Message::deserialize(deserializer)?; let data = match message.kind { - WsMessageType::Subscriptions => - WsMessageData::Subscriptions( - serde_json::from_value(message.data).unwrap_or_default() - ), + WsMessageType::Subscriptions => WsMessageData::Subscriptions( + serde_json::from_value(message.data).unwrap_or_default(), + ), WsMessageType::Devices => { WsMessageData::Devices(serde_json::from_value(message.data).unwrap_or_default()) } @@ -144,6 +146,9 @@ impl<'de> Deserialize<'de> for BackWsMessage { WsMessageType::Info => { WsMessageData::Info(serde_json::from_value(message.data).unwrap_or_default()) } + WsMessageType::Warning => { + WsMessageData::Warning(serde_json::from_value(message.data).unwrap_or_default()) + } }; Ok(Self { @@ -190,14 +195,23 @@ impl WebSocket { let task; if let Ok(handle) = tokio::runtime::Handle::try_current() { re_log::debug!("Using current tokio runtime"); - task = handle.spawn(spawn_ws_client(recv_tx, send_rx, shutdown_clone, connected_clone)); + task = handle.spawn(spawn_ws_client( + recv_tx, + send_rx, + shutdown_clone, + connected_clone, + )); } else { re_log::debug!("Creating new tokio runtime"); - task = tokio::runtime::Builder - ::new_current_thread() + task = tokio::runtime::Builder::new_current_thread() .build() .unwrap() - .spawn(spawn_ws_client(recv_tx, send_rx, shutdown_clone, connected_clone)); + .spawn(spawn_ws_client( + recv_tx, + send_rx, + shutdown_clone, + connected_clone, + )); } Self { receiver: recv_rx, @@ -213,7 +227,8 @@ impl WebSocket { } pub fn shutdown(&mut self) { - self.shutdown.store(true, std::sync::atomic::Ordering::SeqCst); + self.shutdown + .store(true, std::sync::atomic::Ordering::SeqCst); } pub fn receive(&self) -> Option { diff --git a/crates/re_viewer/src/ui/auto_layout.rs b/crates/re_viewer/src/ui/auto_layout.rs index 40fa7d4af6e7..e94a085fe9a1 100644 --- a/crates/re_viewer/src/ui/auto_layout.rs +++ b/crates/re_viewer/src/ui/auto_layout.rs @@ -44,6 +44,7 @@ pub struct SpaceMakeInfo { pub kind: SpaceViewKind, } +#[derive(Clone)] pub(crate) enum LayoutSplit { LeftRight(Box, f32, Box), TopBottom(Box, f32, Box), @@ -215,10 +216,79 @@ fn find_top_left_leaf(tree: &egui_dock::Tree) -> NodeIndex { // (n_splits, right) // } +fn split_2d( + all_2d: Vec, + space_views: &HashMap, +) -> LayoutSplit { + // Try to find 2d views with visible depth and create a left right split + // Otherwise just create a leaf + if all_2d.is_empty() { + LayoutSplit::Leaf(vec![]) + } else { + // Find all 2d views with visible depth + let depths = all_2d + .iter() + .filter(|space| { + if let Some(space_view) = space_views.get(&space.id) { + if let Some(depth_entity) = space_view + .data_blueprint + .entity_paths() + .clone() + .iter() + .find(|entity_path| { + entity_path.last() == Some(&EntityPathPart::Name("Depth".into())) + }) + { + space_view + .data_blueprint + .entity_properties() + .get(depth_entity) + .visible + } else { + false + } + } else { + false + } + }) + .map(|space| space.id) + .collect_vec(); + + if depths.is_empty() { + // This is likely the initial mono layout, split it vertically + // to get a nice quad layout + if all_2d.len() == 2 { + return LayoutSplit::TopBottom( + LayoutSplit::Leaf(vec![all_2d[0].clone()]).into(), + 0.5, + LayoutSplit::Leaf(vec![all_2d[1].clone()]).into(), + ); + } + LayoutSplit::Leaf(all_2d) + } else { + let mut left_split = Vec::new(); + let mut right_split = Vec::new(); + for space in all_2d { + if depths.contains(&space.id) { + right_split.push(space); + } else { + left_split.push(space); + } + } + LayoutSplit::LeftRight( + LayoutSplit::Leaf(left_split.into()).into(), + 0.5, + LayoutSplit::Leaf(right_split.into()).into(), + ) + } + } +} + /// Layout `CAM_A` `CAM_B` | `CAM_C` with 3d views on top and 2d views on the bottom in the same group. (only one 2d and one 3d view visible from the start) fn create_inner_viewport_layout( viewport_size: egui::Vec2, spaces: &Vec, + space_views: &HashMap, ) -> LayoutSplit { let mut groups: HashMap, Vec)> = HashMap::default(); @@ -330,29 +400,33 @@ fn create_inner_viewport_layout( .cloned() .collect_vec(); - let color_split_2d = LayoutSplit::Leaf(colors_2d.clone().into()); + let color_split_2d = split_2d(colors_2d.clone(), space_views); let color_split_3d = LayoutSplit::Leaf(colors_3d.clone().into()); - let mono_split_2d = LayoutSplit::Leaf(monos_2d.clone().into()); + let mono_split_2d = split_2d(monos_2d.clone(), space_views); let mono_split_3d = LayoutSplit::Leaf(monos_3d.clone().into()); let mono_split = if monos_2d.is_empty() && monos_3d.is_empty() { LayoutSplit::Leaf(vec![]) } else if monos_2d.is_empty() && !monos_3d.is_empty() { - mono_split_3d + mono_split_3d.clone() } else if !monos_2d.is_empty() && monos_3d.is_empty() { mono_split_2d } else { - LayoutSplit::TopBottom(mono_split_3d.into(), 0.5, mono_split_2d.into()) + LayoutSplit::TopBottom(mono_split_3d.clone().into(), 0.5, mono_split_2d.into()) }; let color_split = if colors_2d.is_empty() && colors_3d.is_empty() { LayoutSplit::Leaf(vec![]) } else if colors_2d.is_empty() && !colors_3d.is_empty() { - color_split_3d + color_split_3d.clone() } else if !colors_2d.is_empty() && colors_3d.is_empty() { - color_split_2d + color_split_2d.clone() } else { - LayoutSplit::TopBottom(color_split_3d.into(), 0.5, color_split_2d.into()) + LayoutSplit::TopBottom( + color_split_3d.clone().into(), + 0.5, + color_split_2d.clone().into(), + ) }; if color_split.is_empty() && mono_split.is_empty() { @@ -362,7 +436,15 @@ fn create_inner_viewport_layout( } else if !color_split.is_empty() && mono_split.is_empty() { color_split } else { - LayoutSplit::LeftRight(color_split.into(), 0.5, mono_split.into()) + // If there is only one 3d view, we want to make it as big as possible + let ratio = if color_split_3d.is_empty() && !mono_split_3d.is_empty() { + 0.2 + } else if !color_split_3d.is_empty() && mono_split_3d.is_empty() { + 0.8 + } else { + 0.5 + }; + LayoutSplit::LeftRight(color_split.into(), ratio, mono_split.into()) } } @@ -382,11 +464,6 @@ pub(crate) fn default_tree_from_space_views( space_views: &HashMap, is_maximized: bool, ) -> egui_dock::Tree { - // TODO(filip): Implement sensible auto layout when space views changes. - // Something like: - // - Get the tabs that need to be added or removed - // - Removal is easy, just remove the tab - // - Addition should try to layout like currently 3d, 2d views. New views just appear in the top left corner i guess. let mut tree = egui_dock::Tree::new(Vec::new()); let spaces = space_views @@ -450,7 +527,7 @@ pub(crate) fn default_tree_from_space_views( } } else { LayoutSplit::LeftRight( - create_inner_viewport_layout(viewport_size, &spaces).into(), + create_inner_viewport_layout(viewport_size, &spaces, space_views).into(), 0.7, right_panel_split().into(), ) diff --git a/crates/re_viewer/src/ui/data_blueprint.rs b/crates/re_viewer/src/ui/data_blueprint.rs index e965ebeb2c19..884539e2469f 100644 --- a/crates/re_viewer/src/ui/data_blueprint.rs +++ b/crates/re_viewer/src/ui/data_blueprint.rs @@ -149,6 +149,11 @@ impl DataBlueprintTree { &mut self.data_blueprints.individual } + /// Returns multiple individual entity properties as immutable. The hierarchy was not applied to this. + pub fn entity_properties(&self) -> &EntityPropertyMap { + &self.data_blueprints.individual + } + pub fn contains_entity(&self, path: &EntityPath) -> bool { self.entity_paths.contains(path) } diff --git a/crates/re_viewer/src/ui/device_settings_panel.rs b/crates/re_viewer/src/ui/device_settings_panel.rs index 86fba5181006..8cb191fb9bd7 100644 --- a/crates/re_viewer/src/ui/device_settings_panel.rs +++ b/crates/re_viewer/src/ui/device_settings_panel.rs @@ -1,4 +1,7 @@ -use crate::{ depthai::depthai::{ self, CameraBoardSocket }, misc::ViewerContext }; +use crate::{ + depthai::depthai::{self, CameraBoardSocket}, + misc::ViewerContext, +}; use strum::IntoEnumIterator; // Needed for enum::iter() @@ -14,14 +17,13 @@ impl DeviceSettingsPanel { pub fn show_panel(&mut self, ctx: &mut ViewerContext<'_>, ui: &mut egui::Ui) { let mut available_devices = ctx.depthai_state.get_devices(); let currently_selected_device = ctx.depthai_state.selected_device.clone(); - let mut combo_device: depthai::DeviceId = currently_selected_device.clone().id; - if !combo_device.is_empty() && available_devices.is_empty() { + let mut combo_device: depthai::DeviceInfo = currently_selected_device.info.clone(); + if !combo_device.mxid.is_empty() && available_devices.is_empty() { available_devices.push(combo_device.clone()); } let mut show_device_config = true; - egui::CentralPanel - ::default() + egui::CentralPanel::default() .frame(egui::Frame { inner_margin: egui::Margin::same(0.0), fill: egui::Color32::WHITE, @@ -31,56 +33,54 @@ impl DeviceSettingsPanel { (egui::Frame { inner_margin: egui::Margin::same(re_ui::ReUi::view_padding()), ..Default::default() - }).show(ui, |ui| { + }) + .show(ui, |ui| { ui.horizontal(|ui| { // Use up all the horizontal space (color) ui.add_sized( [ui.available_width(), re_ui::ReUi::box_height() + 20.0], |ui: &mut egui::Ui| { ui.horizontal(|ui| { - ctx.re_ui.labeled_combo_box( - ui, - "Device", - if !combo_device.is_empty() { - combo_device.clone() - } else { - "No device selected".to_owned() - }, - true, - |ui: &mut egui::Ui| { - if - ui + ui.add(|ui: &mut egui::Ui| { + egui::ComboBox::from_id_source("device_select") + .selected_text(if !combo_device.mxid.is_empty() { + combo_device.display_text() + } else { + "No device selected".to_owned() + }) + .width(re_ui::ReUi::box_width()) + .wrap(true) + .show_ui(ui, |ui: &mut egui::Ui| { + if ui .selectable_value( &mut combo_device, - String::new(), - "No device" + depthai::DeviceInfo::default(), + "No device", ) .changed() - { - ctx.depthai_state.select_device( - combo_device.clone() - ); - } - for device in available_devices { - if - ui + { + ctx.depthai_state + .select_device(combo_device.mxid.clone()); + } + for device in available_devices { + if ui .selectable_value( &mut combo_device, device.clone(), - device + device.display_text(), ) .changed() - { - ctx.depthai_state.select_device( - combo_device.clone() - ); + { + ctx.depthai_state.select_device( + combo_device.mxid.clone(), + ); + } } - } - } - ); - if - !currently_selected_device.id.is_empty() && - !ctx.depthai_state.is_update_in_progress() + }) + .response + }); + if !currently_selected_device.id.is_empty() + && !ctx.depthai_state.is_update_in_progress() { ui.add_sized( [ @@ -93,8 +93,10 @@ impl DeviceSettingsPanel { // TODO(filip): Create a re_ui bound button with this style let color = ctx.re_ui.design_tokens.error_bg_color; - let hover_color = - ctx.re_ui.design_tokens.error_hover_bg_color; + let hover_color = ctx + .re_ui + .design_tokens + .error_hover_bg_color; style.visuals.widgets.hovered.bg_fill = hover_color; style.visuals.widgets.hovered.weak_bg_fill = @@ -102,23 +104,28 @@ impl DeviceSettingsPanel { style.visuals.widgets.inactive.bg_fill = color; style.visuals.widgets.inactive.weak_bg_fill = color; - style.visuals.widgets.inactive.fg_stroke.color = - egui::Color32::WHITE; + style + .visuals + .widgets + .inactive + .fg_stroke + .color = egui::Color32::WHITE; style.visuals.widgets.hovered.fg_stroke.color = egui::Color32::WHITE; ui.set_style(style); if ui.button("Disconnect").clicked() { - ctx.depthai_state.select_device( - String::new() - ); + ctx.depthai_state + .select_device(String::new()); } - }).response - } + }) + .response + }, ); } - }).response - } + }) + .response + }, ); }); @@ -127,15 +134,14 @@ impl DeviceSettingsPanel { if pipeline_update_in_progress { ui.add_sized([CONFIG_UI_WIDTH, 10.0], |ui: &mut egui::Ui| { ui.with_layout(egui::Layout::left_to_right(egui::Align::Center), |ui| { - ui.label( - if device_selected { - "Updating Pipeline" - } else { - "Selecting Device" - } - ); + ui.label(if device_selected { + "Updating Pipeline" + } else { + "Selecting Device" + }); ui.add(egui::Spinner::new()) - }).response + }) + .response }); show_device_config = false; } @@ -155,50 +161,52 @@ impl DeviceSettingsPanel { ctx: &mut ViewerContext<'_>, ui: &mut egui::Ui, camera_features: &depthai::CameraFeatures, - camera_config: &mut depthai::CameraConfig + camera_config: &mut depthai::CameraConfig, ) { let primary_700 = ctx.re_ui.design_tokens.primary_700; - egui::CollapsingHeader - ::new( - egui::RichText - ::new(camera_features.board_socket.display_name(ctx)) - .color(primary_700) - ) - .default_open(true) - .show(ui, |ui| { - ui.vertical(|ui| { - ui.set_width(CONFIG_UI_WIDTH); - ctx.re_ui.labeled_combo_box( - ui, - "Resolution", - format!("{}", camera_config.resolution), - false, - |ui| { - for res in camera_features.resolutions.clone() { - let disabled = false; - ui.add_enabled_ui(!disabled, |ui| { - ui.selectable_value( - &mut camera_config.resolution, - res, - format!("{res}") - ).on_disabled_hover_ui(|ui| { - ui.label( - format!("{res} will be available in a future release!") - ); - }); + egui::CollapsingHeader::new( + egui::RichText::new(camera_features.board_socket.display_name(ctx)).color(primary_700), + ) + .default_open(true) + .show(ui, |ui| { + ui.vertical(|ui| { + ui.set_width(CONFIG_UI_WIDTH); + ctx.re_ui.labeled_combo_box( + ui, + "Resolution", + format!("{}", camera_config.resolution), + false, + true, + |ui| { + for res in camera_features.resolutions.clone() { + let disabled = false; + ui.add_enabled_ui(!disabled, |ui| { + ui.selectable_value( + &mut camera_config.resolution, + res, + format!("{res}"), + ) + .on_disabled_hover_ui(|ui| { + ui.label(format!( + "{res} will be available in a future release!" + )); }); - } + }); } - ); - ctx.re_ui.labeled_dragvalue( - ui, - "FPS", - &mut camera_config.fps, - 0..=camera_features.max_fps - ); - ctx.re_ui.labeled_checkbox(ui, "Stream", &mut camera_config.stream_enabled); - }); + }, + ); + ctx.re_ui.labeled_dragvalue( + ui, + egui::Id::from("fps"), // TODO(filip): using "fps" as id causes all fps sliders to be linked - This is a bug, but also kind of a feature + None, + "FPS", + &mut camera_config.fps, + 0..=camera_features.max_fps, + ); + ctx.re_ui + .labeled_checkbox(ui, "Stream", &mut camera_config.stream_enabled); }); + }); } fn device_configuration_ui(ctx: &mut ViewerContext<'_>, ui: &mut egui::Ui) { @@ -206,71 +214,79 @@ impl DeviceSettingsPanel { let primary_700 = ctx.re_ui.design_tokens.primary_700; let connected_cameras = ctx.depthai_state.get_connected_cameras().clone(); - ctx.re_ui.styled_scrollbar(ui, re_ui::ScrollAreaDirection::Vertical, [false; 2], |ui| { - (egui::Frame { - fill: ctx.re_ui.design_tokens.gray_50, - inner_margin: egui::Margin::symmetric(30.0, 21.0), - ..Default::default() - }).show(ui, |ui| { - ui.horizontal(|ui| { - ui.vertical(|ui| { - for cam in connected_cameras.clone() { - let Some(config) = device_config.cameras + ctx.re_ui + .styled_scrollbar(ui, re_ui::ScrollAreaDirection::Vertical, [false; 2], |ui| { + (egui::Frame { + fill: ctx.re_ui.design_tokens.gray_50, + inner_margin: egui::Margin::symmetric(30.0, 21.0), + ..Default::default() + }) + .show(ui, |ui| { + ui.horizontal(|ui| { + ui.vertical(|ui| { + for cam in connected_cameras.clone() { + let Some(config) = device_config.cameras .iter_mut() .find(|conf| conf.board_socket == cam.board_socket) else { continue; }; - Self::camera_config_ui(ctx, ui, &cam, config); - } + Self::camera_config_ui(ctx, ui, &cam, config); + } - ui.collapsing(egui::RichText::new("AI settings").color(primary_700), |ui| { - ui.vertical(|ui| { - ui.set_width(CONFIG_UI_WIDTH); - ctx.re_ui.labeled_combo_box( - ui, - "AI Model", - device_config.ai_model.display_name.clone(), - false, - |ui| { - for nn in &ctx.depthai_state.neural_networks { - ui.selectable_value( - &mut device_config.ai_model, - nn.clone(), - &nn.display_name - ); - } - } - ); - ctx.re_ui.labeled_combo_box( - ui, - "Run on", - device_config.ai_model.camera.display_name(ctx), - false, - |ui| { - for cam in &connected_cameras { - ui.selectable_value( - &mut device_config.ai_model.camera, - cam.board_socket, - cam.board_socket.display_name(ctx) - ); - } - } - ); - }); - }); + ui.collapsing( + egui::RichText::new("AI settings").color(primary_700), + |ui| { + ui.vertical(|ui| { + ui.set_width(CONFIG_UI_WIDTH); + ctx.re_ui.labeled_combo_box( + ui, + "AI Model", + device_config.ai_model.display_name.clone(), + false, + true, + |ui| { + for nn in &ctx.depthai_state.neural_networks { + ui.selectable_value( + &mut device_config.ai_model, + nn.clone(), + &nn.display_name, + ); + } + }, + ); + ctx.re_ui.labeled_combo_box( + ui, + "Run on", + device_config.ai_model.camera.display_name(ctx), + false, + true, + |ui| { + for cam in &connected_cameras { + ui.selectable_value( + &mut device_config.ai_model.camera, + cam.board_socket, + cam.board_socket.display_name(ctx), + ); + } + }, + ); + }); + }, + ); - let mut depth = device_config.depth.unwrap_or_default(); - ui.add_enabled_ui( - ctx.depthai_state.selected_device.has_stereo_pairs(), - |ui| { - egui::CollapsingHeader - ::new(egui::RichText::new("Depth Settings").color(primary_700)) + let mut depth = device_config.depth.unwrap_or_default(); + ui.add_enabled_ui( + ctx.depthai_state.selected_device.has_stereo_pairs(), + |ui| { + egui::CollapsingHeader::new( + egui::RichText::new("Depth Settings").color(primary_700), + ) .open( if !ctx.depthai_state.selected_device.has_stereo_pairs() { Some(false) } else { None - } + }, ) .show(ui, |ui| { ui.vertical(|ui| { @@ -285,8 +301,13 @@ impl DeviceSettingsPanel { cam2.display_name(ctx) ), false, + true, |ui| { - for pair in &ctx.depthai_state.selected_device.stereo_pairs { + for pair in &ctx + .depthai_state + .selected_device + .stereo_pairs + { ui.selectable_value( &mut depth.stereo_pair, *pair, @@ -294,161 +315,176 @@ impl DeviceSettingsPanel { "{} {}", pair.0.display_name(ctx), pair.1.display_name(ctx) - ) + ), ); } - } + }, ); ctx.re_ui.labeled_checkbox( ui, "LR Check", - &mut depth.lr_check + &mut depth.lr_check, ); ctx.re_ui.labeled_combo_box( ui, "Align to", depth.align.display_name(ctx), false, + true, |ui| { for align in &connected_cameras { ui.selectable_value( &mut depth.align, align.board_socket, - align.board_socket.display_name(ctx) + align.board_socket.display_name(ctx), ); } - } + }, ); ctx.re_ui.labeled_combo_box( ui, "Median Filter", format!("{:?}", depth.median), false, + true, |ui| { - for filter in depthai::DepthMedianFilter::iter() { + for filter in depthai::DepthMedianFilter::iter() + { ui.selectable_value( &mut depth.median, filter, - format!("{filter:?}") + format!("{filter:?}"), ); } - } + }, ); ctx.re_ui.labeled_dragvalue( ui, + egui::Id::from("LR Threshold"), + Some(100.0), "LR Threshold", &mut depth.lrc_threshold, - 0..=10 + 0..=10, ); ctx.re_ui.labeled_checkbox( ui, "Extended Disparity", - &mut depth.extended_disparity + &mut depth.extended_disparity, ); ctx.re_ui.labeled_checkbox( ui, "Subpixel Disparity", - &mut depth.subpixel_disparity + &mut depth.subpixel_disparity, ); ctx.re_ui.labeled_dragvalue( ui, + egui::Id::from("Sigma"), + Some(100.0), "Sigma", &mut depth.sigma, - 0..=65535 + 0..=65535, ); ctx.re_ui.labeled_dragvalue( ui, + egui::Id::from("Confidence"), + Some(100.0), "Confidence", &mut depth.confidence, - 0..=255 + 0..=255, ); ctx.re_ui.labeled_toggle_switch( ui, "Depth enabled", - &mut device_config.depth_enabled + &mut device_config.depth_enabled, ); }); }) - .header_response.on_disabled_hover_ui(|ui| { - ui.label("Selected device doesn't have any stereo pairs!"); - }); - } - ); + .header_response + .on_disabled_hover_ui( + |ui| { + ui.label( + "Selected device doesn't have any stereo pairs!", + ); + }, + ); + }, + ); - device_config.depth = Some(depth); - ctx.depthai_state.modified_device_config = device_config.clone(); - ui.vertical(|ui| { - ui.horizontal(|ui| { - let apply_enabled = { - if - let Some(applied_config) = + device_config.depth = Some(depth); + ctx.depthai_state.modified_device_config = device_config.clone(); + ui.vertical(|ui| { + ui.horizontal(|ui| { + let apply_enabled = { + if let Some(applied_config) = &ctx.depthai_state.applied_device_config.config - { - let only_runtime_configs_changed = - depthai::State::only_runtime_configs_changed( - applied_config, - &device_config - ); - let apply_enabled = - !only_runtime_configs_changed && - ctx.depthai_state.applied_device_config.config.is_some() && - device_config != applied_config.clone() && - !ctx.depthai_state.selected_device.id.is_empty() && - !ctx.depthai_state.is_update_in_progress(); + { + let only_runtime_configs_changed = + depthai::State::only_runtime_configs_changed( + applied_config, + &device_config, + ); + let apply_enabled = !only_runtime_configs_changed + && ctx + .depthai_state + .applied_device_config + .config + .is_some() + && device_config != applied_config.clone() + && !ctx.depthai_state.selected_device.id.is_empty() + && !ctx.depthai_state.is_update_in_progress(); - if !apply_enabled && only_runtime_configs_changed { - ctx.depthai_state.set_pipeline( - &mut device_config, - true - ); + if !apply_enabled && only_runtime_configs_changed { + ctx.depthai_state + .set_pipeline(&mut device_config, true); + } + apply_enabled + } else { + !ctx.depthai_state + .applied_device_config + .update_in_progress } - apply_enabled - } else { - !ctx.depthai_state.applied_device_config.update_in_progress - } - }; + }; - ui.add_enabled_ui(apply_enabled, |ui| { - ui.scope(|ui| { - let mut style = ui.style_mut().clone(); - if apply_enabled { - let color = ctx.re_ui.design_tokens.primary_bg_color; - let hover_color = - ctx.re_ui.design_tokens.primary_hover_bg_color; - style.visuals.widgets.hovered.bg_fill = hover_color; - style.visuals.widgets.hovered.weak_bg_fill = - hover_color; - style.visuals.widgets.inactive.bg_fill = color; - style.visuals.widgets.inactive.weak_bg_fill = color; - style.visuals.widgets.inactive.fg_stroke.color = - egui::Color32::WHITE; - style.visuals.widgets.hovered.fg_stroke.color = - egui::Color32::WHITE; - } - style.spacing.button_padding = egui::Vec2::new(24.0, 4.0); - ui.set_style(style); - if - ui + ui.add_enabled_ui(apply_enabled, |ui| { + ui.scope(|ui| { + let mut style = ui.style_mut().clone(); + if apply_enabled { + let color = + ctx.re_ui.design_tokens.primary_bg_color; + let hover_color = + ctx.re_ui.design_tokens.primary_hover_bg_color; + style.visuals.widgets.hovered.bg_fill = hover_color; + style.visuals.widgets.hovered.weak_bg_fill = + hover_color; + style.visuals.widgets.inactive.bg_fill = color; + style.visuals.widgets.inactive.weak_bg_fill = color; + style.visuals.widgets.inactive.fg_stroke.color = + egui::Color32::WHITE; + style.visuals.widgets.hovered.fg_stroke.color = + egui::Color32::WHITE; + } + style.spacing.button_padding = + egui::Vec2::new(24.0, 4.0); + ui.set_style(style); + if ui .add_sized( [CONFIG_UI_WIDTH, re_ui::ReUi::box_height()], - egui::Button::new("Apply") + egui::Button::new("Apply"), ) .clicked() - { - ctx.depthai_state.set_pipeline( - &mut device_config, - false - ); - } + { + ctx.depthai_state + .set_pipeline(&mut device_config, false); + } + }); }); }); }); }); + ui.allocate_space(ui.available_size()); }); - ui.allocate_space(ui.available_size()); }); }); - }); // Set a more visible scroll bar color } } diff --git a/crates/re_viewer/src/ui/selection_panel.rs b/crates/re_viewer/src/ui/selection_panel.rs index 9b774a7ce084..7b49beb60f0b 100644 --- a/crates/re_viewer/src/ui/selection_panel.rs +++ b/crates/re_viewer/src/ui/selection_panel.rs @@ -432,8 +432,7 @@ fn colormap_props_ui( ) else { return; }; - - if tensor.is_shaped_like_an_image() + if tensor.meaning() != TensorDataMeaning::Depth && tensor.is_shaped_like_an_image() && ui .selectable_label(current.as_ref() == Some(ent_path), ent_path.to_string()) .clicked() diff --git a/crates/re_viewer/src/ui/space_view.rs b/crates/re_viewer/src/ui/space_view.rs index df5c7111f3c8..79e32423fbf1 100644 --- a/crates/re_viewer/src/ui/space_view.rs +++ b/crates/re_viewer/src/ui/space_view.rs @@ -1,3 +1,4 @@ +use itertools::Itertools; use re_arrow_store::Timeline; use re_data_store::{EntityPath, EntityPropertyMap, EntityTree, InstancePath, TimeInt}; use re_log_types::EntityPathPart; @@ -144,6 +145,11 @@ impl SpaceView { } } + + pub fn duplicate(&self, ctx: &ViewerContext<'_>) -> Self { + Self::new(ctx, self.category, &self.space_path, self.data_blueprint.entity_paths().iter().cloned().collect_vec().as_slice()) + } + pub fn on_frame_start( &mut self, ctx: &mut ViewerContext<'_>, diff --git a/crates/re_viewer/src/ui/space_view_heuristics.rs b/crates/re_viewer/src/ui/space_view_heuristics.rs index 040a76e49096..612134c46639 100644 --- a/crates/re_viewer/src/ui/space_view_heuristics.rs +++ b/crates/re_viewer/src/ui/space_view_heuristics.rs @@ -1,18 +1,21 @@ -use std::collections::BTreeMap; +use std::{borrow::BorrowMut, collections::BTreeMap}; use ahash::HashMap; use itertools::Itertools; use nohash_hasher::IntSet; use re_arrow_store::{DataStore, LatestAtQuery, Timeline}; use re_data_store::{log_db::EntityDb, query_latest_single, ComponentName, EntityPath, EntityTree}; -use re_log_types::{component_types::Tensor, Component, EntityPathPart}; +use re_log_types::{ + component_types::{Tensor, TensorDataMeaning}, + Component, EntityPathPart, +}; use crate::{ misc::{space_info::SpaceInfoCollection, ViewerContext}, ui::{view_category::categorize_entity_path, ViewCategory}, }; -use super::{view_category::ViewCategorySet, SpaceView}; +use super::{view_category::ViewCategorySet, view_spatial::SpatialNavigationMode, SpaceView}; /// List out all space views we allow the user to create. pub fn all_possible_space_views( @@ -139,7 +142,120 @@ pub fn default_created_space_views( spaces_info: &SpaceInfoCollection, ) -> Vec { let candidates = all_possible_space_views(ctx, spaces_info); - default_created_space_views_from_candidates(ctx, &ctx.log_db.entity_db, candidates) + let default_space_views = + default_created_space_views_from_candidates(ctx, &ctx.log_db.entity_db, candidates); + if !ctx.depthai_state.selected_device.id.is_empty() { + return default_depthai_space_views(ctx, default_space_views); + } + default_space_views +} + +fn default_depthai_space_views( + ctx: &ViewerContext<'_>, + default_created_views: Vec, +) -> Vec { + // Remove 3D views that don't have a Depth tensor in them, + let mut cam_with_depth = None; + let mut space_views = default_created_views + .into_iter() + .filter_map(|mut space_view| { + // filter_map, so space_view is moved into the closure. + if space_view.space_path.len() == 1 { + // These are CAM_A, CAM_B, ..., basically the root spaces. + // Check if there is a depth tensor in the space view. + let query = LatestAtQuery::new(Timeline::log_time(), re_arrow_store::TimeInt::MAX); + let entity_db = &ctx.log_db.entity_db; + for entity_path in space_view.data_blueprint.entity_paths() { + if let Ok(entity_view) = re_query::query_entity_with_primary::( + &entity_db.data_store, + &query, + entity_path, + &[], + ) { + for tensor in entity_view.iter_primary_flattened() { + if tensor.meaning() == TensorDataMeaning::Depth { + cam_with_depth = space_view.space_path.iter().last().cloned(); + // We also wan't to remove the Image and Detections from the 3D space view! + let entity_paths = space_view.data_blueprint.entity_paths().clone(); + let entities_to_remove = entity_paths.iter().filter(|ep| { + ep.last() == Some(&EntityPathPart::Name("Image".into())) + || ep.last() + == Some(&EntityPathPart::Name("Detections".into())) + || ep.last() + == Some(&EntityPathPart::Name("Detection".into())) + }); + for entity in entities_to_remove { + let mut props = space_view + .data_blueprint + .data_blueprints_individual() + .get(&entity); + props.visible = false; + space_view + .data_blueprint + .data_blueprints_individual() + .set(entity.clone(), props); + } + return Some(space_view); + } + } + } + } + return None; // No depth tensor was found inside of this 3D view. + } + Some(space_view) + }) + .collect::>(); + + // If a depth tensor is found, we want to find the 2D space view that has the Image + Depth tensor. + // We then wan't to create two separate 2D space views, one for the image and one for the depth. + // But we only want to hide the depth (or image), not remove it from the space view. + if let Some(depth_2d) = space_views + .iter_mut() + .find(|space_view| space_view.space_path.as_slice().first() == cam_with_depth.as_ref()) + { + if let Some(image_entity) = depth_2d + .data_blueprint + .entity_paths() + .iter() + .find(|entity_path| entity_path.last() == Some(&EntityPathPart::Name("Image".into()))) + .cloned() + { + let mut duplicate = depth_2d.duplicate(ctx); + // Change depth 2d to only show depth. + let mut depth_2d_props = depth_2d + .data_blueprint + .data_blueprints_individual() + .get(&image_entity); + depth_2d_props.visible = false; + depth_2d + .data_blueprint + .data_blueprints_individual() + .set(image_entity, depth_2d_props); + + // Change duplicate to only show image. + if let Some(depth_entity) = depth_2d + .data_blueprint + .entity_paths() + .iter() + .find(|entity_path| { + entity_path.last() == Some(&EntityPathPart::Name("Depth".into())) + }) + .cloned() + { + let mut duplicate_props = duplicate + .data_blueprint + .data_blueprints_individual() + .get(&depth_entity); + duplicate_props.visible = false; + duplicate + .data_blueprint + .data_blueprints_individual() + .set(depth_entity, duplicate_props); + } + space_views.push(duplicate); + } + } + space_views } fn default_created_space_views_from_candidates( @@ -209,8 +325,11 @@ fn default_created_space_views_from_candidates( ) { for tensor in entity_view.iter_primary_flattened() { if tensor.is_shaped_like_an_image() { - debug_assert!(matches!(tensor.shape.len(), 2 | 3)); - let dim = (tensor.shape[0].size, tensor.shape[1].size); + debug_assert!(matches!(tensor.real_shape().len(), 2 | 3)); + let dim = ( + tensor.real_shape().as_slice()[0].size, + tensor.real_shape().as_slice()[1].size, + ); images_by_size .entry(dim) .or_default() diff --git a/crates/re_viewer/src/ui/view_spatial/ui.rs b/crates/re_viewer/src/ui/view_spatial/ui.rs index 11b02345faa9..6c9d1ea642bb 100644 --- a/crates/re_viewer/src/ui/view_spatial/ui.rs +++ b/crates/re_viewer/src/ui/view_spatial/ui.rs @@ -270,6 +270,11 @@ impl ViewSpatialState { } } } + if let Some(what_image) = ent_path.iter().last() { + if what_image == &EntityPathPart::from("Depth") { + return; // Doesn't make sense to project depth tensor... + } + } let Some(tensor) = query_latest_single::( &ctx.log_db.entity_db, ent_path, diff --git a/crates/re_viewer/src/ui/viewport.rs b/crates/re_viewer/src/ui/viewport.rs index 4dfa37fe1d6a..172b7051bd36 100644 --- a/crates/re_viewer/src/ui/viewport.rs +++ b/crates/re_viewer/src/ui/viewport.rs @@ -62,6 +62,8 @@ pub struct Viewport { stats_panel_state: StatsPanelState, previous_frame_tree: Option>, + + previous_frame_default_created_sapace_views: Vec, // Space paths from previous frame's call to default_created_space_views } impl Viewport { @@ -70,9 +72,7 @@ impl Viewport { crate::profile_function!(); let mut blueprint = Self::default(); - for space_view in all_possible_space_views(ctx, spaces_info) - { - println!("All possible: {:?}", space_view.space_path); + for space_view in default_created_space_views(ctx, spaces_info) { blueprint.add_space_view(space_view); } blueprint @@ -97,6 +97,7 @@ impl Viewport { device_settings_panel: _, stats_panel_state: _, previous_frame_tree: _, + previous_frame_default_created_sapace_views: _, } = self; if let Some(window) = space_view_entity_window { @@ -272,6 +273,12 @@ impl Viewport { space_views_to_delete } + fn reset_space_views(&mut self) { + self.space_views.clear(); + self.visible.clear(); + self.trees.clear(); + } + pub fn on_frame_start( &mut self, ctx: &mut ViewerContext<'_>, @@ -279,29 +286,47 @@ impl Viewport { ) { crate::profile_function!(); - // for space_view_id in &self.get_space_views_to_delete(ctx, spaces_info) { - // self.remove(space_view_id); - // } + if !ctx.depthai_state.selected_device.id.is_empty() { + for space_view_id in &self.get_space_views_to_delete(ctx, spaces_info) { + self.remove(space_view_id); + } + } self.stats_panel_state.update(ctx); for space_view in self.space_views.values_mut() { space_view.on_frame_start(ctx, spaces_info); } - for space_view_candidate in default_created_space_views(ctx, spaces_info) { - if !self - .has_been_user_edited - .get(&space_view_candidate.space_path) - .unwrap_or(&false) - && self.should_auto_add_space_view(&space_view_candidate) - { - self.add_space_view(space_view_candidate); + + // Maybe it's a hack but it works: Remember previous default_created_space_views and compare them to the current ones + // If there is a diff then reset the space clear the space_views and add the new ones + // This is done to avoid out of hand creation of space views that just have entities with different visibilities (eg. when we hide Image in 3D depth view) + let mut defaults = default_created_space_views(ctx, spaces_info); + if !defaults.iter().all(|space_view| { + self.previous_frame_default_created_sapace_views + .contains(&space_view.space_path) + }) { + self.reset_space_views(); + for space_view_candidate in &mut defaults { + if !self + .has_been_user_edited + .get(&space_view_candidate.space_path) + .unwrap_or(&false) + && self.should_auto_add_space_view(space_view_candidate) + { + self.add_space_view(space_view_candidate.clone()); + } } } + self.previous_frame_default_created_sapace_views = defaults + .iter() + .map(|space_view| space_view.space_path.clone()) + .collect_vec(); } - fn should_auto_add_space_view(&self, space_view_candidate: &SpaceView) -> bool { - for existing_view in self.space_views.values() { + fn should_auto_add_space_view(&mut self, space_view_candidate: &mut SpaceView) -> bool { + crate::profile_function!(); + for existing_view in self.space_views.values_mut() { if existing_view.space_path == space_view_candidate.space_path { if existing_view.entities_determined_by_user { // Since the user edited a space view with the same space path, we can't be sure our new one isn't redundant. @@ -309,17 +334,33 @@ impl Viewport { return false; } - if space_view_candidate - .data_blueprint - .entity_paths() - .is_subset(existing_view.data_blueprint.entity_paths()) - { - // This space view wouldn't add anything we haven't already - return false; + let candidate_entity_paths = + space_view_candidate.data_blueprint.entity_paths().clone(); + if candidate_entity_paths.is_subset(existing_view.data_blueprint.entity_paths()) { + // Check if the entity paths have the same visibilities + // Only add if the visibilities are different + let candidate_vis = candidate_entity_paths.iter().sorted().map(|ep| { + space_view_candidate + .data_blueprint + .data_blueprints_individual() + .get(ep) + .visible + }); + + let existing_entity_paths = existing_view.data_blueprint.entity_paths().clone(); + let existing_vis = existing_entity_paths.iter().sorted().map(|ep| { + existing_view + .data_blueprint + .data_blueprints_individual() + .get(ep) + .visible + }); + + let all_same = candidate_vis.zip(existing_vis).all(|(a, b)| a == b); + return !all_same; } } } - true } @@ -418,42 +459,19 @@ impl Viewport { { match space_view_kind { SpaceViewKind::Data | SpaceViewKind::Stats => { - let mut entities_to_skip = Vec::new(); + let entities_to_skip = Vec::new(); if let Some(space_view) = self.space_views.get_mut(&space_view_id) { let mut is3d = false; - let mut has_depth = false; - let mut image_to_hide = None; space_view.data_blueprint.visit_group_entities_recursively( space_view.data_blueprint.root_handle(), &mut (|entity_path| { - if is3d && has_depth { - if let Some(last_part) = entity_path.iter().last() { - if last_part == &EntityPathPart::from("Image") { - image_to_hide = Some(entity_path.clone()); - } - } - } is3d |= entity_path.len() == 2 && entity_path.iter().last().unwrap() == &EntityPathPart::from("transform"); - if let Some(last_part) = entity_path.iter().last() { - has_depth |= last_part == &EntityPathPart::from("Depth"); - } }), ); - if let Some(image_to_hide) = image_to_hide { - entities_to_skip.push(image_to_hide.clone()); - let mut props = space_view - .data_blueprint - .data_blueprints_individual() - .get(&image_to_hide); - props.visible = false; - space_view - .data_blueprint - .data_blueprints_individual() - .set(image_to_hide.clone(), props); - } } + space_view_options_ui( ctx, ui, @@ -750,13 +768,13 @@ fn space_view_options_ui( ctx.set_single_selection(Item::SpaceView(space_view_id)); } } - let Some(space_view) = viewport.space_views.get_mut(&space_view_id) else { - return; - }; let icon_image = ctx.re_ui.icon_image(&re_ui::icons::GEAR); let texture_id = icon_image.texture_id(ui.ctx()); ui.menu_image_button(texture_id, re_ui::ReUi::small_icon_size(), |ui| { + let Some(space_view) = viewport.space_views.get_mut(&space_view_id) else { + return; + }; ui.style_mut().wrap = Some(false); let entities = space_view.data_blueprint.entity_paths().clone(); let entities = entities.iter().filter(|ep| { @@ -802,8 +820,38 @@ fn space_view_options_ui( } }); + let Some(space_view) = viewport.space_views.get(&space_view_id).cloned() else { + return; + }; + + if ctx + .re_ui + .small_icon_button(ui, &re_ui::icons::ADD) + .clicked() + { + // Create a new space view that is identical to this one + let new_sv = space_view.duplicate(ctx); + let id = new_sv.id.clone(); + viewport.add_space_view(new_sv); + viewport + .has_been_user_edited + .insert(viewport.space_views[&id].space_path.clone(), true); + } + + if ctx + .re_ui + .small_icon_button(ui, &re_ui::icons::REMOVE) + .clicked() + { + viewport.has_been_user_edited.insert( + viewport.space_views[&space_view_id].space_path.clone(), + true, + ); + viewport.remove(&space_view_id); + } + // Show help last, since not all space views have help text - help_text_ui(ui, space_view); + help_text_ui(ui, &space_view); // Put a frame so that the buttons cover any labels they intersect with: let rect = ui.min_rect().expand2(egui::vec2(1.0, -2.0)); diff --git a/rerun_py/depthai_viewer/_backend/config_api.py b/rerun_py/depthai_viewer/_backend/config_api.py index 09630e56a70e..3cc2d2155a12 100644 --- a/rerun_py/depthai_viewer/_backend/config_api.py +++ b/rerun_py/depthai_viewer/_backend/config_api.py @@ -92,7 +92,6 @@ async def ws_api(websocket: WebSocketServerProtocol) -> None: if not message_type: print("Missing message type") continue - print("Got message: ", message) if message_type == MessageType.SUBSCRIPTIONS: data = message.get("data", {}) @@ -113,9 +112,7 @@ async def ws_api(websocket: WebSocketServerProtocol) -> None: elif message_type == MessageType.DEVICES: await send_message( websocket, - DevicesMessage( - [d.getMxId() for d in dai.Device.getAllAvailableDevices()] - ), # type: ignore[call-arg] + DevicesMessage(dai.Device.getAllAvailableDevices()), # type: ignore[call-arg] ) elif message_type == MessageType.DEVICE: @@ -133,6 +130,7 @@ async def ws_api(websocket: WebSocketServerProtocol) -> None: message_to_send = None try: message_to_send = send_message_queue.get(timeout=0.01) + print("Got message to send: ", message_to_send) except QueueEmptyException: pass if message_to_send: diff --git a/rerun_py/depthai_viewer/_backend/device.py b/rerun_py/depthai_viewer/_backend/device.py index f8c45fad1998..d874879f31ec 100644 --- a/rerun_py/depthai_viewer/_backend/device.py +++ b/rerun_py/depthai_viewer/_backend/device.py @@ -1,11 +1,15 @@ import itertools import time +from queue import Queue from typing import Dict, List, Optional, Tuple import depthai as dai import numpy as np from depthai_sdk import OakCamera from depthai_sdk.components import CameraComponent, NNComponent, StereoComponent +from depthai_sdk.components.camera_helper import ( + getClosestIspScale, +) from numpy.typing import NDArray import depthai_viewer as viewer @@ -13,14 +17,22 @@ from depthai_viewer._backend.device_configuration import ( CameraConfiguration, CameraFeatures, + DeviceInfo, DeviceProperties, ImuKind, PipelineConfiguration, + XLinkConnection, calculate_isp_scale, compare_dai_camera_configs, - resolution_to_enum, + get_size_from_resolution, + size_to_resolution, +) +from depthai_viewer._backend.messages import ( + ErrorMessage, + InfoMessage, + Message, + WarningMessage, ) -from depthai_viewer._backend.messages import ErrorMessage, InfoMessage, Message from depthai_viewer._backend.packet_handler import ( AiModelCallbackArgs, DepthCallbackArgs, @@ -67,6 +79,8 @@ class Device: _stereo: StereoComponent = None _nnet: NNComponent = None _xlink_statistics: Optional[XlinkStatistics] = None + _sys_info_q: Optional[Queue] = None # type: ignore[type-arg] + _pipeline_start_t: Optional[float] = None # _profiler = cProfile.Profile() @@ -148,7 +162,15 @@ def get_device_properties(self) -> DeviceProperties: connected_cam_features = self._oak.device.getConnectedCameraFeatures() imu = self._oak.device.getConnectedIMU() imu = ImuKind.NINE_AXIS if "BNO" in imu else None if imu == "NONE" else ImuKind.SIX_AXIS - device_properties = DeviceProperties(id=self.id, imu=imu) + device_info = self._oak.device.getDeviceInfo() + device_info = DeviceInfo( + name=device_info.name, + connection=XLinkConnection.POE + if device_info.protocol == dai.XLinkProtocol.X_LINK_TCP_IP + else XLinkConnection.USB, + mxid=device_info.mxid, + ) + device_properties = DeviceProperties(id=self.id, imu=imu, info=device_info) try: calib = self._oak.device.readCalibration2() left_cam = calib.getStereoLeftCameraId() @@ -156,17 +178,28 @@ def get_device_properties(self) -> DeviceProperties: device_properties.default_stereo_pair = (left_cam, right_cam) except RuntimeError: pass + + ordered_resolutions = list(sorted(size_to_resolution.keys(), key=lambda res: res[0] * res[1])) for cam in connected_cam_features: prioritized_type = cam.supportedTypes[0] + biggest_width, biggest_height = [ + (conf.width, conf.height) for conf in cam.configs[::-1] if conf.type == prioritized_type + ][ + 0 + ] # Only support the prioritized type for now + + all_supported_resolutions = [ + size_to_resolution[(w, h)] + for w, h in ordered_resolutions + if (w * h) <= (biggest_height * biggest_width) + ] + + # Fill in lower resolutions that can be achieved with ISP scaling device_properties.cameras.append( CameraFeatures( board_socket=cam.socket, max_fps=60, - resolutions=[ - resolution_to_enum[(conf.width, conf.height)] - for conf in cam.configs - if conf.type == prioritized_type # Only support the prioritized type for now - ], + resolutions=all_supported_resolutions, supported_types=cam.supportedTypes, stereo_pairs=self._get_possible_stereo_pairs_for_cam(cam, connected_cam_features), name=cam.name.capitalize(), @@ -226,9 +259,14 @@ def _get_camera_config_by_socket( return None return camera[0] - def update_pipeline(self, config: PipelineConfiguration, runtime_only: bool) -> Message: + def update_pipeline(self, runtime_only: bool) -> Message: if self._oak is None: return ErrorMessage("No device selected, can't update pipeline!") + + config = self.store.pipeline_config + if config is None: + return ErrorMessage("No pipeline config, can't update pipeline!") + if self._oak.device.isPipelineRunning(): if runtime_only: if config.depth is not None: @@ -244,6 +282,9 @@ def update_pipeline(self, config: PipelineConfiguration, runtime_only: bool) -> self._cameras = [] self._stereo = None self._packet_handler.reset() + self._sys_info_q = None + self._pipeline_start_t = None + synced_outputs = [] synced_callback_args = SyncedCallbackArgs() @@ -251,29 +292,67 @@ def update_pipeline(self, config: PipelineConfiguration, runtime_only: bool) -> print("Usb speed: ", self._oak.device.getUsbSpeed()) is_usb2 = self._oak.device.getUsbSpeed() == dai.UsbSpeed.HIGH if is_poe: + self.store.send_message_to_frontend( + WarningMessage("Device is connected via PoE. This may cause performance issues.") + ) print("Connected to a PoE device, camera streams will be JPEG encoded...") elif is_usb2: + self.store.send_message_to_frontend( + WarningMessage("Device is connected in USB2 mode. This may cause performance issues.") + ) print("Device is connected in USB2 mode, camera streams will be JPEG encoded...") self.use_encoding = is_poe or is_usb2 + + connected_camera_features = self._oak.device.getConnectedCameraFeatures() for cam in config.cameras: print("Creating camera: ", cam) + + camera_features = next(filter(lambda feat: feat.socket == cam.board_socket, connected_camera_features)) + + # When the resolution is too small, the ISP needs to scale it down + res_x, res_y = get_size_from_resolution(cam.resolution) + + does_sensor_support_resolution = any( + [ + config.width == res_x and config.height == res_y + for config in camera_features.configs + if config.type == camera_features.supportedTypes[0] + ] + ) + + # In case of ISP scaling, don't change the sensor resolution in the pipeline config + # to keep it logical for the user in the UI + sensor_resolution = cam.resolution + if not does_sensor_support_resolution: + smallest_supported_resolution = [ + config for config in camera_features.configs if config.type == camera_features.supportedTypes[0] + ][0] + sensor_resolution = size_to_resolution[ + smallest_supported_resolution.width, smallest_supported_resolution.height + ] + sdk_cam = self._oak.create_camera( cam.board_socket, - cam.resolution.as_sdk_resolution(), + sensor_resolution.as_sdk_resolution(), cam.fps, encode=self.use_encoding, name=cam.name.capitalize(), ) - if cam.stream_enabled: - if config.depth and ( - cam.board_socket == config.depth.align or cam.board_socket in config.depth.stereo_pair - ): - synced_outputs.append(sdk_cam.out.main) - else: - self._oak.callback( - sdk_cam, - self._packet_handler.build_callback(cam.board_socket), + if not does_sensor_support_resolution: + sdk_cam.config_color_camera( + isp_scale=getClosestIspScale( + (smallest_supported_resolution.width, smallest_supported_resolution.height), res_x ) + ) + + is_used_by_depth = config.depth is not None and ( + cam.board_socket == config.depth.align or cam.board_socket in config.depth.stereo_pair + ) + is_used_by_ai = config.ai_model is not None and cam.board_socket == config.ai_model.camera + cam.stream_enabled |= is_used_by_depth or is_used_by_ai + + if cam.stream_enabled: + synced_outputs.append(sdk_cam.out.main) self._cameras.append(sdk_cam) if config.depth: @@ -292,7 +371,6 @@ def update_pipeline(self, config: PipelineConfiguration, runtime_only: bool) -> right_cam.config_color_camera(isp_scale=calculate_isp_scale(right_cam.node.getResolutionWidth())) self._stereo = self._oak.create_stereo(left=left_cam, right=right_cam, name="depth") - # We used to be able to pass in the board socket to align to, but this was removed in depthai 1.10.0 align_component = self._get_component_by_socket(config.depth.align) if not align_component: return ErrorMessage(f"{config.depth.align} is not configured. Couldn't create stereo pair.") @@ -345,14 +423,20 @@ def update_pipeline(self, config: PipelineConfiguration, runtime_only: bool) -> if not camera: return ErrorMessage(f"{config.ai_model.camera} is not configured. Couldn't create NN.") - self._oak.callback( - self._nnet, - self._packet_handler.build_callback( - AiModelCallbackArgs(model_name=config.ai_model.path, camera=camera, labels=labels) - ), + synced_callback_args.ai_args = AiModelCallbackArgs( + model_name=config.ai_model.path, camera=camera, labels=labels ) + synced_outputs.append(self._nnet.out.main) + if synced_outputs: self._oak.sync(synced_outputs, self._packet_handler.build_sync_callback(synced_callback_args)) + + sys_logger_xlink = self._oak.pipeline.createXLinkOut() + logger = self._oak.pipeline.createSystemLogger() + logger.setRate(0.1) + sys_logger_xlink.setStreamName("sys_logger") + logger.out.link(sys_logger_xlink.input) + try: self._oak.start(blocking=False) except RuntimeError as e: @@ -361,6 +445,9 @@ def update_pipeline(self, config: PipelineConfiguration, runtime_only: bool) -> running = self._oak.running() if running: + self._pipeline_start_t = time.time() + self._sys_info_q = self._oak.device.getOutputQueue("sys_logger", 1, False) + self.store.set_pipeline_config(config) # We might have modified the config, so store it try: self._oak.poll() except RuntimeError: @@ -378,9 +465,66 @@ def update(self) -> None: if self._xlink_statistics is not None: self._xlink_statistics.update() + if self._sys_info_q is None: + return + sys_info = self._sys_info_q.tryGet() # type: ignore[attr-defined] + if sys_info is not None and self._pipeline_start_t is not None: + print("----------------------------------------") + print(f"[{int(time.time() - self._pipeline_start_t)}s] System information") + print("----------------------------------------") + print_system_information(sys_info) # if time.time() - self.start > 10: # print("Dumping profiling data") # self._profiler.dump_stats("profile.prof") # self._profiler.disable() # self._profiler.enable() # self.start = time.time() + + +def print_system_information(info: dai.SystemInformation) -> None: + print( + "Ddr used / total - %.2f / %.2f MiB" + % ( + info.ddrMemoryUsage.used / (1024.0 * 1024.0), + info.ddrMemoryUsage.total / (1024.0 * 1024.0), + ) + ) + print( + "Cmx used / total - %.2f / %.2f MiB" + % ( + info.cmxMemoryUsage.used / (1024.0 * 1024.0), + info.cmxMemoryUsage.total / (1024.0 * 1024.0), + ) + ) + print( + "LeonCss heap used / total - %.2f / %.2f MiB" + % ( + info.leonCssMemoryUsage.used / (1024.0 * 1024.0), + info.leonCssMemoryUsage.total / (1024.0 * 1024.0), + ) + ) + print( + "LeonMss heap used / total - %.2f / %.2f MiB" + % ( + info.leonMssMemoryUsage.used / (1024.0 * 1024.0), + info.leonMssMemoryUsage.total / (1024.0 * 1024.0), + ) + ) + t = info.chipTemperature + print( + "Chip temperature - average: %.2f, css: %.2f, mss: %.2f, upa: %.2f, dss: %.2f" + % ( + t.average, + t.css, + t.mss, + t.upa, + t.dss, + ) + ) + print( + "Cpu usage - Leon CSS: %.2f %%, Leon MSS: %.2f %%" + % ( + info.leonCssCpuUsage.average * 100, + info.leonMssCpuUsage.average * 100, + ) + ) diff --git a/rerun_py/depthai_viewer/_backend/device_configuration.py b/rerun_py/depthai_viewer/_backend/device_configuration.py index 8d5fd04655b6..d8729e8d65af 100644 --- a/rerun_py/depthai_viewer/_backend/device_configuration.py +++ b/rerun_py/depthai_viewer/_backend/device_configuration.py @@ -85,8 +85,8 @@ def out_queue_name(self) -> str: class AiModelConfiguration(BaseModel): # type: ignore[misc] - display_name: str = "Yolo V8" - path: str = "yolov8n_coco_640x352" + display_name: str = "MobileNet SSD" + path: str = "mobilenet-ssd" camera: dai.CameraBoardSocket class Config: @@ -115,16 +115,15 @@ class CameraSensorResolution(Enum): THE_480_P: str = "THE_480_P" THE_720_P: str = "THE_720_P" THE_800_P: str = "THE_800_P" + THE_5_MP: str = "THE_5_MP" + THE_1440X1080: str = "THE_1440X1080" THE_1080_P: str = "THE_1080_P" THE_1200_P: str = "THE_1200_P" + THE_4_K: str = "THE_4_K" + THE_4000X3000: str = "THE_4000X3000" THE_12_MP: str = "THE_12_MP" THE_13_MP: str = "THE_13_MP" - THE_1440X1080: str = "THE_1440X1080" - THE_4000X3000: str = "THE_4000X3000" - THE_48_MP: str = "THE_48_MP" - THE_4_K: str = "THE_4_K" THE_5312X6000: str = "THE_5312X6000" - THE_5_MP: str = "THE_5_MP" def dict(self, *args, **kwargs) -> str: # type: ignore[no-untyped-def] return self.value @@ -224,6 +223,17 @@ class PipelineConfiguration(BaseModel): # type: ignore[misc] imu: ImuConfiguration = ImuConfiguration() +class XLinkConnection(Enum): + USB = "Usb" + POE = "PoE" + + +class DeviceInfo(BaseModel): # type: ignore[misc] + name: str = "" + connection: XLinkConnection = XLinkConnection.USB + mxid: str = "" + + class DeviceProperties(BaseModel): # type: ignore[misc] id: str cameras: List[CameraFeatures] = [] @@ -232,6 +242,7 @@ class DeviceProperties(BaseModel): # type: ignore[misc] Tuple[dai.CameraBoardSocket, dai.CameraBoardSocket] ] = [] # Which cameras can be paired for stereo default_stereo_pair: Optional[Tuple[dai.CameraBoardSocket, dai.CameraBoardSocket]] = None + info: DeviceInfo = DeviceInfo() class Config: arbitrary_types_allowed = True @@ -251,22 +262,38 @@ def dict(self, *args, **kwargs) -> Dict[str, Any]: # type: ignore[no-untyped-de "cameras": [cam.dict() for cam in self.cameras], "imu": self.imu, "stereo_pairs": [(left.name, right.name) for left, right in self.stereo_pairs], + "info": { + "name": self.info.name, + "connection": self.info.connection.value, + "mxid": self.info.mxid, + }, } -resolution_to_enum = { +size_to_resolution = { (640, 400): CameraSensorResolution.THE_400_P, + (640, 480): CameraSensorResolution.THE_480_P, # OV7251 (1280, 720): CameraSensorResolution.THE_720_P, - (1280, 800): CameraSensorResolution.THE_800_P, + (1280, 800): CameraSensorResolution.THE_800_P, # OV9782 + (2592, 1944): CameraSensorResolution.THE_5_MP, # OV5645 + (1440, 1080): CameraSensorResolution.THE_1440X1080, (1920, 1080): CameraSensorResolution.THE_1080_P, - (1920, 1200): CameraSensorResolution.THE_1200_P, + (1920, 1200): CameraSensorResolution.THE_1200_P, # AR0234 (3840, 2160): CameraSensorResolution.THE_4_K, - (4056, 3040): CameraSensorResolution.THE_12_MP, - (1440, 1080): CameraSensorResolution.THE_1440X1080, - (5312, 6000): CameraSensorResolution.THE_5312X6000, + (4000, 3000): CameraSensorResolution.THE_4000X3000, # IMX582 with binning enabled + (4056, 3040): CameraSensorResolution.THE_12_MP, # IMX378, IMX477, IMX577 + (4208, 3120): CameraSensorResolution.THE_13_MP, # AR214 + (5312, 6000): CameraSensorResolution.THE_5312X6000, # IMX582 cropped } +def get_size_from_resolution(resolution: CameraSensorResolution) -> Tuple[int, int]: + for size, res in size_to_resolution.items(): + if res == resolution: + return size + raise ValueError(f"Unknown resolution {resolution}") + + def compare_dai_camera_configs(cam1: dai.CameraSensorConfig, cam2: dai.CameraSensorConfig) -> bool: return ( # type: ignore[no-any-return] cam1.height == cam2.height diff --git a/rerun_py/depthai_viewer/_backend/main.py b/rerun_py/depthai_viewer/_backend/main.py index 75d254be9aec..7c6345292d81 100644 --- a/rerun_py/depthai_viewer/_backend/main.py +++ b/rerun_py/depthai_viewer/_backend/main.py @@ -27,16 +27,15 @@ class DepthaiViewerBack: # Queues for communicating with the API process action_queue: Queue # type: ignore[type-arg] result_queue: Queue # type: ignore[type-arg] - send_message_queue: Queue # type: ignore[type-arg] def __init__(self) -> None: self.action_queue = Queue() self.result_queue = Queue() - self.send_message_queue = Queue() self.store = Store() self.api_process = threading.Thread( - target=start_api, args=(self.action_queue, self.result_queue, self.send_message_queue) + target=start_api, + args=(self.action_queue, self.result_queue, self.store._send_message_queue), # This is a bit ugly ) self.api_process.start() self.run() @@ -73,10 +72,10 @@ def on_select_device(self, device_id: str) -> Message: except RuntimeError as e: print("Failed to get device properties:", e) self.on_reset() - self.send_message_queue.put(ErrorMessage("Device disconnected!")) + self.store.send_message_to_frontend(ErrorMessage("Device disconnected!")) print("Restarting backend...") # For now exit the backend, the frontend will restart it - # (TODO(filip): Why does "Device already closed or disconnected: Input/output error happen") + # TODO(filip): Why does "Device already closed or disconnected: Input/output error happen" exit(-1) def on_update_pipeline(self, runtime_only: bool) -> Message: @@ -84,9 +83,7 @@ def on_update_pipeline(self, runtime_only: bool) -> Message: print("No device selected, can't update pipeline!") return ErrorMessage("No device selected, can't update pipeline!") print("Updating pipeline...") - message: Message = ErrorMessage("Couldn't update pipeline") - if self.store.pipeline_config is not None: - message = self._device.update_pipeline(self.store.pipeline_config, runtime_only) + message = self._device.update_pipeline(runtime_only) if isinstance(message, InfoMessage): return PipelineMessage(self.store.pipeline_config) return message @@ -134,10 +131,16 @@ def run(self) -> None: pass if self._device: - self._device.update() + try: + self._device.update() + except Exception as e: + print("Error while updating device:", e) + self.store.send_message_to_frontend(ErrorMessage("Depthai error: " + str(e))) + self.on_reset() + continue if self._device.is_closed(): self.on_reset() - self.send_message_queue.put(ErrorMessage("Device disconnected")) + self.store.send_message_to_frontend(ErrorMessage("Device disconnected")) if __name__ == "__main__": diff --git a/rerun_py/depthai_viewer/_backend/messages.py b/rerun_py/depthai_viewer/_backend/messages.py index 25da4acffd28..10358dbb8cb4 100644 --- a/rerun_py/depthai_viewer/_backend/messages.py +++ b/rerun_py/depthai_viewer/_backend/messages.py @@ -2,6 +2,8 @@ from enum import Enum from typing import List, Optional +import depthai as dai + from depthai_viewer._backend.device_configuration import DeviceProperties, PipelineConfiguration @@ -12,6 +14,7 @@ class MessageType: DEVICE = "DeviceProperties" # Get or set device ERROR = "Error" # Error message INFO = "Info" # Info message + WARNING = "Warning" # Warning message class ErrorAction(Enum): @@ -32,6 +35,14 @@ def json(self) -> str: raise NotImplementedError +class WarningMessage(Message): + def __init__(self, message: str): + self.message = message + + def json(self) -> str: + return json.dumps({"type": MessageType.WARNING, "data": {"message": self.message}}) + + class ErrorMessage(Message): def __init__(self, message: str, action: ErrorAction = ErrorAction.FULL_RESET): self.action = action @@ -42,8 +53,15 @@ def json(self) -> str: class DevicesMessage(Message): - def __init__(self, devices: List[str], message: Optional[str] = None): - self.devices = devices + def __init__(self, devices: List[dai.DeviceInfo], message: Optional[str] = None): + self.devices = [ + { + "mxid": d.getMxId(), + "connection": "PoE" if d.protocol == dai.XLinkProtocol.X_LINK_TCP_IP else "Usb", + "name": d.name, + } + for d in devices + ] self.message = message def json(self) -> str: diff --git a/rerun_py/depthai_viewer/_backend/packet_handler.py b/rerun_py/depthai_viewer/_backend/packet_handler.py index 6dd1c3f02a39..ea3cfca0e1c8 100644 --- a/rerun_py/depthai_viewer/_backend/packet_handler.py +++ b/rerun_py/depthai_viewer/_backend/packet_handler.py @@ -14,7 +14,6 @@ ) from numpy.typing import NDArray from pydantic import BaseModel -from turbojpeg import TJFLAG_FASTDCT, TJFLAG_FASTUPSAMPLE, TurboJPEG import depthai_viewer as viewer from depthai_viewer._backend.device_configuration import CameraConfiguration @@ -46,13 +45,13 @@ class Config: class SyncedCallbackArgs(BaseModel): # type: ignore[misc] depth_args: Optional[DepthCallbackArgs] = None + ai_args: Optional[AiModelCallbackArgs] = None class PacketHandler: store: Store _ahrs: Mahony _get_camera_intrinsics: Callable[[dai.CameraBoardSocket, int, int], NDArray[np.float32]] - _jpeg_decoder: TurboJPEG = TurboJPEG() def __init__( self, store: Store, intrinsics_getter: Callable[[dai.CameraBoardSocket, int, int], NDArray[np.float32]] @@ -89,6 +88,14 @@ def _on_synced_packets(self, args: SyncedCallbackArgs, packets: Dict[str, Any]) if args.depth_args is None: continue self._on_stereo_frame(packet, args.depth_args) + elif type(packet) is DetectionPacket: + if args.ai_args is None: + continue + self._on_detections(packet, args.ai_args) + elif type(packet) is TwoStagePacket: + if args.ai_args is None: + continue + self._on_age_gender_packet(packet, args.ai_args) def build_callback( self, args: Union[dai.CameraBoardSocket, DepthCallbackArgs, AiModelCallbackArgs] @@ -105,8 +112,16 @@ def build_callback( raise ValueError(f"Unknown callback args type: {type(args)}") def _on_camera_frame(self, packet: FramePacket, board_socket: dai.CameraBoardSocket) -> None: - viewer.log_rigid3(f"{board_socket.name}/transform", child_from_parent=([0, 0, 0], self._ahrs.Q), xyz="RDF") + viewer.log_rigid3( + f"{board_socket.name}/transform", child_from_parent=([0, 0, 0], [1, 0, 0, 0]), xyz="RDF" + ) # TODO(filip): Enable the user to lock the camera rotation in the UI + + img_frame = packet.frame if packet.msg.getType() == dai.RawImgFrame.Type.RAW8 else packet.msg.getData() h, w = packet.msg.getHeight(), packet.msg.getWidth() + if packet.msg.getType() == dai.ImgFrame.Type.BITSTREAM: + img_frame = cv2.cvtColor(cv2.imdecode(img_frame, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + h, w = img_frame.shape[:2] + child_from_parent: NDArray[np.float32] try: child_from_parent = self._get_camera_intrinsics( # type: ignore[call-arg, misc, arg-type] @@ -122,12 +137,7 @@ def _on_camera_frame(self, packet: FramePacket, board_socket: dai.CameraBoardSoc width=w, height=h, ) - img_frame = packet.frame if packet.msg.getType() == dai.RawImgFrame.Type.RAW8 else packet.msg.getData() entity_path = f"{board_socket.name}/transform/{cam}/Image" - if packet.msg.getType() == dai.ImgFrame.Type.BITSTREAM: - img_frame = cv2.cvtColor( - self._jpeg_decoder.decode(img_frame, flags=TJFLAG_FASTUPSAMPLE | TJFLAG_FASTDCT), cv2.COLOR_BGR2RGB - ) if packet.msg.getType() == dai.RawImgFrame.Type.NV12: viewer.log_encoded_image( @@ -180,7 +190,7 @@ def _detections_to_rects_colors_labels( colors = [] labels = [] for detection in packet.detections: - rects.append(self._rect_from_detection(detection)) + rects.append(self._rect_from_detection(detection, packet.frame.shape[0], packet.frame.shape[1])) colors.append([0, 255, 0]) label: str = detection.label # Open model zoo models output label index @@ -202,16 +212,18 @@ def _on_age_gender_packet(self, packet: TwoStagePacket, args: AiModelCallbackArg cam = cam_kind_from_sensor_kind(args.camera.kind) viewer.log_rect( f"{args.camera.board_socket.name}/transform/{cam}/Detection", - self._rect_from_detection(det), + self._rect_from_detection(det, packet.frame.shape[0], packet.frame.shape[1]), rect_format=RectFormat.XYXY, color=color, label=label, ) - def _rect_from_detection(self, detection: _Detection) -> List[int]: + def _rect_from_detection(self, detection: _Detection, max_height: int, max_width: int) -> List[int]: return [ - *detection.bottom_right, - *detection.top_left, + max(min(detection.bottom_right[0], max_width), 0), + max(min(detection.bottom_right[1], max_height), 0), + max(min(detection.top_left[0], max_width), 0), + max(min(detection.top_left[1], max_height), 0), ] diff --git a/rerun_py/depthai_viewer/_backend/store.py b/rerun_py/depthai_viewer/_backend/store.py index c5756b56c089..dbf074715e5a 100644 --- a/rerun_py/depthai_viewer/_backend/store.py +++ b/rerun_py/depthai_viewer/_backend/store.py @@ -1,12 +1,20 @@ +from multiprocessing import Queue from typing import List, Optional from depthai_viewer._backend.device_configuration import PipelineConfiguration +from depthai_viewer._backend.messages import Message from depthai_viewer._backend.topic import Topic class Store: + """Used to store common data that is used by the backend.""" + _pipeline_config: Optional[PipelineConfiguration] = None _subscriptions: List[Topic] = [] + _send_message_queue: Queue # type: ignore[type-arg] + + def __init__(self) -> None: + self._send_message_queue = Queue() def set_pipeline_config(self, pipeline_config: PipelineConfiguration) -> None: self._pipeline_config = pipeline_config @@ -25,3 +33,6 @@ def pipeline_config(self) -> Optional[PipelineConfiguration]: @property def subscriptions(self) -> List[Topic]: return self._subscriptions + + def send_message_to_frontend(self, message: Message) -> None: + self._send_message_queue.put(message) diff --git a/rerun_py/depthai_viewer/requirements.txt b/rerun_py/depthai_viewer/requirements.txt index 6951baf051db..9db721fae628 100644 --- a/rerun_py/depthai_viewer/requirements.txt +++ b/rerun_py/depthai_viewer/requirements.txt @@ -7,4 +7,3 @@ depthai==2.22.0.0 websockets pydantic deprecated -pyturbojpeg==1.7.1