diff --git a/crates/re_log_types/src/component_types/mod.rs b/crates/re_log_types/src/component_types/mod.rs index b1e4ddec13a6..8f5b9894b570 100644 --- a/crates/re_log_types/src/component_types/mod.rs +++ b/crates/re_log_types/src/component_types/mod.rs @@ -65,7 +65,7 @@ pub use scalar::{Scalar, ScalarPlotProps}; pub use size::Size3D; pub use tensor::{ DecodedTensor, Tensor, TensorCastError, TensorData, TensorDataMeaning, TensorDimension, - TensorId, + TensorId, TensorColormap }; #[cfg(feature = "image")] pub use tensor::{TensorImageLoadError, TensorImageSaveError}; diff --git a/crates/re_log_types/src/component_types/tensor.rs b/crates/re_log_types/src/component_types/tensor.rs index 06c1d8401ca7..bdfff6de3ca0 100644 --- a/crates/re_log_types/src/component_types/tensor.rs +++ b/crates/re_log_types/src/component_types/tensor.rs @@ -342,6 +342,20 @@ pub enum TensorDataMeaning { Depth, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ArrowField, ArrowSerialize, ArrowDeserialize)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +#[arrow_field(type = "dense")] +pub enum TensorColormap { + None, // Not sure how to serialize an Option with pyarrow. + Grayscale, + Inferno, + Magma, + Plasma, + #[default] + Turbo, + Viridis +} + /// A Multi-dimensional Tensor. /// /// All clones are shallow. @@ -400,6 +414,10 @@ pub struct Tensor { /// Reciprocal scale of meter unit for depth images pub meter: Option, + + pub colormap: TensorColormap, + + pub unit: Option, } impl Tensor { @@ -626,6 +644,8 @@ macro_rules! tensor_type { data: TensorData::$variant(Vec::from(slice).into()), meaning: TensorDataMeaning::Unknown, meter: None, + colormap: TensorColormap::None, + unit: None, }), None => Ok(Tensor { tensor_id: TensorId::random(), @@ -633,6 +653,8 @@ macro_rules! tensor_type { data: TensorData::$variant(view.iter().cloned().collect::>().into()), meaning: TensorDataMeaning::Unknown, meter: None, + colormap: TensorColormap::None, + unit: None, }), } } @@ -658,6 +680,8 @@ macro_rules! tensor_type { data: TensorData::$variant(value.into_raw_vec().into()), meaning: TensorDataMeaning::Unknown, meter: None, + colormap: TensorColormap::None, + unit: None, }) .ok_or(TensorCastError::NotContiguousStdOrder) } @@ -749,7 +773,9 @@ impl Tensor { shape: Vec, data: TensorData, meaning: TensorDataMeaning, - meter: Option + meter: Option, + colormap: TensorColormap, + unit: Option, ) -> Self { Self { tensor_id, @@ -757,6 +783,8 @@ impl Tensor { data, meaning, meter, + colormap, + unit, } } } @@ -796,6 +824,8 @@ impl Tensor { data: TensorData::JPEG(jpeg_bytes.into()), meaning: TensorDataMeaning::Unknown, meter: None, + colormap: TensorColormap::None, + unit: None, }) } @@ -1036,6 +1066,8 @@ impl DecodedTensor { data, meaning: TensorDataMeaning::Unknown, meter: None, + colormap: TensorColormap::None, + unit: None, }; Ok(DecodedTensor(tensor)) } @@ -1121,6 +1153,8 @@ fn test_ndarray() { data: TensorData::U16(vec![1, 2, 3, 4].into()), meaning: TensorDataMeaning::Unknown, meter: None, + colormap: TensorColormap::None, + unit: None, }; let a0: ndarray::ArrayViewD<'_, u16> = (&t0).try_into().unwrap(); dbg!(a0); // NOLINT @@ -1144,6 +1178,8 @@ fn test_arrow() { data: TensorData::U16(vec![1, 2, 3, 4].into()), meaning: TensorDataMeaning::Unknown, meter: Some(1000.0), + colormap: TensorColormap::None, + unit: None, }, Tensor { tensor_id: TensorId(std::default::Default::default()), @@ -1154,6 +1190,8 @@ fn test_arrow() { data: TensorData::F32(vec![1.23, 2.45].into()), meaning: TensorDataMeaning::Unknown, meter: None, + colormap: TensorColormap::None, + unit: None, } ]; diff --git a/crates/re_sdk/src/lib.rs b/crates/re_sdk/src/lib.rs index bddf42773f19..a64d155a214c 100644 --- a/crates/re_sdk/src/lib.rs +++ b/crates/re_sdk/src/lib.rs @@ -78,7 +78,7 @@ pub mod components { MeshFormat, MeshId, Pinhole, Point2D, Point3D, Quaternion, Radius, RawMesh3D, Rect2D, Rigid3, Scalar, ScalarPlotProps, Size3D, Tensor, TensorData, TensorDataMeaning, TensorDimension, TensorId, TextEntry, Transform, Vec2D, Vec3D, Vec4D, - ViewCoordinates, + ViewCoordinates, TensorColormap }; } diff --git a/crates/re_viewer/pipeline.json b/crates/re_viewer/pipeline.json new file mode 100644 index 000000000000..89d59abee35c --- /dev/null +++ b/crates/re_viewer/pipeline.json @@ -0,0 +1 @@ +{"assetStorage": [], "assets": {"map": {}}, "pipeline": {"connections": [{"node1Id": 3, "node1Output": "out", "node1OutputGroup": "", "node2Id": 2, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "raw", "node1OutputGroup": "", "node2Id": 1, "node2Input": "in", "node2InputGroup": ""}], "globalProperties": {"calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1}, "nodes": [[0, {"id": 0, "ioInfo": [[["", "video"], {"blocking": false, "group": "", "id": 11, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "still"], {"blocking": false, "group": "", "id": 9, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "isp"], {"blocking": false, "group": "", "id": 8, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "preview"], {"blocking": false, "group": "", "id": 10, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 7, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "frameEvent"], {"blocking": false, "group": "", "id": 6, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "inputConfig"], {"blocking": false, "group": "", "id": 5, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 4, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false}]], "name": "Camera", "properties": {"boardSocket": 4, "calibAlpha": null, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 30.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": {"height": 0, "priority": 0, "width": 0, "x": 0, "y": 0}, "afRegion": {"height": 0, "priority": 0, "width": 0, "x": 0, "y": 0}, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": {"exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0}, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": {"activeLevel": 0, "enable": 0, "gpioNumber": 0}, "strobeTimings": {"durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0}, "wbColorTemp": 0}, "interleaved": true, "isp3aFps": 0, "ispScale": {"horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0}, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolutionHeight": -1, "resolutionWidth": -1, "sensorCropX": -1.0, "sensorCropY": -1.0, "sensorType": -1, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1, "warpMeshHeight": 0, "warpMeshSource": -1, "warpMeshStepHeight": 32, "warpMeshStepWidth": 32, "warpMeshUri": "", "warpMeshWidth": 0}}], [1, {"id": 1, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 3, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true}]], "name": "XLinkOut", "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "thermal_cam"}}], [2, {"id": 2, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 2, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true}]], "name": "XLinkOut", "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "sys_logger"}}], [3, {"id": 3, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 1, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}]], "name": "SystemLogger", "properties": {"rateHz": 0.10000000149011612}}]]}} \ No newline at end of file diff --git a/crates/re_viewer/src/depthai/depthai.rs b/crates/re_viewer/src/depthai/depthai.rs index a7761b06547f..a1827714b99a 100644 --- a/crates/re_viewer/src/depthai/depthai.rs +++ b/crates/re_viewer/src/depthai/depthai.rs @@ -122,6 +122,7 @@ pub enum ImuKind { #[derive(serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Debug)] #[allow(non_camel_case_types)] pub enum CameraSensorResolution { + THE_256X192, THE_400_P, THE_480_P, THE_720_P, @@ -142,6 +143,7 @@ pub enum CameraSensorResolution { impl fmt::Display for CameraSensorResolution { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::THE_256X192 => write!(f, "256x192"), Self::THE_400_P => write!(f, "400p"), Self::THE_480_P => write!(f, "480p"), Self::THE_720_P => write!(f, "720p"), @@ -322,7 +324,7 @@ pub struct DeviceConfig { pub depth_enabled: bool, // Much easier to have an explicit bool for checkbox #[serde(default = "StereoDepthConfig::default_as_option")] pub depth: Option, - pub ai_model: AiModel, + pub ai_model: Option, #[serde(skip)] pub dot_brightness: u32, #[serde(skip)] @@ -336,7 +338,7 @@ impl Default for DeviceConfig { cameras: Vec::new(), depth_enabled: true, depth: Some(StereoDepthConfig::default()), - ai_model: AiModel::default(), + ai_model: None, dot_brightness: 0, flood_brightness: 0, } @@ -370,7 +372,7 @@ impl From<&DeviceProperties> for DeviceConfig { }) .collect(); config.depth = Option::::from(props); - config.ai_model = AiModel::from(props); + config.ai_model = Option::::from(props); config } } @@ -498,7 +500,10 @@ pub struct AiModel { impl Default for AiModel { fn default() -> Self { - default_neural_networks()[2].clone() + match default_neural_networks()[2].clone() { + Some(model) => model, + None => panic!("Default neural network not found!") + } } } @@ -512,16 +517,16 @@ impl AiModel { } } -impl From<&DeviceProperties> for AiModel { +impl From<&DeviceProperties> for Option { fn from(props: &DeviceProperties) -> Self { - let mut model = Self::default(); + let mut model = AiModel::default(); if let Some(cam) = props.cameras.iter().find(|cam| cam.is_color_camera()) { model.camera = cam.board_socket; - } else if let Some(cam) = props.cameras.first() { - model.camera = cam.board_socket; + Some(model) + } else { + None } - model } } @@ -582,8 +587,8 @@ pub struct State { pub backend_comms: BackendCommChannel, #[serde(skip)] poll_instant: Option, - #[serde(default = "default_neural_networks")] - pub neural_networks: Vec, + #[serde(skip, default = "default_neural_networks")] + pub neural_networks: Vec>, #[serde(skip)] update_timeout_timer: Option, #[serde(skip, default = "bool_true")] @@ -597,29 +602,34 @@ fn all_subscriptions() -> Vec { } #[inline] -fn default_neural_networks() -> Vec { +fn default_neural_networks() -> Vec> { vec![ - AiModel::none(), - AiModel { + None, + Some(AiModel { path: String::from("yolov8n_coco_640x352"), display_name: String::from("Yolo V8"), camera: CameraBoardSocket::CAM_A, - }, - AiModel { + }), + Some(AiModel { path: String::from("mobilenet-ssd"), display_name: String::from("MobileNet SSD"), camera: CameraBoardSocket::CAM_A, - }, - AiModel { + }), + Some(AiModel { path: String::from("face-detection-retail-0004"), display_name: String::from("Face Detection"), camera: CameraBoardSocket::CAM_A, - }, - AiModel { + }), + Some(AiModel { path: String::from("age-gender-recognition-retail-0013"), display_name: String::from("Age gender recognition"), camera: CameraBoardSocket::CAM_A, - }, + }), + Some(AiModel { + path: String::from("yolov6n_thermal_people_256x192"), + display_name: String::from("Thermal Person Detection"), + camera: CameraBoardSocket::CAM_E + }), ] } diff --git a/crates/re_viewer/src/gpu_bridge/tensor_to_gpu.rs b/crates/re_viewer/src/gpu_bridge/tensor_to_gpu.rs index f9d5b6a2a83b..0bc97d33f450 100644 --- a/crates/re_viewer/src/gpu_bridge/tensor_to_gpu.rs +++ b/crates/re_viewer/src/gpu_bridge/tensor_to_gpu.rs @@ -8,7 +8,7 @@ use bytemuck::{ allocation::pod_collect_to_vec, cast_slice, Pod }; use egui::util::hash; use wgpu::TextureFormat; -use re_log_types::component_types::{ DecodedTensor, Tensor, TensorData }; +use re_log_types::component_types::{ DecodedTensor, Tensor, TensorData, TensorColormap }; use re_renderer::{ renderer::{ ColorMapper, ColormappedTexture, TextureEncoding }, resource_managers::Texture2DCreationDesc, @@ -148,8 +148,16 @@ fn color_tensor_to_gpu( encoding != Some(TextureEncoding::Nv12) && re_renderer::texture_info::num_texture_components(texture_format) == 1 { - // Single-channel images = luminance = grayscale - Some(ColorMapper::Function(re_renderer::Colormap::Grayscale)) + match tensor.inner().colormap { + TensorColormap::Grayscale => Some(ColorMapper::Function(re_renderer::Colormap::Grayscale)), + TensorColormap::Viridis => Some(ColorMapper::Function(re_renderer::Colormap::Viridis)), + TensorColormap::Plasma => Some(ColorMapper::Function(re_renderer::Colormap::Plasma)), + TensorColormap::Inferno => Some(ColorMapper::Function(re_renderer::Colormap::Inferno)), + TensorColormap::Magma => Some(ColorMapper::Function(re_renderer::Colormap::Magma)), + TensorColormap::Turbo => Some(ColorMapper::Function(re_renderer::Colormap::Turbo)), + // Single-channel images = luminance = grayscale + TensorColormap::None => Some(ColorMapper::Function(re_renderer::Colormap::Grayscale)), + } } else { None }; diff --git a/crates/re_viewer/src/ui/data_ui/image.rs b/crates/re_viewer/src/ui/data_ui/image.rs index 8746e87130e6..89b1610f2a03 100644 --- a/crates/re_viewer/src/ui/data_ui/image.rs +++ b/crates/re_viewer/src/ui/data_ui/image.rs @@ -220,7 +220,7 @@ pub fn tensor_summary_ui_grid_contents( tensor: &Tensor, tensor_stats: &TensorStats ) { - let Tensor { tensor_id: _, shape, data, meaning, meter } = tensor; + let Tensor { tensor_id: _, shape, data, meaning, meter, colormap: _, unit: _ } = tensor; re_ui .grid_left_hand_label(ui, "Data type") @@ -565,7 +565,20 @@ fn tensor_pixel_value_ui( _ => unreachable!("NV12 should only contain u8"), } }), - _ => tensor.get(&[y, x]).map(|v| format!("Val: {v}")), + _ => { + match &tensor.unit { + Some(unit) => { + if tensor.dtype().is_float() { + tensor.get(&[y, x]).map(|v| format!("Val: {v:.2} {unit}")) + } else { + tensor.get(&[y, x]).map(|v| format!("Val: {v} {unit}")) + } + }, + None => { + tensor.get(&[y, x]).map(|v| format!("Val: {v}")) + } + } + } } 3 => match tensor.real_shape().as_slice()[2].size { diff --git a/crates/re_viewer/src/ui/device_settings_panel.rs b/crates/re_viewer/src/ui/device_settings_panel.rs index 263d923db422..ac4f61e546c5 100644 --- a/crates/re_viewer/src/ui/device_settings_panel.rs +++ b/crates/re_viewer/src/ui/device_settings_panel.rs @@ -252,7 +252,10 @@ impl DeviceSettingsPanel { ctx.re_ui.labeled_combo_box( ui, "AI Model", - device_config.ai_model.display_name.clone(), + match &device_config.ai_model { + Some(model) => model.display_name.clone(), + None => "No Model".to_string(), + }, false, true, |ui| { @@ -260,40 +263,48 @@ impl DeviceSettingsPanel { ui.selectable_value( &mut device_config.ai_model, nn.clone(), - &nn.display_name, - ); - } - }, - ); - ctx.re_ui.labeled_combo_box( - ui, - "Run on", - device_config.ai_model.camera.display_name(ctx), - false, - true, - |ui| { - let filtered_cameras: Vec<_> = connected_cameras - .iter() // iterates over references - .filter(|cam| { - !(cam.supported_types.contains( - &depthai::CameraSensorKind::THERMAL, - ) || cam.supported_types.contains( - &depthai::CameraSensorKind::TOF, - )) - }) - .collect(); - for cam in filtered_cameras { - ui.selectable_value( - &mut device_config.ai_model.camera, - cam.board_socket, - cam.board_socket.display_name(ctx), + match &nn { + Some(model) => model.display_name.clone(), + None => "No Model".to_string(), + }, ); } }, ); + match &mut device_config.ai_model { + Some(model) => { + ctx.re_ui.labeled_combo_box( + ui, + "Run on", + model.camera.display_name(ctx), + false, + true, + |ui| { + let filtered_cameras: Vec<_> = connected_cameras + .iter() // iterates over references + .filter(|cam| { + !(cam.supported_types.contains( + &depthai::CameraSensorKind::THERMAL, + ) || cam.supported_types.contains( + &depthai::CameraSensorKind::TOF, + )) + }) + .collect(); + for cam in filtered_cameras { + ui.selectable_value( + &mut model.camera, + cam.board_socket, + cam.board_socket.display_name(ctx), + ); + } + }, + ); + } + None => { + } + }; }); - }, - ); + }); let mut depth = device_config.depth.unwrap_or_default(); ui.add_enabled_ui( diff --git a/nonworking.log b/nonworking.log new file mode 100644 index 000000000000..fbe896707f11 --- /dev/null +++ b/nonworking.log @@ -0,0 +1,37 @@ +Selecting device: 194430100196762700 +Oak cam: +No intrisics for cam: CameraBoardSocket.CAM_A +No intrisics for cam: CameraBoardSocket.CAM_E +Default stereo pair: None +Pipeline config: auto=True cameras=[CameraConfiguration(fps=30, resolution=, kind=, board_socket=, stream_enabled=True, name='Rgb'), CameraConfiguration(fps=30, resolution=, kind=, board_socket=, stream_enabled=True, name='Thermal')] depth=None ai_model=None imu=ImuConfiguration(report_rate=100, batch_report_threshold=5) +Update pipeline: auto=True cameras=[CameraConfiguration(fps=30, resolution=, kind=, board_socket=, stream_enabled=True, name='Rgb'), CameraConfiguration(fps=30, resolution=, kind=, board_socket=, stream_enabled=True, name='Thermal')] depth=None ai_model=None imu=ImuConfiguration(report_rate=100, batch_report_threshold=5) +Updating pipeline... +Usb speed: UsbSpeed.HIGH +Device is connected in USB2 mode, camera streams will be JPEG encoded... +dai version: 2.24.0.0.dev0+c014e27e224f7ef3f6407be6b3f05be6c2fffd13 +AI: None +Connected camera features [{socket: CAM_A, sensorName: IMX462, width: 1920, height: 1080, orientation: ROTATE_180_DEG, supportedTypes: [COLOR], hasAutofocus: 0, hasAutofocusIC: 0, name: rgb}, {socket: CAM_E, sensorName: TINY1C, width: 256, height: 192, orientation: AUTO, supportedTypes: [THERMAL], hasAutofocus: 0, hasAutofocusIC: 0, name: thermal}] +Creating camera: fps=30 resolution= kind= board_socket= stream_enabled=True name='Thermal' +Creating camera: fps=30 resolution= kind= board_socket= stream_enabled=False name='Color' +Starting pipeline +{'assetStorage': [], 'assets': {'map': {}}, 'pipeline': {'connections': [{'node1Id': 3, 'node1Output': 'out', 'node1OutputGroup': '', 'node2Id': 2, 'node2Input': 'in', 'node2InputGroup': ''}, {'node1Id': 0, 'node1Output': 'raw', 'node1OutputGroup': '', 'node2Id': 1, 'node2Input': 'in', 'node2InputGroup': ''}], 'globalProperties': {'calibData': None, 'cameraTuningBlobSize': None, 'cameraTuningBlobUri': '', 'leonCssFrequencyHz': 700000000.0, 'leonMssFrequencyHz': 700000000.0, 'pipelineName': None, 'pipelineVersion': None, 'sippBufferSize': 18432, 'sippDmaBufferSize': 16384, 'xlinkChunkSize': -1}, 'nodes': [[0, {'id': 0, 'ioInfo': [[['', 'video'], {'blocking': False, 'group': '', 'id': 11, 'name': 'video', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'still'], {'blocking': False, 'group': '', 'id': 9, 'name': 'still', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'isp'], {'blocking': False, 'group': '', 'id': 8, 'name': 'isp', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'preview'], {'blocking': False, 'group': '', 'id': 10, 'name': 'preview', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'raw'], {'blocking': False, 'group': '', 'id': 7, 'name': 'raw', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'frameEvent'], {'blocking': False, 'group': '', 'id': 6, 'name': 'frameEvent', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'inputConfig'], {'blocking': False, 'group': '', 'id': 5, 'name': 'inputConfig', 'queueSize': 8, 'type': 3, 'waitForMessage': False}], [['', 'inputControl'], {'blocking': True, 'group': '', 'id': 4, 'name': 'inputControl', 'queueSize': 8, 'type': 3, 'waitForMessage': False}]], 'name': 'Camera', 'properties': {'boardSocket': 4, 'calibAlpha': None, 'cameraName': '', 'colorOrder': 0, 'fp16': False, 'fps': 30.0, 'imageOrientation': -1, 'initialControl': {'aeLockMode': False, 'aeMaxExposureTimeUs': 0, 'aeRegion': {'height': 0, 'priority': 0, 'width': 0, 'x': 0, 'y': 0}, 'afRegion': {'height': 0, 'priority': 0, 'width': 0, 'x': 0, 'y': 0}, 'antiBandingMode': 0, 'autoFocusMode': 3, 'awbLockMode': False, 'awbMode': 0, 'brightness': 0, 'captureIntent': 0, 'chromaDenoise': 0, 'cmdMask': 0, 'contrast': 0, 'controlMode': 0, 'effectMode': 0, 'expCompensation': 0, 'expManual': {'exposureTimeUs': 0, 'frameDurationUs': 0, 'sensitivityIso': 0}, 'frameSyncMode': 0, 'lensPosAutoInfinity': 0, 'lensPosAutoMacro': 0, 'lensPosition': 0, 'lensPositionRaw': 0.0, 'lowPowerNumFramesBurst': 0, 'lowPowerNumFramesDiscard': 0, 'lumaDenoise': 0, 'saturation': 0, 'sceneMode': 0, 'sharpness': 0, 'strobeConfig': {'activeLevel': 0, 'enable': 0, 'gpioNumber': 0}, 'strobeTimings': {'durationUs': 0, 'exposureBeginOffsetUs': 0, 'exposureEndOffsetUs': 0}, 'wbColorTemp': 0}, 'interleaved': True, 'isp3aFps': 0, 'ispScale': {'horizDenominator': 0, 'horizNumerator': 0, 'vertDenominator': 0, 'vertNumerator': 0}, 'numFramesPoolIsp': 3, 'numFramesPoolPreview': 4, 'numFramesPoolRaw': 3, 'numFramesPoolStill': 4, 'numFramesPoolVideo': 4, 'previewHeight': 300, 'previewKeepAspectRatio': True, 'previewWidth': 300, 'rawPacked': None, 'resolutionHeight': -1, 'resolutionWidth': -1, 'sensorCropX': -1.0, 'sensorCropY': -1.0, 'sensorType': -1, 'stillHeight': -1, 'stillWidth': -1, 'videoHeight': -1, 'videoWidth': -1, 'warpMeshHeight': 0, 'warpMeshSource': -1, 'warpMeshStepHeight': 32, 'warpMeshStepWidth': 32, 'warpMeshUri': '', 'warpMeshWidth': 0}}], [1, {'id': 1, 'ioInfo': [[['', 'in'], {'blocking': True, 'group': '', 'id': 3, 'name': 'in', 'queueSize': 8, 'type': 3, 'waitForMessage': True}]], 'name': 'XLinkOut', 'properties': {'maxFpsLimit': -1.0, 'metadataOnly': False, 'streamName': 'thermal_cam'}}], [2, {'id': 2, 'ioInfo': [[['', 'in'], {'blocking': True, 'group': '', 'id': 2, 'name': 'in', 'queueSize': 8, 'type': 3, 'waitForMessage': True}]], 'name': 'XLinkOut', 'properties': {'maxFpsLimit': -1.0, 'metadataOnly': False, 'streamName': 'sys_logger'}}], [3, {'id': 3, 'ioInfo': [[['', 'out'], {'blocking': False, 'group': '', 'id': 1, 'name': 'out', 'queueSize': 8, 'type': 0, 'waitForMessage': False}]], 'name': 'SystemLogger', 'properties': {'rateHz': 0.10000000149011612}}]]}} +[194430100196762700] [1.1.4.3.2] [6.050] [Camera(0)] [warning] PlgSrcTiny1c::Create: Capping fps from 30 to 25 +Got message to send: +Sending message: +Error while updating device:Traceback (most recent call last): + + File "/home/filip/Documents/depthai-viewer/rerun_py/depthai_viewer/_backend/main.py", line 167, in run + self._device.update() + + File "/home/filip/Documents/depthai-viewer/rerun_py/depthai_viewer/_backend/device.py", line 588, in update + packet = queue.tryGet() + +RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'thermal_cam' (X_LINK_ERROR)' + +Resetting... +Closing device... +[2024-02-22T14:23:38Z WARN re_viewer::depthai::depthai] Device is connected in USB2 mode. This may cause performance issues. +Done +Got message to send: +Sending message: +[2024-02-22T14:23:39Z ERROR re_viewer::depthai::depthai] Error: Depthai error: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'thermal_cam' (X_LINK_ERROR)' diff --git a/rerun_py/depthai_viewer/__init__.py b/rerun_py/depthai_viewer/__init__.py index aaa3020afa9a..53d023592f0b 100644 --- a/rerun_py/depthai_viewer/__init__.py +++ b/rerun_py/depthai_viewer/__init__.py @@ -6,7 +6,7 @@ import depthai_viewer_bindings as bindings # type: ignore[attr-defined] from depthai_viewer import _backend -from depthai_viewer.components.tensor import ImageEncoding +from depthai_viewer.components.tensor import Colormap, ImageEncoding from depthai_viewer.log import log_cleared from depthai_viewer.log.annotation import ( AnnotationInfo, @@ -95,6 +95,7 @@ "_backend", "rerun_shutdown", "ImageEncoding", + "Colormap", ] diff --git a/rerun_py/depthai_viewer/_backend/config_api.py b/rerun_py/depthai_viewer/_backend/config_api.py index 63dd28a08480..075b09519882 100644 --- a/rerun_py/depthai_viewer/_backend/config_api.py +++ b/rerun_py/depthai_viewer/_backend/config_api.py @@ -162,7 +162,10 @@ async def main(port: int = 9001) -> None: def start_api( - _dispatch_action_queue: Queue, _result_queue: Queue, _send_message_queue: Queue, port: int = 9001 # type: ignore[type-arg] + _dispatch_action_queue: Queue, # type: ignore[type-arg] + _result_queue: Queue, # type: ignore[type-arg] + _send_message_queue: Queue, # type: ignore[type-arg] + port: int = 9001, ) -> None: """ Starts the websocket API. diff --git a/rerun_py/depthai_viewer/_backend/device.py b/rerun_py/depthai_viewer/_backend/device.py index 55560699a259..450d1a5ccaa8 100644 --- a/rerun_py/depthai_viewer/_backend/device.py +++ b/rerun_py/depthai_viewer/_backend/device.py @@ -1,4 +1,5 @@ import itertools +import os import time from queue import Empty as QueueEmpty from queue import Queue @@ -32,14 +33,16 @@ get_size_from_resolution, size_to_resolution, ) +from depthai_viewer._backend.device_defaults import oak_t_default from depthai_viewer._backend.messages import ( ErrorMessage, InfoMessage, Message, WarningMessage, ) -from depthai_viewer._backend.packet_handler import PacketHandler +from depthai_viewer._backend.packet_handler import DetectionContext, PacketHandler, PacketHandlerContext from depthai_viewer._backend.store import Store +from depthai_viewer.install_requirements import model_dir class XlinkStatistics: @@ -75,13 +78,13 @@ class Device: _packet_handler: PacketHandler _oak: Optional[OakCamera] = None - _cameras: List[CameraComponent] = [] _stereo: StereoComponent = None _nnet: NNComponent = None _xlink_statistics: Optional[XlinkStatistics] = None _sys_info_q: Optional[Queue] = None # type: ignore[type-arg] _pipeline_start_t: Optional[float] = None _queues: List[Tuple[Component, ComponentOutput]] = [] + _dai_queues: List[Tuple[dai.Node, dai.DataOutputQueue, Optional[PacketHandlerContext]]] = [] # _profiler = cProfile.Profile() @@ -171,9 +174,9 @@ def get_device_properties(self) -> DeviceProperties: device_info = self._oak.device.getDeviceInfo() device_info = DeviceInfo( name=device_info.name, - connection=XLinkConnection.POE - if device_info.protocol == dai.XLinkProtocol.X_LINK_TCP_IP - else XLinkConnection.USB, + connection=( + XLinkConnection.POE if device_info.protocol == dai.XLinkProtocol.X_LINK_TCP_IP else XLinkConnection.USB + ), mxid=device_info.mxid, ) device_properties = DeviceProperties(id=self.id, imu=imu, info=device_info) @@ -238,8 +241,8 @@ def get_device_properties(self) -> DeviceProperties: def close_oak(self) -> None: if self._oak is None: return - if self._oak.running(): - self._oak.device.__exit__(0, 0, 0) + if self._oak.device: + self._oak.device.close() def reconnect_to_oak(self) -> Message: """ @@ -270,7 +273,12 @@ def reconnect_to_oak(self) -> Message: return ErrorMessage("Failed to create oak camera") def _get_component_by_socket(self, socket: dai.CameraBoardSocket) -> Optional[CameraComponent]: - component = list(filter(lambda c: c.node.getBoardSocket() == socket, self._cameras)) + component = list( + filter( # type: ignore[arg-type] + lambda c: isinstance(c, CameraComponent) and c.node.getBoardSocket() == socket, + self._oak._components if self._oak else [], + ) + ) if not component: return None return component[0] @@ -285,6 +293,7 @@ def _get_camera_config_by_socket( return camera[0] def _create_auto_pipeline_config(self, config: PipelineConfiguration) -> Message: + print("Creating auto pipeline config") if self._oak is None: return ErrorMessage("Oak device unavailable, can't create auto pipeline config!") if self._oak.device is None: @@ -346,7 +355,7 @@ def _create_auto_pipeline_config(self, config: PipelineConfiguration) -> Message config.ai_model = ALL_NEURAL_NETWORKS[1] # Mobilenet SSd config.ai_model.camera = nnet_cam_sock else: - config.ai_model = None + config.ai_model = ALL_NEURAL_NETWORKS[1] return InfoMessage("Created auto pipeline config") def update_pipeline(self, runtime_only: bool) -> Message: @@ -369,10 +378,16 @@ def update_pipeline(self, runtime_only: bool) -> Message: if isinstance(message, ErrorMessage): return message + self._queues = [] + self._dai_queues = [] + if config.auto: - self._create_auto_pipeline_config(config) + if self._oak.device.getDeviceName() == "OAK-T": + config = oak_t_default.config + else: + self._create_auto_pipeline_config(config) - self._cameras = [] + create_dai_queues_after_start: Dict[str, Tuple[dai.Node, Optional[PacketHandlerContext]]] = {} self._stereo = None self._packet_handler.reset() self._sys_info_q = None @@ -434,6 +449,16 @@ def update_pipeline(self, runtime_only: bool) -> Message: if cam.stream_enabled: if dai.CameraSensorType.TOF in camera_features.supportedTypes: sdk_cam = self._oak.create_tof(cam.board_socket) + self._queues.append((sdk_cam, self._oak.queue(sdk_cam.out.main))) + elif dai.CameraSensorType.THERMAL in camera_features.supportedTypes: + thermal_cam = self._oak.pipeline.create(dai.node.Camera) + # Hardcoded for OAK-T. The correct size is needed for correct detection parsing + thermal_cam.setSize(256, 192) + thermal_cam.setBoardSocket(cam.board_socket) + xout_thermal = self._oak.pipeline.create(dai.node.XLinkOut) + xout_thermal.setStreamName("thermal_cam") + thermal_cam.raw.link(xout_thermal.input) + create_dai_queues_after_start["thermal_cam"] = (thermal_cam, None) elif sensor_resolution is not None: sdk_cam = self._oak.create_camera( cam.board_socket, @@ -448,11 +473,10 @@ def update_pipeline(self, runtime_only: bool) -> Message: (smallest_supported_resolution.width, smallest_supported_resolution.height), res_x ) ) - self._cameras.append(sdk_cam) + self._queues.append((sdk_cam, self._oak.queue(sdk_cam.out.main))) else: print("Skipped creating camera:", cam.board_socket, "because no valid sensor resolution was found.") continue - self._queues.append((sdk_cam, self._oak.queue(sdk_cam.out.main))) if config.depth: print("Creating depth") @@ -506,19 +530,53 @@ def update_pipeline(self, runtime_only: bool) -> Message: if config.ai_model and config.ai_model.path: cam_component = self._get_component_by_socket(config.ai_model.camera) - if not cam_component: - return ErrorMessage(f"{config.ai_model.camera} is not configured. Couldn't create NN.") - if config.ai_model.path == "age-gender-recognition-retail-0013": - face_detection = self._oak.create_nn("face-detection-retail-0004", cam_component) - self._nnet = self._oak.create_nn("age-gender-recognition-retail-0013", input=face_detection) + dai_camnode = [ + node + for node, _ in create_dai_queues_after_start.values() + if isinstance(node, dai.node.Camera) and node.getBoardSocket() == config.ai_model.camera + ] + model_path = config.ai_model.path + if len(dai_camnode) > 0: + model_path = os.path.join( + model_dir, + config.ai_model.path + + "_openvino_" + + dai.OpenVINO.getVersionName(dai.OpenVINO.DEFAULT_VERSION) + + "_6shave" + + ".blob", + ) + cam_node = dai_camnode[0] + if "yolo" in config.ai_model.path: + yolo = self._oak.pipeline.createYoloDetectionNetwork() + yolo.setBlobPath(model_path) + yolo.setConfidenceThreshold(0.5) + if "yolov6n_thermal_people_256x192" == config.ai_model.path: + yolo.setNumClasses(1) + yolo.setCoordinateSize(4) + cam_node.raw.link(yolo.input) + xlink_out_yolo = self._oak.pipeline.createXLinkOut() + xlink_out_yolo.setStreamName("yolo") + yolo.out.link(xlink_out_yolo.input) + create_dai_queues_after_start["yolo"] = ( + yolo, + DetectionContext( + labels=["person"], + frame_width=cam_node.getWidth(), + frame_height=cam_node.getHeight(), + board_socket=config.ai_model.camera, + ), + ) + elif not cam_component: + self.store.send_message_to_frontend( + WarningMessage(f"{config.ai_model.camera} is not configured, won't create NNET.") + ) + elif config.ai_model.path == "age-gender-recognition-retail-0013": + face_detection = self._oak.create_nn(model_path, cam_component) + self._nnet = self._oak.create_nn(model_path, input=face_detection) else: - self._nnet = self._oak.create_nn(config.ai_model.path, cam_component) - - camera = self._get_camera_config_by_socket(config, config.ai_model.camera) - if not camera: - return ErrorMessage(f"{config.ai_model.camera} is not configured. Couldn't create NN.") - - self._queues.append((self._nnet, self._oak.queue(self._nnet.out.main))) + self._nnet = self._oak.create_nn(model_path, cam_component) + if self._nnet: + self._queues.append((self._nnet, self._oak.queue(self._nnet.out.main))) sys_logger_xlink = self._oak.pipeline.createXLinkOut() logger = self._oak.pipeline.createSystemLogger() @@ -527,6 +585,7 @@ def update_pipeline(self, runtime_only: bool) -> Message: logger.out.link(sys_logger_xlink.input) try: + print("Starting pipeline") self._oak.start(blocking=False) except RuntimeError as e: print("Couldn't start pipeline: ", e) @@ -534,6 +593,8 @@ def update_pipeline(self, runtime_only: bool) -> Message: running = self._oak.running() if running: + for q_name, (node, context) in create_dai_queues_after_start.items(): + self._dai_queues.append((node, self._oak.device.getOutputQueue(q_name, 2, False), context)) self._pipeline_start_t = time.time() self._sys_info_q = self._oak.device.getOutputQueue("sys_logger", 1, False) # We might have modified the config, so store it @@ -549,9 +610,9 @@ def update_pipeline(self, runtime_only: bool) -> Message: def update(self) -> None: if self._oak is None: return - if not self._oak.running(): + if not self._oak.device.isPipelineRunning(): return - self._oak.poll() + # self._oak.poll() for component, queue in self._queues: try: @@ -560,6 +621,11 @@ def update(self) -> None: except QueueEmpty: continue + for dai_node, queue, context in self._dai_queues: + packet = queue.tryGet() + if packet is not None: + self._packet_handler.log_dai_packet(dai_node, packet, context) + if self._xlink_statistics is not None: self._xlink_statistics.update() diff --git a/rerun_py/depthai_viewer/_backend/device_configuration.py b/rerun_py/depthai_viewer/_backend/device_configuration.py index 73835f918d4b..1b2e93a691c5 100644 --- a/rerun_py/depthai_viewer/_backend/device_configuration.py +++ b/rerun_py/depthai_viewer/_backend/device_configuration.py @@ -53,25 +53,29 @@ def dict(self, *args, **kwargs) -> Dict[str, Any]: # type: ignore[no-untyped-de def to_runtime_controls(self) -> Dict[str, Any]: return { "algorithm_control": { - "align": "RECTIFIED_LEFT" - if self.align == dai.CameraBoardSocket.LEFT - else "RECTIFIED_RIGHT" - if self.align == dai.CameraBoardSocket.RIGHT - else "CENTER", + "align": ( + "RECTIFIED_LEFT" + if self.align == dai.CameraBoardSocket.LEFT + else "RECTIFIED_RIGHT" + if self.align == dai.CameraBoardSocket.RIGHT + else "CENTER" + ), "lr_check": self.lr_check, "lrc_check_threshold": self.lrc_threshold, "extended": self.extended_disparity, "subpixel": self.subpixel_disparity, }, "postprocessing": { - "median": { - dai.MedianFilter.MEDIAN_OFF: 0, - dai.MedianFilter.KERNEL_3x3: 3, - dai.MedianFilter.KERNEL_5x5: 5, - dai.MedianFilter.KERNEL_7x7: 7, - }[self.median] - if self.median - else 0, + "median": ( + { + dai.MedianFilter.MEDIAN_OFF: 0, + dai.MedianFilter.KERNEL_3x3: 3, + dai.MedianFilter.KERNEL_5x5: 5, + dai.MedianFilter.KERNEL_7x7: 7, + }[self.median] + if self.median + else 0 + ), "bilateral_sigma": self.sigma, }, "cost_matching": { @@ -126,6 +130,11 @@ def dict(self, *args, **kwargs): # type: ignore[no-untyped-def] display_name="Age gender recognition", camera=dai.CameraBoardSocket.CAM_A, ), + AiModelConfiguration( + path="yolov6n_thermal_people_256x192", + display_name="Thermal Person Detection", + camera=dai.CameraBoardSocket.CAM_E, + ), ] @@ -135,6 +144,7 @@ class ImuConfiguration(BaseModel): # type: ignore[misc] class CameraSensorResolution(Enum): + THE_256X192: str = "THE_256X192" THE_400_P: str = "THE_400_P" THE_480_P: str = "THE_480_P" THE_720_P: str = "THE_720_P" @@ -296,13 +306,16 @@ def dict(self, *args, **kwargs) -> Dict[str, Any]: # type: ignore[no-untyped-de "connection": self.info.connection.value, "mxid": self.info.mxid, }, - "default_stereo_pair": (self.default_stereo_pair[0].name, self.default_stereo_pair[1].name) - if self.default_stereo_pair - else None, + "default_stereo_pair": ( + (self.default_stereo_pair[0].name, self.default_stereo_pair[1].name) + if self.default_stereo_pair + else None + ), } size_to_resolution = { + (256, 192): CameraSensorResolution.THE_256X192, (640, 400): CameraSensorResolution.THE_400_P, (640, 480): CameraSensorResolution.THE_480_P, # OV7251 (1280, 720): CameraSensorResolution.THE_720_P, diff --git a/rerun_py/depthai_viewer/_backend/device_defaults/oak_t_default.py b/rerun_py/depthai_viewer/_backend/device_defaults/oak_t_default.py new file mode 100644 index 000000000000..13e42e87d545 --- /dev/null +++ b/rerun_py/depthai_viewer/_backend/device_defaults/oak_t_default.py @@ -0,0 +1,31 @@ +import depthai as dai +from depthai_viewer._backend.device_configuration import ( + AiModelConfiguration, + CameraConfiguration, + CameraSensorResolution, + PipelineConfiguration, +) + +config = PipelineConfiguration( + cameras=[ + CameraConfiguration( + resolution=CameraSensorResolution.THE_256X192, + kind=dai.CameraSensorType.THERMAL, + board_socket=dai.CameraBoardSocket.CAM_E, + name="Thermal", + ), + CameraConfiguration( + resolution=CameraSensorResolution.THE_1080_P, + kind=dai.CameraSensorType.COLOR, + board_socket=dai.CameraBoardSocket.CAM_A, + stream_enabled=False, + name="Color", + ), + ], + depth=None, + ai_model=AiModelConfiguration( + display_name="Thermal Person Detection", + path="yolov6n_thermal_people_256x192", + camera=dai.CameraBoardSocket.CAM_E, + ), +) diff --git a/rerun_py/depthai_viewer/_backend/main.py b/rerun_py/depthai_viewer/_backend/main.py index dcec93ee64b4..eaa3432e0d66 100644 --- a/rerun_py/depthai_viewer/_backend/main.py +++ b/rerun_py/depthai_viewer/_backend/main.py @@ -6,7 +6,6 @@ import sentry_sdk -import depthai_viewer as viewer from depthai_viewer import version as depthai_viewer_version from depthai_viewer._backend.config_api import Action, start_api from depthai_viewer._backend.device import Device diff --git a/rerun_py/depthai_viewer/_backend/packet_handler.py b/rerun_py/depthai_viewer/_backend/packet_handler.py index 4112a5335533..24e80a1dc4b5 100644 --- a/rerun_py/depthai_viewer/_backend/packet_handler.py +++ b/rerun_py/depthai_viewer/_backend/packet_handler.py @@ -7,7 +7,6 @@ from depthai_sdk.classes.packets import ( # PointcloudPacket, BasePacket, DepthPacket, - Detection, DetectionPacket, DisparityDepthPacket, FramePacket, @@ -17,6 +16,7 @@ from depthai_sdk.components import CameraComponent, Component, NNComponent, StereoComponent from depthai_sdk.components.tof_component import ToFComponent from numpy.typing import NDArray +from pydantic import BaseModel import depthai_viewer as viewer from depthai_viewer._backend.store import Store @@ -24,6 +24,18 @@ from depthai_viewer.components.rect2d import RectFormat +class PacketHandlerContext(BaseModel): # type: ignore[misc] + class Config: + arbitrary_types_allowed = True + + +class DetectionContext(PacketHandlerContext): + labels: List[str] + frame_width: int + frame_height: int + board_socket: dai.CameraBoardSocket + + class PacketHandler: store: Store _ahrs: Mahony @@ -50,6 +62,56 @@ def set_camera_intrinsics_getter( # type: ignore[assignment, misc] self._get_camera_intrinsics = camera_intrinsics_getter + def log_dai_packet(self, node: dai.Node, packet: dai.Buffer, context: Optional[PacketHandlerContext]) -> None: + if isinstance(packet, dai.ImgFrame): + board_socket = None + if isinstance(node, dai.node.ColorCamera): + board_socket = node.getBoardSocket() + elif isinstance(node, dai.node.MonoCamera): + board_socket = node.getBoardSocket() + elif isinstance(node, dai.node.Camera): + board_socket = node.getBoardSocket() + if board_socket is not None: + self._on_camera_frame(FramePacket("", packet), board_socket) + else: + print("Unknown node type:", type(node), "for packet:", type(packet)) + elif isinstance(packet, dai.ImgDetections): + if context is None or not isinstance(context, DetectionContext): + print("Invalid context for detections packet", context) + return + self._on_dai_detections(packet, context) + else: + print("Unknown dai packet type:", type(packet)) + + def _dai_detections_to_rects_colors_labels( + self, packet: dai.ImgDetections, context: DetectionContext + ) -> Tuple[List[List[int]], List[List[int]], List[str]]: + rects = [] + colors = [] + labels = [] + for detection in packet.detections: + rects.append(self._rect_from_detection(detection, context.frame_width, context.frame_height)) + colors.append([0, 255, 0]) + label = "" + # Open model zoo models output label index + if context.labels is not None: + label += context.labels[detection.label] + label += ", " + str(int(detection.confidence * 100)) + "%" + labels.append(label) + return rects, colors, labels + pass + + def _on_dai_detections(self, packet: dai.ImgDetections, context: DetectionContext) -> None: + packet.detections + rects, colors, labels = self._dai_detections_to_rects_colors_labels(packet, context) + viewer.log_rects( + f"{context.board_socket.name}/transform/color_cam/Detections", + rects, + rect_format=RectFormat.XYXY, + colors=colors, + labels=labels, + ) + def log_packet( self, component: Component, @@ -114,6 +176,9 @@ def _on_camera_frame(self, packet: FramePacket, board_socket: dai.CameraBoardSoc height=h, encoding=viewer.ImageEncoding.NV12, ) + elif packet.msg.getType() == dai.RawImgFrame.Type.GRAYF16: + img = img_frame.view(np.float16).reshape(h, w) + viewer.log_image(entity_path, img, colormap=viewer.Colormap.Magma, unit="°C") else: viewer.log_image(entity_path, img_frame) @@ -176,7 +241,9 @@ def _detections_to_rects_colors_labels( colors = [] labels = [] for detection in packet.detections: - rects.append(self._rect_from_detection(detection, packet.frame.shape[0], packet.frame.shape[1])) + rects.append( + self._rect_from_detection(detection.img_detection, packet.frame.shape[0], packet.frame.shape[1]) + ) colors.append([0, 255, 0]) label: str = detection.label_str # Open model zoo models output label index @@ -198,18 +265,18 @@ def _on_age_gender_packet(self, packet: TwoStagePacket, component: NNComponent) cam = "color_cam" if component._get_camera_comp().is_color() else "mono_cam" viewer.log_rect( f"{component._get_camera_comp()._socket.name}/transform/{cam}/Detection", - self._rect_from_detection(det, packet.frame.shape[0], packet.frame.shape[1]), + self._rect_from_detection(det.img_detection, packet.frame.shape[0], packet.frame.shape[1]), rect_format=RectFormat.XYXY, color=color, label=label, ) - def _rect_from_detection(self, detection: Detection, max_height: int, max_width: int) -> List[int]: + def _rect_from_detection(self, detection: dai.ImgDetection, max_height: int, max_width: int) -> List[int]: return [ - max(min(detection.bottom_right[0], max_width), 0) * max_width, - max(min(detection.bottom_right[1], max_height), 0) * max_height, - max(min(detection.top_left[0], max_width), 0) * max_width, - max(min(detection.top_left[1], max_height), 0) * max_height, + int(max(min(detection.xmin, max_width), 0) * max_width), + int(max(min(detection.xmax, max_height), 0) * max_height), + int(max(min(detection.ymax, max_width), 0) * max_width), + int(max(min(detection.ymin, max_height), 0) * max_height), ] diff --git a/rerun_py/depthai_viewer/components/tensor.py b/rerun_py/depthai_viewer/components/tensor.py index 4d054120b60b..a525aa8040ac 100644 --- a/rerun_py/depthai_viewer/components/tensor.py +++ b/rerun_py/depthai_viewer/components/tensor.py @@ -27,6 +27,15 @@ class ImageEncoding(Enum): NV12 = "NV12" +class Colormap(Enum): + Grayscale: str = "grayscale" + Inferno: str = "inferno" + Magma: str = "magma" + Plasma: str = "plasma" + Turbo: str = "turbo" + Viridis: str = "viridis" + + TensorDType = Union[ np.uint8, np.uint16, @@ -61,9 +70,11 @@ class TensorArray(pa.ExtensionArray): # type: ignore[misc] def from_numpy( array: npt.NDArray[TensorDType], encoding: ImageEncoding | None = None, + colormap: Colormap | None = None, names: Iterable[str | None] | None = None, meaning: bindings.TensorDataMeaning = None, meter: float | None = None, + unit: str | None = None, ) -> TensorArray: """Build a `TensorArray` from an numpy array.""" # Build a random tensor_id @@ -98,7 +109,11 @@ def from_numpy( discriminant=discriminant, child=pa.array([True], type=pa.bool_()), ) - + colormap = build_dense_union( + TensorType.storage_type["colormap"].type, + discriminant=colormap.name if colormap is not None else "None", + child=pa.array([True], type=pa.bool_()), + ) # Note: the pa.array mask is backwards from expectations # Mask is True for elements which are not valid. if meter is None: @@ -106,8 +121,10 @@ def from_numpy( else: meter = pa.array([meter], mask=[False], type=pa.float32()) + unit = pa.array([unit if unit is not None else ""], type=pa.string()) + storage = pa.StructArray.from_arrays( - [tensor_id, shape, data, meaning, meter], + [tensor_id, shape, data, meaning, meter, colormap, unit], fields=list(TensorType.storage_type), ).cast(TensorType.storage_type) storage.validate(full=True) diff --git a/rerun_py/depthai_viewer/install_requirements.py b/rerun_py/depthai_viewer/install_requirements.py index d810ccade7e4..1a5cac475e19 100644 --- a/rerun_py/depthai_viewer/install_requirements.py +++ b/rerun_py/depthai_viewer/install_requirements.py @@ -5,7 +5,7 @@ import subprocess import sys import traceback -from typing import Any, Dict +from pathlib import Path # type: ignore[attr-defined] from depthai_viewer import version as depthai_viewer_version @@ -15,6 +15,8 @@ venv_python = ( os.path.join(venv_dir, "Scripts", "python") if sys.platform == "win32" else os.path.join(venv_dir, "bin", "python") ) +# The default blobconverter location. __ protected... +model_dir = Path.home() / Path(".cache/blobconverter") def delete_partially_created_venv(path: str) -> None: @@ -43,25 +45,19 @@ def get_site_packages() -> str: def download_blobs() -> None: import blobconverter - from depthai_sdk.components.nn_helper import getSupportedModels - - models = [ - "yolov8n_coco_640x352", - "mobilenet-ssd", - "face-detection-retail-0004", - "age-gender-recognition-retail-0013", - ] - sdk_models = getSupportedModels(printModels=False) - for model in models: - zoo_type = None - if model in sdk_models: - model_config_file = sdk_models[model] / "config.json" - config = json.load(open(model_config_file)) - if "model" in config: - model_config: Dict[str, Any] = config["model"] - if "model_name" in model_config: - zoo_type = model_config.get("zoo", "intel") - blobconverter.from_zoo(model, zoo_type=zoo_type, shaves=6) + + if not os.path.exists(model_dir): + os.makedirs(model_dir) + models = { + "yolov8n_coco_640x352": "depthai", + "mobilenet-ssd": "intel", + "face-detection-retail-0004": "intel", + "age-gender-recognition-retail-0013": "intel", + "yolov6n_thermal_people_256x192": "depthai", + } + for model, zoo_type in models.items(): + # With use_cache=True, blobconverter will not download / move the blob to model_dir... + blobconverter.from_zoo(model, zoo_type=zoo_type, shaves=6, output_dir=model_dir, use_cache=False) def dependencies_installed() -> bool: diff --git a/rerun_py/depthai_viewer/log/image.py b/rerun_py/depthai_viewer/log/image.py index 87e293d52ef6..a2ba0c32e04f 100644 --- a/rerun_py/depthai_viewer/log/image.py +++ b/rerun_py/depthai_viewer/log/image.py @@ -4,7 +4,7 @@ import numpy.typing as npt from depthai_viewer import bindings -from depthai_viewer.components.tensor import ImageEncoding +from depthai_viewer.components.tensor import Colormap, ImageEncoding from depthai_viewer.log.error_utils import _send_warning from depthai_viewer.log.log_decorator import log_decorator from depthai_viewer.log.tensor import Tensor, _log_tensor, _to_numpy @@ -22,6 +22,8 @@ def log_image( entity_path: str, image: Tensor, *, + colormap: Optional[Colormap] = None, + unit: Optional[str] = None, ext: Optional[Dict[str, Any]] = None, timeless: bool = False, ) -> None: @@ -44,6 +46,10 @@ def log_image( Path to the image in the space hierarchy. image: A [Tensor][rerun.log.tensor.Tensor] representing the image to log. + colormap: + Optional colormap to apply to single channel images. + unit: + Optional unit of the single channel image. ext: Optional dictionary of extension components. See [rerun.log_extension_components][] timeless: @@ -75,7 +81,7 @@ def log_image( if interpretable_as_image and num_non_empty_dims != len(shape): image = np.squeeze(image) - _log_tensor(entity_path, image, ext=ext, timeless=timeless) + _log_tensor(entity_path, image, ext=ext, timeless=timeless, colormap=colormap, unit=unit) @log_decorator diff --git a/rerun_py/depthai_viewer/log/tensor.py b/rerun_py/depthai_viewer/log/tensor.py index bac2da9e41b5..e24e46a9a999 100644 --- a/rerun_py/depthai_viewer/log/tensor.py +++ b/rerun_py/depthai_viewer/log/tensor.py @@ -5,7 +5,7 @@ from depthai_viewer import bindings from depthai_viewer.components.instance import InstanceArray -from depthai_viewer.components.tensor import ImageEncoding, TensorArray +from depthai_viewer.components.tensor import Colormap, ImageEncoding, TensorArray from depthai_viewer.log.error_utils import _send_warning from depthai_viewer.log.extension_components import _add_extension_components from depthai_viewer.log.log_decorator import log_decorator @@ -86,6 +86,8 @@ def _log_tensor( ext: Optional[Dict[str, Any]] = None, timeless: bool = False, encoding: Optional[ImageEncoding] = None, + colormap: Optional[Colormap] = None, + unit: Optional[str] = None, ) -> None: """Log a general tensor, perhaps with named dimensions.""" @@ -128,7 +130,7 @@ def _log_tensor( instanced: Dict[str, Any] = {} splats: Dict[str, Any] = {} - instanced["rerun.tensor"] = TensorArray.from_numpy(tensor, encoding, names, meaning, meter) + instanced["rerun.tensor"] = TensorArray.from_numpy(tensor, encoding, colormap, names, meaning, meter, unit) if ext: _add_extension_components(instanced, splats, ext, None) diff --git a/rerun_py/depthai_viewer/requirements.txt b/rerun_py/depthai_viewer/requirements.txt index 433c05a4c9f2..b7ed15103bc7 100644 --- a/rerun_py/depthai_viewer/requirements.txt +++ b/rerun_py/depthai_viewer/requirements.txt @@ -3,7 +3,8 @@ pyarrow==10.0.1 setuptools ahrs # depthai_sdk conflicts with depthai, so it's installed seperatelly in __main__.py -depthai==2.23.0.0 +--extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-snapshot-local +depthai==2.24.0.0.dev0+c014e27e224f7ef3f6407be6b3f05be6c2fffd13 websockets pydantic==1.9 deprecated diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index bd075684e84a..ed1e0387fa9c 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -25,7 +25,7 @@ pub use depthai_viewer::{ EncodedMesh3D, InstanceKey, KeypointId, Label, LineStrip2D, LineStrip3D, Mat3x3, Mesh3D, MeshFormat, MeshId, Pinhole, Point2D, Point3D, Quaternion, Radius, RawMesh3D, Rect2D, Rigid3, Scalar, ScalarPlotProps, Size3D, Tensor, TensorData, TensorDimension, TensorId, - TextEntry, Transform, Vec2D, Vec3D, Vec4D, ViewCoordinates, + TextEntry, Transform, Vec2D, Vec3D, Vec4D, ViewCoordinates, TensorColormap, }, coordinates::{Axis3, Handedness, Sign, SignedAxis3}, }; @@ -943,6 +943,8 @@ fn log_image_file( data: re_log_types::component_types::TensorData::JPEG(img_bytes.into()), meaning: re_log_types::component_types::TensorDataMeaning::Unknown, meter: None, + colormap: TensorColormap::None, + unit: None, }; let row = DataRow::from_cells1( diff --git a/working.log b/working.log new file mode 100644 index 000000000000..8445a03171ac --- /dev/null +++ b/working.log @@ -0,0 +1,23 @@ +Selecting device: 194430100196762700 +/home/filip/Documents/depthai-viewer/rerun_py/depthai_viewer/venv-0.1.7/lib/python3.10/site-packages/depthai_sdk/oak_camera.py:220: UsbWarning: Device connected in USB2 mode! This might cause some issues. In such case, please try using a (different) USB3 cable, or force USB2 mode 'with OakCamera(usb_speed='usb2') as oak:' + warnings.warn("Device connected in USB2 mode! This might cause some issues. " +Oak cam: +No intrisics for cam: CameraBoardSocket.CAM_A +No intrisics for cam: CameraBoardSocket.CAM_E +Default stereo pair: None +Pipeline config: auto=True cameras=[CameraConfiguration(fps=30, resolution=, kind=, board_socket=, stream_enabled=True, name='Rgb'), CameraConfiguration(fps=30, resolution=, kind=, board_socket=, stream_enabled=True, name='Thermal')] depth=None ai_model=None imu=ImuConfiguration(report_rate=100, batch_report_threshold=5) +Update pipeline: auto=True cameras=[CameraConfiguration(fps=30, resolution=, kind=, board_socket=, stream_enabled=True, name='Rgb'), CameraConfiguration(fps=30, resolution=, kind=, board_socket=, stream_enabled=True, name='Thermal')] depth=None ai_model=None imu=ImuConfiguration(report_rate=100, batch_report_threshold=5) +Updating pipeline... +Usb speed: UsbSpeed.HIGH +Device is connected in USB2 mode, camera streams will be JPEG encoded... +dai version: 2.24.0.0.dev0+c014e27e224f7ef3f6407be6b3f05be6c2fffd13 +AI: None +Connected camera features [{socket: CAM_A, sensorName: IMX462, width: 1920, height: 1080, orientation: ROTATE_180_DEG, supportedTypes: [COLOR], hasAutofocus: 0, hasAutofocusIC: 0, name: rgb}, {socket: CAM_E, sensorName: TINY1C, width: 256, height: 192, orientation: AUTO, supportedTypes: [THERMAL], hasAutofocus: 0, hasAutofocusIC: 0, name: thermal}] +Creating camera: fps=30 resolution= kind= board_socket= stream_enabled=True name='Thermal' +Creating camera: fps=30 resolution= kind= board_socket= stream_enabled=False name='Color' +Starting pipeline +{'assetStorage': [], 'assets': {'map': {}}, 'pipeline': {'connections': [{'node1Id': 3, 'node1Output': 'out', 'node1OutputGroup': '', 'node2Id': 2, 'node2Input': 'in', 'node2InputGroup': ''}, {'node1Id': 0, 'node1Output': 'raw', 'node1OutputGroup': '', 'node2Id': 1, 'node2Input': 'in', 'node2InputGroup': ''}], 'globalProperties': {'calibData': None, 'cameraTuningBlobSize': None, 'cameraTuningBlobUri': '', 'leonCssFrequencyHz': 700000000.0, 'leonMssFrequencyHz': 700000000.0, 'pipelineName': None, 'pipelineVersion': None, 'sippBufferSize': 18432, 'sippDmaBufferSize': 16384, 'xlinkChunkSize': -1}, 'nodes': [[0, {'id': 0, 'ioInfo': [[['', 'video'], {'blocking': False, 'group': '', 'id': 11, 'name': 'video', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'still'], {'blocking': False, 'group': '', 'id': 9, 'name': 'still', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'isp'], {'blocking': False, 'group': '', 'id': 8, 'name': 'isp', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'preview'], {'blocking': False, 'group': '', 'id': 10, 'name': 'preview', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'raw'], {'blocking': False, 'group': '', 'id': 7, 'name': 'raw', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'frameEvent'], {'blocking': False, 'group': '', 'id': 6, 'name': 'frameEvent', 'queueSize': 8, 'type': 0, 'waitForMessage': False}], [['', 'inputConfig'], {'blocking': False, 'group': '', 'id': 5, 'name': 'inputConfig', 'queueSize': 8, 'type': 3, 'waitForMessage': False}], [['', 'inputControl'], {'blocking': True, 'group': '', 'id': 4, 'name': 'inputControl', 'queueSize': 8, 'type': 3, 'waitForMessage': False}]], 'name': 'Camera', 'properties': {'boardSocket': 4, 'calibAlpha': None, 'cameraName': '', 'colorOrder': 0, 'fp16': False, 'fps': 30.0, 'imageOrientation': -1, 'initialControl': {'aeLockMode': False, 'aeMaxExposureTimeUs': 0, 'aeRegion': {'height': 0, 'priority': 0, 'width': 0, 'x': 0, 'y': 0}, 'afRegion': {'height': 0, 'priority': 0, 'width': 0, 'x': 0, 'y': 0}, 'antiBandingMode': 0, 'autoFocusMode': 3, 'awbLockMode': False, 'awbMode': 0, 'brightness': 0, 'captureIntent': 0, 'chromaDenoise': 0, 'cmdMask': 0, 'contrast': 0, 'controlMode': 0, 'effectMode': 0, 'expCompensation': 0, 'expManual': {'exposureTimeUs': 0, 'frameDurationUs': 0, 'sensitivityIso': 0}, 'frameSyncMode': 0, 'lensPosAutoInfinity': 0, 'lensPosAutoMacro': 0, 'lensPosition': 0, 'lensPositionRaw': 0.0, 'lowPowerNumFramesBurst': 0, 'lowPowerNumFramesDiscard': 0, 'lumaDenoise': 0, 'saturation': 0, 'sceneMode': 0, 'sharpness': 0, 'strobeConfig': {'activeLevel': 0, 'enable': 0, 'gpioNumber': 0}, 'strobeTimings': {'durationUs': 0, 'exposureBeginOffsetUs': 0, 'exposureEndOffsetUs': 0}, 'wbColorTemp': 0}, 'interleaved': True, 'isp3aFps': 0, 'ispScale': {'horizDenominator': 0, 'horizNumerator': 0, 'vertDenominator': 0, 'vertNumerator': 0}, 'numFramesPoolIsp': 3, 'numFramesPoolPreview': 4, 'numFramesPoolRaw': 3, 'numFramesPoolStill': 4, 'numFramesPoolVideo': 4, 'previewHeight': 300, 'previewKeepAspectRatio': True, 'previewWidth': 300, 'rawPacked': None, 'resolutionHeight': -1, 'resolutionWidth': -1, 'sensorCropX': -1.0, 'sensorCropY': -1.0, 'sensorType': -1, 'stillHeight': -1, 'stillWidth': -1, 'videoHeight': -1, 'videoWidth': -1, 'warpMeshHeight': 0, 'warpMeshSource': -1, 'warpMeshStepHeight': 32, 'warpMeshStepWidth': 32, 'warpMeshUri': '', 'warpMeshWidth': 0}}], [1, {'id': 1, 'ioInfo': [[['', 'in'], {'blocking': True, 'group': '', 'id': 3, 'name': 'in', 'queueSize': 8, 'type': 3, 'waitForMessage': True}]], 'name': 'XLinkOut', 'properties': {'maxFpsLimit': -1.0, 'metadataOnly': False, 'streamName': 'thermal_cam'}}], [2, {'id': 2, 'ioInfo': [[['', 'in'], {'blocking': True, 'group': '', 'id': 2, 'name': 'in', 'queueSize': 8, 'type': 3, 'waitForMessage': True}]], 'name': 'XLinkOut', 'properties': {'maxFpsLimit': -1.0, 'metadataOnly': False, 'streamName': 'sys_logger'}}], [3, {'id': 3, 'ioInfo': [[['', 'out'], {'blocking': False, 'group': '', 'id': 1, 'name': 'out', 'queueSize': 8, 'type': 0, 'waitForMessage': False}]], 'name': 'SystemLogger', 'properties': {'rateHz': 0.10000000149011612}}]]}} +[194430100196762700] [1.1.4.3.2] [6.226] [Camera(0)] [warning] PlgSrcTiny1c::Create: Capping fps from 30 to 25 +Got message to send: +Sending message: +[2024-02-22T14:23:06Z WARN re_viewer::depthai::depthai] Device is connected in USB2 mode. This may cause performance issues.