diff --git a/viewer/client_eet.py b/viewer/client_eet.py index 98796d07..3a87f463 100644 --- a/viewer/client_eet.py +++ b/viewer/client_eet.py @@ -1,6 +1,5 @@ #------------------------------------------------------------------------------ # This script receives extended eye tracking data from the HoloLens. -# See https://learn.microsoft.com/en-us/windows/mixed-reality/develop/native/extended-eye-tracking-native #------------------------------------------------------------------------------ import hl2ss @@ -25,26 +24,21 @@ data = client.get_next_packet() data.payload = hl2ss.unpack_eet(data.payload) -print(data.timestamp) -print(data.payload._reserved) - -print(data.payload.combined_ray.origin) -print(data.payload.combined_ray.direction) -print(data.payload.left_ray.origin) -print(data.payload.left_ray.direction) -print(data.payload.right_ray.origin) -print(data.payload.right_ray.direction) -print(data.payload.left_openness) -print(data.payload.right_openness) -print(data.payload.vergence_distance) - -print(data.payload.calibration_valid) -print(data.payload.combined_ray_valid) -print(data.payload.left_ray_valid) -print(data.payload.right_ray_valid) -print(data.payload.left_openness_valid) -print(data.payload.right_openness_valid) -print(data.payload.vergence_distance_valid) +# See +# https://learn.microsoft.com/en-us/windows/mixed-reality/develop/native/extended-eye-tracking-native +# for details + +print(f'Tracking status at time {data.timestamp}') +print('Pose') print(data.pose) +print(f'Calibration valid: {data.payload.calibration_valid}') +print(f'Combined eye gaze: Valid={data.payload.combined_ray_valid} Origin={data.payload.combined_ray.origin} Direction={data.payload.combined_ray.direction}') +print(f'Left eye gaze: Valid={data.payload.left_ray_valid} Origin={data.payload.left_ray.origin} Direction={data.payload.left_ray.direction}') +print(f'Right eye gaze: Valid={data.payload.right_ray_valid} Origin={data.payload.right_ray.origin} Direction={data.payload.right_ray.direction}') + +# "...not supported by HoloLens 2 at this time" +print(f'Left eye openness: Valid={data.payload.left_openness_valid} Value={data.payload.left_openness}') +print(f'Right eye openness: Valid={data.payload.right_openness_valid} Value={data.payload.right_openness}') +print(f'Vergence distance: Valid={data.payload.vergence_distance_valid} Value={data.payload.vergence_distance}') client.close() diff --git a/viewer/client_microphone.py b/viewer/client_microphone.py index 5a44afa5..121dff2e 100644 --- a/viewer/client_microphone.py +++ b/viewer/client_microphone.py @@ -1,8 +1,8 @@ #------------------------------------------------------------------------------ -# This script receives AAC encoded microphone audio from the HoloLens and -# plays it. The main thread receives the data, decodes it, and puts the decoded -# audio samples in a queue. A second thread gets the samples from the queue and -# plays them. Audio stream configuration is fixed to 2 channels, 48000 Hz. +# This script receives microphone audio from the HoloLens and plays it. The +# main thread receives the data, decodes it, and puts the decoded audio samples +# in a queue. A second thread gets the samples from the queue and plays them. +# Audio stream configuration is fixed to 2 channels, 48000 Hz. # Press esc to stop. #------------------------------------------------------------------------------ @@ -27,12 +27,15 @@ #------------------------------------------------------------------------------ +# RAW format is s16 packed, AAC decoded format is f32 planar +audio_format = pyaudio.paInt16 if (profile == hl2ss.AudioProfile.RAW) else pyaudio.paFloat32 enable = True def pcmworker(pcmqueue): global enable + global audio_format p = pyaudio.PyAudio() - stream = p.open(format=pyaudio.paFloat32, channels=hl2ss.Parameters_MICROPHONE.CHANNELS, rate=hl2ss.Parameters_MICROPHONE.SAMPLE_RATE, output=True) + stream = p.open(format=audio_format, channels=hl2ss.Parameters_MICROPHONE.CHANNELS, rate=hl2ss.Parameters_MICROPHONE.SAMPLE_RATE, output=True) stream.start_stream() while (enable): stream.write(pcmqueue.get()) @@ -55,7 +58,8 @@ def on_press(key): while (enable): data = client.get_next_packet() - audio = hl2ss_utilities.microphone_planar_to_packed(data.payload) + # RAW format is s16 packed, AAC decoded format is f32 planar + audio = hl2ss_utilities.microphone_planar_to_packed(data.payload) if (profile != hl2ss.AudioProfile.RAW) else data.payload pcmqueue.put(audio.tobytes()) client.close() diff --git a/viewer/client_microphone_raw.py b/viewer/client_microphone_raw.py deleted file mode 100644 index 16d1c91c..00000000 --- a/viewer/client_microphone_raw.py +++ /dev/null @@ -1,64 +0,0 @@ -#------------------------------------------------------------------------------ -# This script receives uncompressed microphone audio from the HoloLens and -# plays it. The main thread receives the data, decodes it, and puts the decoded -# audio samples in a queue. A second thread gets the samples from the queue and -# plays them. Audio stream configuration is fixed to 2 channels, 48000 Hz. -# Press esc to stop. -#------------------------------------------------------------------------------ - -from pynput import keyboard - -import hl2ss -import pyaudio -import queue -import threading - -# Settings -------------------------------------------------------------------- - -# HoloLens address -host = "192.168.1.7" - -# Port -port = hl2ss.StreamPort.MICROPHONE - -# Audio encoding profile -profile = hl2ss.AudioProfile.RAW - -#------------------------------------------------------------------------------ - -enable = True - -def pcmworker(pcmqueue): - global enable - p = pyaudio.PyAudio() - stream = p.open(format=pyaudio.paInt16, channels=hl2ss.Parameters_MICROPHONE.CHANNELS, rate=hl2ss.Parameters_MICROPHONE.SAMPLE_RATE, output=True) - stream.start_stream() - while (enable): - stream.write(pcmqueue.get()) - stream.stop_stream() - stream.close() - -def on_press(key): - global enable - enable = key != keyboard.Key.esc - return enable - -pcmqueue = queue.Queue() -thread = threading.Thread(target=pcmworker, args=(pcmqueue,)) -listener = keyboard.Listener(on_press=on_press) -thread.start() -listener.start() - -client = hl2ss.rx_microphone(host, port, hl2ss.ChunkSize.MICROPHONE, profile) -client.open() - -while (enable): - data = client.get_next_packet() - pcmqueue.put(bytes(data.payload)) - -client.close() - -enable = False -pcmqueue.put(b'') -thread.join() -listener.join() diff --git a/viewer/client_pv.py b/viewer/client_pv.py index bb4ac014..ad5966ac 100644 --- a/viewer/client_pv.py +++ b/viewer/client_pv.py @@ -1,11 +1,11 @@ #------------------------------------------------------------------------------ -# This script receives encoded video from the HoloLens front RGB camera and -# plays it. The camera support various resolutions and framerates. See -# etc/hl2_capture_formats.txt for a list of supported formats. The default -# configuration is 1080p 30 FPS. The stream supports three operating modes: -# 0) video, 1) video + camera pose, 2) query calibration (single transfer). -# Press esc to stop. Note that the ahat stream cannot be used while the pv -# subsystem is on. +# This script receives video from the HoloLens front RGB camera and plays it. +# The camera supports various resolutions and framerates. See +# https://github.com/jdibenes/hl2ss/blob/main/etc/pv_configurations.txt +# for a list of supported formats. The default configuration is 1080p 30 FPS. +# The stream supports three operating modes: 0) video, 1) video + camera pose, +# 2) query calibration (single transfer). +# Press esc to stop. #------------------------------------------------------------------------------ from pynput import keyboard @@ -38,9 +38,15 @@ # Encoded stream average bits per second # Must be > 0 -bitrate = 5*1024*1024 +bitrate = hl2ss.get_video_codec_bitrate(width, height, framerate, hl2ss.get_video_codec_default_factor(profile)) # Decoded format +# Options include: +# 'bgr24' +# 'rgb24' +# 'bgra' +# 'rgba' +# 'gray8' decoded_format = 'bgr24' #------------------------------------------------------------------------------ @@ -50,11 +56,13 @@ if (mode == hl2ss.StreamMode.MODE_2): data = hl2ss.download_calibration_pv(host, port, width, height, framerate) print('Calibration') - print(data.focal_length) - print(data.principal_point) - print(data.radial_distortion) - print(data.tangential_distortion) + print(f'Focal length: {data.focal_length}') + print(f'Principal point: {data.principal_point}') + print(f'Radial distortion: {data.radial_distortion}') + print(f'Tangential distortion: {data.tangential_distortion}') + print('Projection') print(data.projection) + print('Intrinsics') print(data.intrinsics) else: enable = True @@ -72,12 +80,12 @@ def on_press(key): while (enable): data = client.get_next_packet() + print('Pose at time {ts}'.format(ts=data.timestamp)) print(data.pose) - print('Focal length') - print(data.payload.focal_length) - print('Principal point') - print(data.payload.principal_point) + print(f'Focal length: {data.payload.focal_length}') + print(f'Principal point: {data.payload.principal_point}') + cv2.imshow('Video', data.payload.image) cv2.waitKey(1) diff --git a/viewer/client_pv_raw.py b/viewer/client_pv_raw.py deleted file mode 100644 index 9760a4e5..00000000 --- a/viewer/client_pv_raw.py +++ /dev/null @@ -1,82 +0,0 @@ -#------------------------------------------------------------------------------ -# This script receives uncompressed video from the HoloLens front RGB camera -# and plays it. The camera support various resolutions and framerates. See -# etc/hl2_capture_formats.txt for a list of supported formats. The default -# configuration is 1080p 30 FPS. The stream supports three operating modes: -# 0) video, 1) video + camera pose, 2) query calibration (single transfer). -# Press esc to stop. Note that the ahat stream cannot be used while the pv -# subsystem is on. -#------------------------------------------------------------------------------ - -from pynput import keyboard - -import time -import numpy as np -import cv2 -import hl2ss_imshow -import hl2ss - -# Settings -------------------------------------------------------------------- - -# HoloLens address -host = "192.168.1.7" - -# Port -port = hl2ss.StreamPort.PERSONAL_VIDEO - -# Operating mode -# 0: video -# 1: video + camera pose -# 2: query calibration (single transfer) -mode = hl2ss.StreamMode.MODE_1 - -# Camera parameters -width = 640 -height = 360 -framerate = 15 - -# Video encoding profile -profile = hl2ss.VideoProfile.RAW - -#------------------------------------------------------------------------------ - -hl2ss.start_subsystem_pv(host, port) - -if (mode == hl2ss.StreamMode.MODE_2): - data = hl2ss.download_calibration_pv(host, port, width, height, framerate) - print('Calibration') - print(data.focal_length) - print(data.principal_point) - print(data.radial_distortion) - print(data.tangential_distortion) - print(data.projection) - print(data.intrinsics) -else: - enable = True - - def on_press(key): - global enable - enable = key != keyboard.Key.esc - return enable - - listener = keyboard.Listener(on_press=on_press) - listener.start() - - client = hl2ss.rx_raw_pv(host, port, hl2ss.ChunkSize.PERSONAL_VIDEO, mode, width, height, framerate, profile, 1, 'bgr24') - client.open() - - stride = hl2ss.get_nv12_stride(width) - - while (enable): - data = client.get_next_packet() - - print('Pose at time {ts}'.format(ts=data.timestamp)) - print(data.pose) - - cv2.imshow('Video', data.payload.image) - cv2.waitKey(1) - - client.close() - listener.join() - -hl2ss.stop_subsystem_pv(host, port) diff --git a/viewer/client_rc.py b/viewer/client_rc.py index 3bb7390d..37cbe69e 100644 --- a/viewer/client_rc.py +++ b/viewer/client_rc.py @@ -64,17 +64,19 @@ client.open() version = client.get_application_version() -print('Installed version {v0}.{v1}.{v2}.{v3}'.format(v0=version[0], v1=version[1], v2=version[2], v3=version[3])) +print(f'Installed version {version[0]}.{version[1]}.{version[2]}.{version[3]}') -utc_offset = client.get_utc_offset(32) # Add this offset to timestamps to convert to utc -print('QPC timestamp to UTC offset is {offset} hundreds of nanoseconds'.format(offset=utc_offset)) +# Add this offset to timestamps to convert to utc (Windows FILETIME) +utc_offset = client.get_utc_offset(32) +print(f'QPC timestamp to UTC offset is {utc_offset} hundreds of nanoseconds') client.set_hs_marker_state(marker_state) # PV camera configuration pv_status = client.get_pv_subsystem_status() -print('PV subsystem is {status}'.format(status=('On' if pv_status else 'Off'))) +print(f'PV subsystem is {("On" if (pv_status) else "Off")}') +# Ignored if PV subsystem is Off client.set_pv_focus(focus_mode, auto_focus_range, manual_focus_distance, focus_value, driver_fallback) client.set_pv_video_temporal_denoising(video_temporal_denoising) client.set_pv_white_balance_preset(white_balance_preset) diff --git a/viewer/client_rm_depth_ahat.py b/viewer/client_rm_depth_ahat.py index ac7c7e90..84ce315f 100644 --- a/viewer/client_rm_depth_ahat.py +++ b/viewer/client_rm_depth_ahat.py @@ -1,11 +1,10 @@ #------------------------------------------------------------------------------ -# This script receives encoded video from the HoloLens depth camera in ahat -# mode and plays it. The resolution is 512x512 @ 45 FPS. The stream supports -# three operating modes: 0) video, 1) video + rig pose, 2) query calibration -# (single transfer). Press esc to stop. Depth and AB data are scaled for -# visibility. Note that 1) the ahat stream cannot be used while the pv -# subsystem is on, and 2) the ahat and long throw streams cannot be used -# simultaneously. +# This script receives video from the HoloLens depth camera in ahat mode and +# plays it. The resolution is 512x512 @ 45 FPS. The stream supports three +# operating modes: 0) video, 1) video + rig pose, 2) query calibration (single +# transfer). Depth and AB data are scaled for visibility. The ahat and long +# throw streams cannot be used simultaneously. +# Press esc to stop. #------------------------------------------------------------------------------ from pynput import keyboard @@ -41,11 +40,15 @@ if (mode == hl2ss.StreamMode.MODE_2): data = hl2ss.download_calibration_rm_depth_ahat(host, port) print('Calibration data') - print(data.uv2xy.shape) + print('Image point to unit plane') + print(data.uv2xy) + print('Extrinsics') print(data.extrinsics) - print(data.scale) - print(data.alias) - print(data.undistort_map.shape) + print(f'Scale: {data.scale}') + print(f'Alias: {data.alias}') + print('Undistort map') + print(data.undistort_map) + print('Intrinsics (undistorted only)') print(data.intrinsics) quit() @@ -64,10 +67,10 @@ def on_press(key): while (enable): data = client.get_next_packet() - print('Pose at time {ts}'.format(ts=data.timestamp)) + print(f'Pose at time {data.timestamp}') print(data.pose) - cv2.imshow('Depth', data.payload.depth / np.max(data.payload.depth)) # Normalized for visibility - cv2.imshow('AB', data.payload.ab / np.max(data.payload.ab)) # Normalized for visibility + cv2.imshow('Depth', data.payload.depth / np.max(data.payload.depth)) # Scaled for visibility + cv2.imshow('AB', data.payload.ab / np.max(data.payload.ab)) # Scaled for visibility cv2.waitKey(1) client.close() diff --git a/viewer/client_rm_depth_ahat_raw.py b/viewer/client_rm_depth_ahat_raw.py deleted file mode 100644 index 8b3eaaf4..00000000 --- a/viewer/client_rm_depth_ahat_raw.py +++ /dev/null @@ -1,72 +0,0 @@ -#------------------------------------------------------------------------------ -# This script receives uncompressed video from the HoloLens depth camera in -# ahat mode and plays it. The resolution is 512x512 @ 45 FPS. The stream -# supports three operating modes: 0) video, 1) video + rig pose, 2) query -# calibration (single transfer). Press esc to stop. Depth and AB data are -# scaled for visibility. Note that 1) the ahat stream cannot be used while the -# pv subsystem is on, and 2) the ahat and long throw streams cannot be used -# simultaneously. -#------------------------------------------------------------------------------ - -from pynput import keyboard - -import numpy as np -import cv2 -import hl2ss_imshow -import hl2ss - -# Settings -------------------------------------------------------------------- - -# HoloLens address -host = "192.168.1.7" - -# Port -port = hl2ss.StreamPort.RM_DEPTH_AHAT - -# Operating mode -# 0: video -# 1: video + rig pose -# 2: query calibration (single transfer) -mode = hl2ss.StreamMode.MODE_1 - -# Video encoding profile -profile = hl2ss.VideoProfile.RAW - -#------------------------------------------------------------------------------ - -if (mode == hl2ss.StreamMode.MODE_2): - data = hl2ss.download_calibration_rm_depth_ahat(host, port) - print('Calibration data') - print(data.uv2xy.shape) - print(data.extrinsics) - print(data.scale) - print(data.alias) - print(data.undistort_map.shape) - print(data.intrinsics) - quit() - -enable = True - -def on_press(key): - global enable - enable = key != keyboard.Key.esc - return enable - -listener = keyboard.Listener(on_press=on_press) -listener.start() - -client = hl2ss.rx_raw_rm_depth_ahat(host, port, hl2ss.ChunkSize.RM_DEPTH_AHAT, mode, profile, 1) -client.open() - -while (enable): - data = client.get_next_packet() - - print('Pose at time {ts}'.format(ts=data.timestamp)) - print(data.pose) - - cv2.imshow('Depth', data.payload.depth / np.max(data.payload.depth)) - cv2.imshow('AB', data.payload.ab) # max ~ 12000 - cv2.waitKey(1) - -client.close() -listener.join() diff --git a/viewer/client_rm_depth_longthrow.py b/viewer/client_rm_depth_longthrow.py index 63d8da87..8318b7d5 100644 --- a/viewer/client_rm_depth_longthrow.py +++ b/viewer/client_rm_depth_longthrow.py @@ -1,10 +1,10 @@ #------------------------------------------------------------------------------ -# This script receives encoded video from the HoloLens depth camera in long -# throw mode and plays it. The resolution is 320x288 @ 5 FPS. The stream -# supports three operating modes: 0) video, 1) video + rig pose, 2) query -# calibration (single transfer). Press esc to stop. Depth and AB data are -# scaled for visibility. Note that the ahat and long throw streams cannot be -# used simultaneously. +# This script receives video from the HoloLens depth camera in long throw mode +# and plays it. The resolution is 320x288 @ 5 FPS. The stream supports three +# operating modes: 0) video, 1) video + rig pose, 2) query calibration (single +# transfer). Depth and AB data are scaled for visibility. The ahat and long +# throw streams cannot be used simultaneously. +# Press esc to stop. #------------------------------------------------------------------------------ from pynput import keyboard @@ -36,10 +36,14 @@ if (mode == hl2ss.StreamMode.MODE_2): data = hl2ss.download_calibration_rm_depth_longthrow(host, port) print('Calibration data') - print(data.uv2xy.shape) + print('Image point to unit plane') + print(data.uv2xy) + print('Extrinsics') print(data.extrinsics) - print(data.scale) - print(data.undistort_map.shape) + print(f'Scale: {data.scale}') + print('Undistort map') + print(data.undistort_map) + print('Intrinsics (undistorted only)') print(data.intrinsics) quit() @@ -58,10 +62,10 @@ def on_press(key): while (enable): data = client.get_next_packet() - print('Pose at time {ts}'.format(ts=data.timestamp)) + print(f'Pose at time {data.timestamp}') print(data.pose) - cv2.imshow('Depth', data.payload.depth / np.max(data.payload.depth)) # Normalized for visibility - cv2.imshow('AB', data.payload.ab / np.max(data.payload.ab)) # Normalized for visibility + cv2.imshow('Depth', data.payload.depth / np.max(data.payload.depth)) # Scaled for visibility + cv2.imshow('AB', data.payload.ab / np.max(data.payload.ab)) # Scaled for visibility cv2.waitKey(1) client.close() diff --git a/viewer/client_rm_imu.py b/viewer/client_rm_imu.py index 66f66902..dd5e6813 100644 --- a/viewer/client_rm_imu.py +++ b/viewer/client_rm_imu.py @@ -1,12 +1,13 @@ #------------------------------------------------------------------------------ # This script receives IMU samples from the HoloLens and prints them. # Sensor details: -# Accelerometer: 93 samples per frame, sample rate ~1100 Hz -# Gyroscope: 315 samples per frame, sample rate ~6550 Hz -# Magnetometer: 11 samples per frame, sample rate ~50 Hz +# Accelerometer: 93 samples per frame, sample rate ~1100 Hz effective ~12 Hz +# Gyroscope: 315 samples per frame, sample rate ~7500 Hz effective ~24 Hz +# Magnetometer: 11 samples per frame, sample rate ~50 Hz effective ~5 Hz # The streams support three operating modes: 0) samples, 1) samples + rig pose, # 2) query calibration (single transfer), except for the magnetometer stream -# which does not support mode 2. Press esc to stop. +# which does not support mode 2. +# Press esc to stop. #------------------------------------------------------------------------------ from pynput import keyboard @@ -43,6 +44,7 @@ if (mode == hl2ss.StreamMode.MODE_2): data = hl2ss.download_calibration_rm_imu(host, port) print('Calibration data') + print('Extrinsics') print(data.extrinsics) quit() @@ -61,12 +63,13 @@ def on_press(key): while (enable): data = client.get_next_packet() - print('Pose at time {ts}'.format(ts=data.timestamp)) + print(f'Pose at time {data.timestamp}') print(data.pose) imu_data = hl2ss.unpack_rm_imu(data.payload) count = imu_data.get_count() sample = imu_data.get_frame(0) - print(f'Got {count} samples at time {data.timestamp}, first sample is (ticks = {sample.vinyl_hup_ticks} | {sample.soc_ticks}, x = {sample.x}, y = {sample.y}, z = {sample.z}, temperature={sample.temperature})') + print(f'Got {count} samples at time {data.timestamp}') + print(f'First sample is (ticks = {sample.vinyl_hup_ticks} | {sample.soc_ticks}, x = {sample.x}, y = {sample.y}, z = {sample.z}, temperature={sample.temperature})') client.close() listener.join() diff --git a/viewer/client_rm_vlc.py b/viewer/client_rm_vlc.py index 112a288f..f366be8a 100644 --- a/viewer/client_rm_vlc.py +++ b/viewer/client_rm_vlc.py @@ -1,8 +1,9 @@ #------------------------------------------------------------------------------ -# This script receives encoded video from one of the HoloLens sideview -# grayscale cameras and plays it. The camera resolution is 640x480 @ 30 FPS. -# The stream supports three operating modes: 0) video, 1) video + rig pose, -# 2) query calibration (single transfer). Press esc to stop. +# This script receives video from one of the HoloLens sideview grayscale +# cameras and plays it. The camera resolution is 640x480 @ 30 FPS. The stream +# supports three operating modes: 0) video, 1) video + rig pose, 2) query +# calibration (single transfer). +# Press esc to stop. #------------------------------------------------------------------------------ from pynput import keyboard @@ -42,9 +43,13 @@ if (mode == hl2ss.StreamMode.MODE_2): data = hl2ss.download_calibration_rm_vlc(host, port) print('Calibration data') - print(data.uv2xy.shape) + print('Image point to unit plane') + print(data.uv2xy) + print('Extrinsics') print(data.extrinsics) - print(data.undistort_map.shape) + print('Undistort map') + print(data.undistort_map) + print('Intrinsics (undistorted only)') print(data.intrinsics) quit() @@ -63,7 +68,7 @@ def on_press(key): while (enable): data = client.get_next_packet() - print('Pose at time {ts}'.format(ts=data.timestamp)) + print(f'Pose at time {data.timestamp}') print(data.pose) cv2.imshow('Video', data.payload) cv2.waitKey(1) diff --git a/viewer/client_rm_vlc_raw.py b/viewer/client_rm_vlc_raw.py deleted file mode 100644 index cf466a20..00000000 --- a/viewer/client_rm_vlc_raw.py +++ /dev/null @@ -1,69 +0,0 @@ -#------------------------------------------------------------------------------ -# This script receives uncompressed video from one of the HoloLens sideview -# grayscale cameras and plays it. The camera resolution is 640x480 @ 30 FPS. -# The stream supports three operating modes: 0) video, 1) video + rig pose, -# 2) query calibration (single transfer). Press esc to stop. -#------------------------------------------------------------------------------ - -from pynput import keyboard - -import numpy as np -import cv2 -import hl2ss_imshow -import hl2ss - -# Settings -------------------------------------------------------------------- - -# HoloLens address -host = "192.168.1.7" - -# Port -# Options: -# hl2ss.StreamPort.RM_VLC_LEFTFRONT -# hl2ss.StreamPort.RM_VLC_LEFTLEFT -# hl2ss.StreamPort.RM_VLC_RIGHTFRONT -# hl2ss.StreamPort.RM_VLC_RIGHTRIGHT -port = hl2ss.StreamPort.RM_VLC_LEFTFRONT - -# Operating mode -# 0: video -# 1: video + rig pose -# 2: query calibration (single transfer) -mode = hl2ss.StreamMode.MODE_1 - -# Video encoding profile -profile = hl2ss.VideoProfile.RAW - -#------------------------------------------------------------------------------ - -if (mode == hl2ss.StreamMode.MODE_2): - data = hl2ss.download_calibration_rm_vlc(host, port) - print('Calibration data') - print(data.uv2xy.shape) - print(data.extrinsics) - print(data.undistort_map.shape) - print(data.intrinsics) - quit() - -enable = True - -def on_press(key): - global enable - enable = key != keyboard.Key.esc - return enable - -listener = keyboard.Listener(on_press=on_press) -listener.start() - -client = hl2ss.rx_raw_rm_vlc(host, port, hl2ss.ChunkSize.RM_VLC, mode, profile, 1) -client.open() - -while (enable): - data = client.get_next_packet() - print('Pose at time {ts}'.format(ts=data.timestamp)) - print(data.pose) - cv2.imshow('Video', data.payload) - cv2.waitKey(1) - -client.close() -listener.join() diff --git a/viewer/client_si.py b/viewer/client_si.py index b0c5d2a8..3a221efc 100644 --- a/viewer/client_si.py +++ b/viewer/client_si.py @@ -1,8 +1,6 @@ #------------------------------------------------------------------------------ # This script receives spatial input data from the HoloLens, which comprises: -# 1) Head pose, 2) Eye ray, 3) Hand tracking. The HoloLens sends this data at -# display framerate (60 Hz). If the display framerate drops, so does this -# stream. +# 1) Head pose, 2) Eye ray, 3) Hand tracking. Sample rate is 30 Hz. #------------------------------------------------------------------------------ import hl2ss @@ -23,14 +21,14 @@ data = client.get_next_packet() si = hl2ss.unpack_si(data.payload) -print('Tracking status at time {ts}'.format(ts=data.timestamp)) +print(f'Tracking status at time {data.timestamp}') if (si.is_valid_head_pose()): head_pose = si.get_head_pose() print('Head pose') - print(head_pose.position) - print(head_pose.forward) - print(head_pose.up) + print(f'Position: {head_pose.position}') + print(f'Forward: {head_pose.forward}') + print(f'Up: {head_pose.up}') # right = cross(up, -forward) # up => y, forward => -z, right => x else: @@ -39,19 +37,23 @@ if (si.is_valid_eye_ray()): eye_ray = si.get_eye_ray() print('Eye ray') - print(eye_ray.origin) - print(eye_ray.direction) + print(f'Origin: {eye_ray.origin}') + print(f'Direction: {eye_ray.direction}') else: print('No eye tracking data') +# See +# https://learn.microsoft.com/en-us/uwp/api/windows.perception.people.jointpose?view=winrt-22621 +# for hand data details + if (si.is_valid_hand_left()): hand_left = si.get_hand_left() pose = hand_left.get_joint_pose(hl2ss.SI_HandJointKind.Wrist) print('Left wrist pose') - print(pose.orientation) - print(pose.position) - print(pose.radius) - print(pose.accuracy) + print(f'Orientation: {pose.orientation}') # Quaternion: x, y, z, w + print(f'Position: {pose.position}') + print(f'Radius: {pose.radius}') + print(f'Accuracy: {pose.accuracy}') # 0: High, 1: Approximate else: print('No left hand data') @@ -59,10 +61,10 @@ hand_right = si.get_hand_right() pose = hand_right.get_joint_pose(hl2ss.SI_HandJointKind.Wrist) print('Right wrist pose') - print(pose.orientation) - print(pose.position) - print(pose.radius) - print(pose.accuracy) + print(f'Orientation: {pose.orientation}') # Quaternion: x, y, z, w + print(f'Position: {pose.position}') + print(f'Radius: {pose.radius}') + print(f'Accuracy: {pose.accuracy}') # 0: High, 1: Approximate else: print('No right hand data') diff --git a/viewer/client_sm.py b/viewer/client_sm.py index 101a6ab7..ba42bfea 100644 --- a/viewer/client_sm.py +++ b/viewer/client_sm.py @@ -77,6 +77,7 @@ mesh.unpack(vpf, tif, vnf) + # Surface timestamps are given in Windows FILETIME (utc) print(f'Task {index}: surface id {id_hex} @ {timestamp} has {mesh.vertex_positions.shape[0]} vertices {mesh.triangle_indices.shape[0]} triangles {mesh.vertex_normals.shape[0]} normals') mesh.vertex_positions[:, 0:3] = mesh.vertex_positions[:, 0:3] * mesh.vertex_position_scale diff --git a/viewer/client_su.py b/viewer/client_su.py index 7e9e03a4..36c5fabf 100644 --- a/viewer/client_su.py +++ b/viewer/client_su.py @@ -1,5 +1,6 @@ #------------------------------------------------------------------------------ -# This script downloads Scene Understanding data. +# This script downloads Scene Understanding data from the HoloLens and displays +# it. #------------------------------------------------------------------------------ import open3d as o3d @@ -28,10 +29,26 @@ get_quad = True get_meshes = True get_collider_meshes = True -guid_list = [] + +# To track surfaces between scenes +# Create a new scene using SU_Create.NewFromPrevious and add the GUID of the +# surface(s) of interest found in the previous scene +# If the surface is found in the new scene it will be returned +guid_list = [] #------------------------------------------------------------------------------ +kind_color = { + hl2ss.SU_Kind.Background : [0, 0, 0], + hl2ss.SU_Kind.Ceiling : [0, 0, 1], + hl2ss.SU_Kind.CompletelyInferred : [0, 1, 0], + hl2ss.SU_Kind.Floor : [0, 1, 1], + hl2ss.SU_Kind.Platform : [1, 0, 0], + hl2ss.SU_Kind.Unknown : [1, 0, 1], + hl2ss.SU_Kind.Wall : [1, 1, 0], + hl2ss.SU_Kind.World : [1, 1, 1], +} + # Download data --------------------------------------------------------------- # See # https://learn.microsoft.com/en-us/windows/mixed-reality/develop/unity/scene-understanding-sdk @@ -73,7 +90,7 @@ for item in result.items: item.unpack() - print(f'SceneObject {item.id.hex()} {item.kind} {item.orientation} {item.position} {item.alignment} {item.extents}') + print(f'SceneObject ID={item.id.hex()} Kind={item.kind} Orientation={item.orientation} Position={item.position} Alignment={item.alignment} Extents={item.extents}') print('Location') print(item.location) print(f'Meshes: {len(item.meshes)}') @@ -85,6 +102,7 @@ open3d_mesh.vertices = o3d.utility.Vector3dVector((mesh.vertex_positions @ item.location[:3, :3]) + item.location[3, :3]) open3d_mesh.triangles = o3d.utility.Vector3iVector(mesh.triangle_indices) open3d_mesh.compute_vertex_normals() + open3d_mesh.paint_uniform_color(kind_color[int(item.kind)]) open3d_meshes.append(open3d_mesh) o3d.visualization.draw_geometries(open3d_meshes, mesh_show_back_face=True) diff --git a/viewer/client_umq.py b/viewer/client_umq.py index cead4f47..b3fd2685 100644 --- a/viewer/client_umq.py +++ b/viewer/client_umq.py @@ -28,7 +28,7 @@ def debug_message(self, text): # Command params: string encoded as utf-8 self.add(0xFFFFFFFE, text.encode('utf-8')) # Use the add method from hl2ss.umq_command_buffer to pack commands - # See rus.py and the unity_demo scripts for more examples. + # See hl2ss_rus.py and the unity_demo scripts for more examples. client = hl2ss.ipc_umq(host, hl2ss.IPCPort.UNITY_MESSAGE_QUEUE) # Create hl2ss client object diff --git a/viewer/client_vi.py b/viewer/client_vi.py index bae01ab3..5e28e470 100644 --- a/viewer/client_vi.py +++ b/viewer/client_vi.py @@ -1,5 +1,6 @@ #------------------------------------------------------------------------------ -# This script registers voice commands on the HoloLens. +# This script registers voice commands on the HoloLens and continously checks +# if any of the registered commands has been heard. # Press esc to stop. #------------------------------------------------------------------------------ @@ -46,12 +47,15 @@ def get_word(strings, index): client.create_recognizer() if (client.register_commands(True, strings)): print('Ready. Try saying any of the commands you defined.') - client.start() + client.start() while (enable): events = client.pop() for event in events: event.unpack() - print(f'Event: {get_word(strings, event.index)} {event.index} {event.confidence} {event.phrase_duration} {event.phrase_start_time} {event.raw_confidence}') + # See + # https://learn.microsoft.com/en-us/uwp/api/windows.media.speechrecognition.speechrecognitionresult?view=winrt-22621 + # for result details + print(f'Event: Command={get_word(strings, event.index)} Index={event.index} Confidence={event.confidence} Duration={event.phrase_duration} Start={event.phrase_start_time} RawConfidence={event.raw_confidence}') client.stop() client.clear() diff --git a/viewer/hl2ss.py b/viewer/hl2ss.py index 26d2e9da..445d0be6 100644 --- a/viewer/hl2ss.py +++ b/viewer/hl2ss.py @@ -45,6 +45,7 @@ class ChunkSize: PERSONAL_VIDEO = 4096 MICROPHONE = 512 SPATIAL_INPUT = 1024 + EXTENDED_EYE_TRACKER = 256 SINGLE_TRANSFER = 4096 @@ -707,7 +708,7 @@ def get_video_codec_bitrate(width, height, fps, factor): # RM VLC Decoder #------------------------------------------------------------------------------ -class decode_rm_vlc: +class _decode_rm_vlc: def __init__(self, profile): self.profile = profile @@ -721,6 +722,18 @@ def decode(self, payload): return None +class _unpack_rm_vlc: + def create(self): + pass + + def decode(self, payload): + return np.frombuffer(payload, dtype=np.uint8).reshape(Parameters_RM_VLC.SHAPE) + + +def decode_rm_vlc(profile): + return _unpack_rm_vlc() if (profile == VideoProfile.RAW) else _decode_rm_vlc(profile) + + #------------------------------------------------------------------------------ # RM Depth Decoder #------------------------------------------------------------------------------ @@ -755,7 +768,7 @@ def _unpack_rm_depth_ahat_nv12_as_yuv420p(yuv): return _RM_Depth_Frame(y, ab) -class decode_rm_depth_ahat: +class _decode_rm_depth_ahat: def __init__(self, profile): self.profile = profile @@ -769,6 +782,21 @@ def decode(self, payload): return None +class _unpack_rm_depth_ahat: + def create(self): + pass + + def decode(self, payload): + depth = np.frombuffer(payload, dtype=np.uint16, offset=0, count=Parameters_RM_DEPTH_AHAT.PIXELS).reshape(Parameters_RM_DEPTH_AHAT.SHAPE) + ab = np.frombuffer(payload, dtype=np.uint16, offset=Parameters_RM_DEPTH_AHAT.PIXELS*_SIZEOF.WORD, count=Parameters_RM_DEPTH_AHAT.PIXELS).reshape(Parameters_RM_DEPTH_AHAT.SHAPE) + depth[depth >= 4090] = 0 + return _RM_Depth_Frame(depth, ab) + + +def decode_rm_depth_ahat(profile): + return _unpack_rm_depth_ahat() if (profile == VideoProfile.RAW) else _decode_rm_depth_ahat(profile) + + def decode_rm_depth_longthrow(payload): composite = cv2.imdecode(np.frombuffer(payload, dtype=np.uint8), cv2.IMREAD_UNCHANGED) h, w, _ = composite.shape @@ -838,11 +866,11 @@ def get_nv12_stride(width): return width + ((64 - (width & 63)) & 63) -class decode_pv: +class _decode_pv: def __init__(self, profile): self.profile = profile - def create(self): + def create(self, width, height): self._codec = av.CodecContext.create(get_video_codec_name(self.profile), 'r') def decode(self, payload, format): @@ -852,11 +880,36 @@ def decode(self, payload, format): return None +class _unpack_pv: + _cv2_nv12_format = { + 'rgb24' : cv2.COLOR_YUV2RGB_NV12, + 'bgr24' : cv2.COLOR_YUV2BGR_NV12, + 'rgba' : cv2.COLOR_YUV2RGBA_NV12, + 'bgra' : cv2.COLOR_YUV2BGRA_NV12, + 'gray8' : cv2.COLOR_YUV2GRAY_NV12, + 'nv12' : None + } + + def create(self, width, height): + self.width = width + self.height = height + self.stride = get_nv12_stride(width) + + def decode(self, payload, format): + image = np.frombuffer(payload, dtype=np.uint8).reshape((int(self.height*3/2), self.stride))[:, :self.width] + sf = _unpack_pv._cv2_nv12_format[format] + return image if (sf is None) else cv2.cvtColor(image, sf) + + +def decode_pv(profile): + return _unpack_pv() if (profile == VideoProfile.RAW) else _decode_pv(profile) + + #------------------------------------------------------------------------------ # Microphone Decoder #------------------------------------------------------------------------------ -class decode_microphone: +class _decode_microphone: def __init__(self, profile): self.profile = profile @@ -870,6 +923,18 @@ def decode(self, payload): return None +class _unpack_microphone: + def create(self): + pass + + def decode(self, payload): + return np.frombuffer(payload, dtype=np.int16).reshape((1, -1)) + + +def decode_microphone(profile): + return _unpack_microphone() if (profile == AudioProfile.RAW) else _decode_microphone(profile) + + #------------------------------------------------------------------------------ # SI Unpacker #------------------------------------------------------------------------------ @@ -1118,7 +1183,7 @@ def __init__(self, host, port, chunk, mode, width, height, framerate, profile, b self._codec = decode_pv(profile) def open(self): - self._codec.create() + self._codec.create(self.width, self.height) super().open() self.get_next_packet() @@ -1150,48 +1215,6 @@ def close(self): super().close() -#------------------------------------------------------------------------------ -# RAW Receivers -#------------------------------------------------------------------------------ - -class rx_raw_rm_vlc(rx_rm_vlc): - def get_next_packet(self): - data = super().get_next_packet() - data.payload = np.frombuffer(data.payload, dtype=np.uint8).reshape(Parameters_RM_VLC.SHAPE) - return data - - -class rx_raw_rm_depth_ahat(rx_rm_depth_ahat): - def get_next_packet(self): - data = super().get_next_packet() - depth = np.frombuffer(data.payload, dtype=np.uint16, offset=0, count=Parameters_RM_DEPTH_AHAT.PIXELS).reshape(Parameters_RM_DEPTH_AHAT.SHAPE) - ab = np.frombuffer(data.payload, dtype=np.uint16, offset=Parameters_RM_DEPTH_AHAT.PIXELS*_SIZEOF.WORD, count=Parameters_RM_DEPTH_AHAT.PIXELS).reshape(Parameters_RM_DEPTH_AHAT.SHAPE) - depth[depth >= 4090] = 0 - data.payload = _RM_Depth_Frame(depth, ab) - return data - - -class rx_raw_pv(rx_pv): - _cv2_nv12_format = { - 'rgb24' : cv2.COLOR_YUV2RGB_NV12, - 'bgr24' : cv2.COLOR_YUV2BGR_NV12, - 'rgba' : cv2.COLOR_YUV2RGBA_NV12, - 'bgra' : cv2.COLOR_YUV2BGRA_NV12, - 'gray8' : cv2.COLOR_YUV2GRAY_NV12, - } - - def __init__(self, host, port, chunk, mode, width, height, framerate, profile, bitrate, format): - super().__init__(host, port, chunk, mode, width, height, framerate, profile, bitrate) - self.format = rx_raw_pv._cv2_nv12_format[format] - self.stride = get_nv12_stride(self.width) - - def get_next_packet(self): - data = super().get_next_packet() - data.payload = unpack_pv(data.payload) - data.payload.image = cv2.cvtColor(np.frombuffer(data.payload.image, dtype=np.uint8).reshape((int(self.height*3/2), self.stride))[:, :self.width], self.format) - return data - - #------------------------------------------------------------------------------ # Mode 2 Data Acquisition #------------------------------------------------------------------------------ @@ -1820,14 +1843,14 @@ class SU_Create: class SU_Kind: - Background = 0, - Wall = 1, - Floor = 2, - Ceiling = 3, - Platform = 4, - Unknown = 247, - World = 248, - CompletelyInferred = 249, + Background = 0 + Wall = 1 + Floor = 2 + Ceiling = 3 + Platform = 4 + Unknown = 247 + World = 248 + CompletelyInferred = 249 class su_task: diff --git a/viewer/hl2ss_io.py b/viewer/hl2ss_io.py index 4bdc5ace..be582c8b 100644 --- a/viewer/hl2ss_io.py +++ b/viewer/hl2ss_io.py @@ -30,12 +30,8 @@ def close(self): # Header Pack #------------------------------------------------------------------------------ -def _create_header(port): - return struct.pack(f'<{len(_MAGIC)}sH', _MAGIC.encode(), port) - - -def _create_user(user): - return struct.pack(' max) else v - - #------------------------------------------------------------------------------ # Timing #------------------------------------------------------------------------------ diff --git a/viewer/demo_2d_to_3d_segmentation.py b/viewer/sample_2d_to_3d_segmentation.py similarity index 100% rename from viewer/demo_2d_to_3d_segmentation.py rename to viewer/sample_2d_to_3d_segmentation.py diff --git a/viewer/open3d_pointcloud.py b/viewer/sample_depth_to_pointcloud.py similarity index 98% rename from viewer/open3d_pointcloud.py rename to viewer/sample_depth_to_pointcloud.py index c3d1d2be..7a464197 100644 --- a/viewer/open3d_pointcloud.py +++ b/viewer/sample_depth_to_pointcloud.py @@ -6,7 +6,6 @@ import open3d as o3d import hl2ss import hl2ss_3dcv -import hl2ss_utilities # Settings -------------------------------------------------------------------- diff --git a/viewer/open3d_integrator.py b/viewer/sample_integrator.py similarity index 68% rename from viewer/open3d_integrator.py rename to viewer/sample_integrator.py index a6ffbc78..86432994 100644 --- a/viewer/open3d_integrator.py +++ b/viewer/sample_integrator.py @@ -1,5 +1,6 @@ #------------------------------------------------------------------------------ -# RGBD integration using Open3D. Press space to stop. +# RGBD integration using Open3D. "Color" information come from AB component. +# Press space to stop. #------------------------------------------------------------------------------ from pynput import keyboard @@ -16,7 +17,7 @@ # HoloLens address host = '192.168.1.7' -# Calibration path +# Calibration path (must exist but can be empty) calibration_path = '../calibration' # Buffer length in seconds @@ -30,6 +31,7 @@ #------------------------------------------------------------------------------ if __name__ == '__main__': + # Keyboard events --------------------------------------------------------- enable = True def on_press(key): @@ -40,46 +42,59 @@ def on_press(key): listener = keyboard.Listener(on_press=on_press) listener.start() + # Get RM Depth Long Throw calibration ------------------------------------- + # Calibration data will be downloaded if it's not in the calibration folder calibration_lt = hl2ss_3dcv.get_calibration_rm(host, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, calibration_path) uv2xy = hl2ss_3dcv.compute_uv2xy(calibration_lt.intrinsics, hl2ss.Parameters_RM_DEPTH_LONGTHROW.WIDTH, hl2ss.Parameters_RM_DEPTH_LONGTHROW.HEIGHT) xy1, scale = hl2ss_3dcv.rm_depth_compute_rays(uv2xy, calibration_lt.scale) + # Create Open3D integrator and visualizer --------------------------------- volume = o3d.pipelines.integration.ScalableTSDFVolume(voxel_length=voxel_length, sdf_trunc=sdf_trunc, color_type=o3d.pipelines.integration.TSDFVolumeColorType.RGB8) intrinsics_depth = o3d.camera.PinholeCameraIntrinsic(hl2ss.Parameters_RM_DEPTH_LONGTHROW.WIDTH, hl2ss.Parameters_RM_DEPTH_LONGTHROW.HEIGHT, calibration_lt.intrinsics[0, 0], calibration_lt.intrinsics[1, 1], calibration_lt.intrinsics[2, 0], calibration_lt.intrinsics[2, 1]) + vis = o3d.visualization.Visualizer() vis.create_window() first_pcd = True + # Start RM Depth Long Throw stream ---------------------------------------- producer = hl2ss_mp.producer() producer.configure_rm_depth_longthrow(True, host, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, hl2ss.ChunkSize.RM_DEPTH_LONGTHROW, hl2ss.StreamMode.MODE_1, hl2ss.PngFilterMode.Paeth) producer.initialize(hl2ss.StreamPort.RM_DEPTH_LONGTHROW, hl2ss.Parameters_RM_DEPTH_LONGTHROW.FPS * buffer_length) producer.start(hl2ss.StreamPort.RM_DEPTH_LONGTHROW) - manager = mp.Manager() consumer = hl2ss_mp.consumer() + manager = mp.Manager() sink_depth = consumer.create_sink(producer, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, manager, ...) - sinks = [sink_depth] - - [sink.get_attach_response() for sink in sinks] + sink_depth.get_attach_response() + # Main Loop --------------------------------------------------------------- while (enable): + # Wait for RM Depth Long Throw frame ---------------------------------- sink_depth.acquire() + # Get RM Depth Long Throw frame --------------------------------------- _, data_depth = sink_depth.get_most_recent_frame() if ((data_depth is None) or (not hl2ss.is_valid_pose(data_depth.pose))): continue + # Preprocess frames --------------------------------------------------- depth = hl2ss_3dcv.rm_depth_undistort(data_depth.payload.depth, calibration_lt.undistort_map) depth = hl2ss_3dcv.rm_depth_normalize(depth, scale) color = cv2.remap(data_depth.payload.ab, calibration_lt.undistort_map[:, :, 0], calibration_lt.undistort_map[:, :, 1], cv2.INTER_LINEAR) + + # Convert to Open3D RGBD image ---------------------------------------- color = hl2ss_3dcv.rm_depth_to_uint8(color) color = hl2ss_3dcv.rm_depth_to_rgb(color) - - rgbd = o3d.geometry.RGBDImage.create_from_color_and_depth(o3d.geometry.Image(color), o3d.geometry.Image(depth), depth_scale=1, depth_trunc=max_depth, convert_rgb_to_intensity=False) + color_image = o3d.geometry.Image(color) + depth_image = o3d.geometry.Image(depth) + rgbd = o3d.geometry.RGBDImage.create_from_color_and_depth(color_image, depth_image, depth_scale=1, depth_trunc=max_depth, convert_rgb_to_intensity=False) + + # Compute world to RM Depth Long Throw camera transformation matrix --- depth_world_to_camera = hl2ss_3dcv.world_to_reference(data_depth.pose) @ hl2ss_3dcv.rignode_to_camera(calibration_lt.extrinsics) + # Integrate RGBD and display point cloud ------------------------------ volume.integrate(rgbd, intrinsics_depth, depth_world_to_camera.transpose()) pcd_tmp = volume.extract_point_cloud() @@ -95,8 +110,12 @@ def on_press(key): vis.poll_events() vis.update_renderer() - [sink.detach() for sink in sinks] + # Stop RM Depth Long Throw stream ----------------------------------------- + sink_depth.detach() producer.stop(hl2ss.StreamPort.RM_DEPTH_LONGTHROW) + + # Stop keyboard events ---------------------------------------------------- listener.join() + # Show final point cloud -------------------------------------------------- vis.run() diff --git a/viewer/open3d_integrator_pv.py b/viewer/sample_integrator_pv.py similarity index 68% rename from viewer/open3d_integrator_pv.py rename to viewer/sample_integrator_pv.py index 725f98d3..a36a7910 100644 --- a/viewer/open3d_integrator_pv.py +++ b/viewer/sample_integrator_pv.py @@ -1,8 +1,8 @@ #------------------------------------------------------------------------------ # RGBD integration using Open3D. Color information comes from the front RGB -# camera. Press space to stop. +# camera. +# Press space to stop. #------------------------------------------------------------------------------ -# TODO: USE CAMERA POSES FOR MORE ACCURATE REGISTRATION from pynput import keyboard @@ -19,15 +19,15 @@ # HoloLens address host = '192.168.1.7' +# Calibration path (must exist but can be empty) calibration_path = '../calibration' # Camera parameters -focus = 1000 width = 640 height = 360 framerate = 30 profile = hl2ss.VideoProfile.H265_MAIN -bitrate = 5*1024*1024 +bitrate = hl2ss.get_video_codec_bitrate(width, height, framerate, hl2ss.get_video_codec_default_factor(profile)) exposure_mode = hl2ss.PV_ExposureMode.Manual exposure = hl2ss.PV_ExposureValue.Max // 4 iso_speed_mode = hl2ss.PV_IsoSpeedMode.Manual @@ -45,6 +45,7 @@ #------------------------------------------------------------------------------ if __name__ == '__main__': + # Keyboard events --------------------------------------------------------- enable = True def on_press(key): @@ -55,13 +56,26 @@ def on_press(key): listener = keyboard.Listener(on_press=on_press) listener.start() + # Start PV Subsystem ------------------------------------------------------ hl2ss.start_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO) + # Wait for PV subsystem and fix exposure, iso speed, and white balance ---- + ipc_rc = hl2ss.ipc_rc(host, hl2ss.IPCPort.REMOTE_CONFIGURATION) + ipc_rc.open() + ipc_rc.wait_for_pv_subsystem(True) + ipc_rc.set_pv_exposure(exposure_mode, exposure) + ipc_rc.set_pv_iso_speed(iso_speed_mode, iso_speed_value) + ipc_rc.set_pv_white_balance_preset(white_balance) + ipc_rc.close() + + # Get RM Depth Long Throw calibration ------------------------------------- + # Calibration data will be downloaded if it's not in the calibration folder calibration_lt = hl2ss_3dcv.get_calibration_rm(host, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, calibration_path) uv2xy = hl2ss_3dcv.compute_uv2xy(calibration_lt.intrinsics, hl2ss.Parameters_RM_DEPTH_LONGTHROW.WIDTH, hl2ss.Parameters_RM_DEPTH_LONGTHROW.HEIGHT) xy1, scale = hl2ss_3dcv.rm_depth_compute_rays(uv2xy, calibration_lt.scale) + # Create Open3D integrator and visualizer --------------------------------- volume = o3d.pipelines.integration.ScalableTSDFVolume(voxel_length=voxel_length, sdf_trunc=sdf_trunc, color_type=o3d.pipelines.integration.TSDFVolumeColorType.RGB8) intrinsics_depth = o3d.camera.PinholeCameraIntrinsic(hl2ss.Parameters_RM_DEPTH_LONGTHROW.WIDTH, hl2ss.Parameters_RM_DEPTH_LONGTHROW.HEIGHT, calibration_lt.intrinsics[0, 0], calibration_lt.intrinsics[1, 1], calibration_lt.intrinsics[2, 0], calibration_lt.intrinsics[2, 1]) @@ -69,6 +83,7 @@ def on_press(key): vis.create_window() first_pcd = True + # Start PV and RM Depth Long Throw streams -------------------------------- producer = hl2ss_mp.producer() producer.configure_pv(True, host, hl2ss.StreamPort.PERSONAL_VIDEO, hl2ss.ChunkSize.PERSONAL_VIDEO, hl2ss.StreamMode.MODE_1, width, height, framerate, profile, bitrate, 'rgb24') producer.configure_rm_depth_longthrow(True, host, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, hl2ss.ChunkSize.RM_DEPTH_LONGTHROW, hl2ss.StreamMode.MODE_1, hl2ss.PngFilterMode.Paeth) @@ -77,20 +92,24 @@ def on_press(key): producer.start(hl2ss.StreamPort.PERSONAL_VIDEO) producer.start(hl2ss.StreamPort.RM_DEPTH_LONGTHROW) - manager = mp.Manager() consumer = hl2ss_mp.consumer() + manager = mp.Manager() sink_pv = consumer.create_sink(producer, hl2ss.StreamPort.PERSONAL_VIDEO, manager, None) sink_depth = consumer.create_sink(producer, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, manager, ...) - sinks = [sink_pv, sink_depth] - - [sink.get_attach_response() for sink in sinks] + sink_pv.get_attach_response() + sink_depth.get_attach_response() + # Initialize PV intrinsics and extrinsics --------------------------------- pv_intrinsics = hl2ss.create_pv_intrinsics_placeholder() + pv_extrinsics = np.eye(4, 4, dtype=np.float32) + # Main Loop --------------------------------------------------------------- while (enable): + # Wait for RM Depth Long Throw frame ---------------------------------- sink_depth.acquire() + # Get RM Depth Long Throw frame and nearest (in time) PV frame -------- _, data_lt = sink_depth.get_most_recent_frame() if ((data_lt is None) or (not hl2ss.is_valid_pose(data_lt.pose))): continue @@ -99,15 +118,21 @@ def on_press(key): if ((data_pv is None) or (not hl2ss.is_valid_pose(data_pv.pose))): continue + # Preprocess frames --------------------------------------------------- depth = hl2ss_3dcv.rm_depth_undistort(data_lt.payload.depth, calibration_lt.undistort_map) depth = hl2ss_3dcv.rm_depth_normalize(depth, scale) color = data_pv.payload.image - pv_intrinsics = hl2ss.update_pv_intrinsics(pv_intrinsics, data_pv.payload.focal_length, data_pv.payload.principal_point) + # Update PV intrinsics ------------------------------------------------ + # PV intrinsics may change between frames due to autofocus + pv_intrinsics = hl2ss.update_pv_intrinsics(pv_intrinsics, data_pv.payload.focal_length, data_pv.payload.principal_point) + color_intrinsics, color_extrinsics = hl2ss_3dcv.pv_fix_calibration(pv_intrinsics, pv_extrinsics) + + # Generate aligned RGBD image ----------------------------------------- lt_points = hl2ss_3dcv.rm_depth_to_points(xy1, depth) lt_to_world = hl2ss_3dcv.camera_to_rignode(calibration_lt.extrinsics) @ hl2ss_3dcv.reference_to_world(data_lt.pose) world_to_lt = hl2ss_3dcv.world_to_reference(data_lt.pose) @ hl2ss_3dcv.rignode_to_camera(calibration_lt.extrinsics) - world_to_pv_image = hl2ss_3dcv.world_to_reference(data_pv.pose) @ hl2ss_3dcv.camera_to_image(pv_intrinsics) + world_to_pv_image = hl2ss_3dcv.world_to_reference(data_pv.pose) @ hl2ss_3dcv.rignode_to_camera(color_extrinsics) @ hl2ss_3dcv.camera_to_image(color_intrinsics) world_points = hl2ss_3dcv.transform(lt_points, lt_to_world) pv_uv = hl2ss_3dcv.project(world_points, world_to_pv_image) color = cv2.remap(color, pv_uv[:, :, 0], pv_uv[:, :, 1], cv2.INTER_LINEAR) @@ -115,12 +140,15 @@ def on_press(key): mask_uv = hl2ss_3dcv.slice_to_block((pv_uv[:, :, 0] < 0) | (pv_uv[:, :, 0] >= width) | (pv_uv[:, :, 1] < 0) | (pv_uv[:, :, 1] >= height)) depth[mask_uv] = 0 + # Convert to Open3D RGBD image ---------------------------------------- color_image = o3d.geometry.Image(color) depth_image = o3d.geometry.Image(depth) rgbd = o3d.geometry.RGBDImage.create_from_color_and_depth(color_image, depth_image, depth_scale=1, depth_trunc=max_depth, convert_rgb_to_intensity=False) + # Compute world to RM Depth Long Throw camera transformation matrix --- depth_world_to_camera = hl2ss_3dcv.world_to_reference(data_lt.pose) @ hl2ss_3dcv.rignode_to_camera(calibration_lt.extrinsics) + # Integrate RGBD and display point cloud ------------------------------ volume.integrate(rgbd, intrinsics_depth, depth_world_to_camera.transpose()) pcd_tmp = volume.extract_point_cloud() @@ -136,11 +164,18 @@ def on_press(key): vis.poll_events() vis.update_renderer() - [sink.detach() for sink in sinks] + # Stop PV and RM Depth Long Throw streams --------------------------------- + sink_pv.detach() + sink_depth.detach() producer.stop(hl2ss.StreamPort.PERSONAL_VIDEO) producer.stop(hl2ss.StreamPort.RM_DEPTH_LONGTHROW) + + # Stop PV subsystem ------------------------------------------------------- + hl2ss.stop_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO) + + # Stop keyboard events ---------------------------------------------------- listener.join() + # Show final point cloud -------------------------------------------------- vis.run() - hl2ss.stop_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO) diff --git a/viewer/open3d_integrator_rm_vlc.py b/viewer/sample_integrator_rm_vlc.py similarity index 58% rename from viewer/open3d_integrator_rm_vlc.py rename to viewer/sample_integrator_rm_vlc.py index 07f05473..886288b5 100644 --- a/viewer/open3d_integrator_rm_vlc.py +++ b/viewer/sample_integrator_rm_vlc.py @@ -1,6 +1,7 @@ #------------------------------------------------------------------------------ -# RGBD integration using Open3D. Color information comes from one of the -# sideview grayscale cameras. Press space to stop. +# RGBD integration using Open3D. "Color" information comes from one of the +# sideview grayscale cameras. +# Press space to stop. #------------------------------------------------------------------------------ from pynput import keyboard @@ -17,12 +18,13 @@ # HoloLens address host = '192.168.1.7' +# Calibration path (must exist but can be empty) calibration_path = '../calibration' # Camera selection and parameters port = hl2ss.StreamPort.RM_VLC_LEFTFRONT -profile = hl2ss.VideoProfile.H264_HIGH -bitrate = 2*1024*1024 +profile = hl2ss.VideoProfile.H265_MAIN +bitrate = hl2ss.get_video_codec_bitrate(hl2ss.Parameters_RM_VLC.WIDTH, hl2ss.Parameters_RM_VLC.HEIGHT, hl2ss.Parameters_RM_VLC.FPS, hl2ss.get_video_codec_default_factor(profile)) # Buffer length in seconds buffer_length = 10 @@ -35,6 +37,7 @@ #------------------------------------------------------------------------------ if __name__ == '__main__': + # Keyboard events --------------------------------------------------------- enable = True def on_press(key): @@ -45,18 +48,23 @@ def on_press(key): listener = keyboard.Listener(on_press=on_press) listener.start() + # Get RM VLC and RM Depth Long Throw calibration -------------------------- + # Calibration data will be downloaded if it's not in the calibration folder calibration_vlc = hl2ss_3dcv.get_calibration_rm(host, port, calibration_path) calibration_lt = hl2ss_3dcv.get_calibration_rm(host, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, calibration_path) uv2xy = hl2ss_3dcv.compute_uv2xy(calibration_lt.intrinsics, hl2ss.Parameters_RM_DEPTH_LONGTHROW.WIDTH, hl2ss.Parameters_RM_DEPTH_LONGTHROW.HEIGHT) xy1, scale = hl2ss_3dcv.rm_depth_compute_rays(uv2xy, calibration_lt.scale) + # Create Open3D integrator and visualizer --------------------------------- volume = o3d.pipelines.integration.ScalableTSDFVolume(voxel_length=voxel_length, sdf_trunc=sdf_trunc, color_type=o3d.pipelines.integration.TSDFVolumeColorType.RGB8) intrinsics_depth = o3d.camera.PinholeCameraIntrinsic(hl2ss.Parameters_RM_DEPTH_LONGTHROW.WIDTH, hl2ss.Parameters_RM_DEPTH_LONGTHROW.HEIGHT, calibration_lt.intrinsics[0, 0], calibration_lt.intrinsics[1, 1], calibration_lt.intrinsics[2, 0], calibration_lt.intrinsics[2, 1]) + vis = o3d.visualization.Visualizer() vis.create_window() first_pcd = True + # Start RM VLC and RM Depth Long Throw streams ---------------------------- producer = hl2ss_mp.producer() producer.configure_rm_vlc(True, host, port, hl2ss.ChunkSize.RM_VLC, hl2ss.StreamMode.MODE_1, profile, bitrate) producer.configure_rm_depth_longthrow(True, host, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, hl2ss.ChunkSize.RM_DEPTH_LONGTHROW, hl2ss.StreamMode.MODE_1, hl2ss.PngFilterMode.Paeth) @@ -65,18 +73,20 @@ def on_press(key): producer.start(port) producer.start(hl2ss.StreamPort.RM_DEPTH_LONGTHROW) - manager = mp.Manager() consumer = hl2ss_mp.consumer() + manager = mp.Manager() sink_vlc = consumer.create_sink(producer, port, manager, None) sink_depth = consumer.create_sink(producer, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, manager, ...) - sinks = [sink_vlc, sink_depth] - - [sink.get_attach_response() for sink in sinks] + sink_vlc.get_attach_response() + sink_depth.get_attach_response() + # Main Loop --------------------------------------------------------------- while (enable): + # Wait for RM Depth Long Throw frame ---------------------------------- sink_depth.acquire() + # Get RM Depth Long Throw frame and nearest (in time) RM VLC frame ---- _, data_depth = sink_depth.get_most_recent_frame() if ((data_depth is None) or (not hl2ss.is_valid_pose(data_depth.pose))): continue @@ -85,25 +95,33 @@ def on_press(key): if ((data_vlc is None) or (not hl2ss.is_valid_pose(data_vlc.pose))): continue + # Preprocess frames --------------------------------------------------- depth = hl2ss_3dcv.rm_depth_undistort(data_depth.payload.depth, calibration_lt.undistort_map) depth = hl2ss_3dcv.rm_depth_normalize(depth, scale) color = cv2.remap(data_vlc.payload, calibration_vlc.undistort_map[:, :, 0], calibration_vlc.undistort_map[:, :, 1], cv2.INTER_LINEAR) - lt_points = hl2ss_3dcv.rm_depth_to_points(xy1, depth) - lt_to_world = hl2ss_3dcv.camera_to_rignode(calibration_lt.extrinsics) @ hl2ss_3dcv.reference_to_world(data_depth.pose) - world_to_lt = hl2ss_3dcv.world_to_reference(data_depth.pose) @ hl2ss_3dcv.rignode_to_camera(calibration_lt.extrinsics) - world_to_pv_image = hl2ss_3dcv.world_to_reference(data_vlc.pose) @ hl2ss_3dcv.rignode_to_camera(calibration_vlc.extrinsics) @ hl2ss_3dcv.camera_to_image(calibration_vlc.intrinsics) - world_points = hl2ss_3dcv.transform(lt_points, lt_to_world) - pv_uv = hl2ss_3dcv.project(world_points, world_to_pv_image) - color = cv2.remap(color, pv_uv[:, :, 0], pv_uv[:, :, 1], cv2.INTER_LINEAR) + # Generate aligned RGBD image ----------------------------------------- + lt_points = hl2ss_3dcv.rm_depth_to_points(xy1, depth) + lt_to_world = hl2ss_3dcv.camera_to_rignode(calibration_lt.extrinsics) @ hl2ss_3dcv.reference_to_world(data_depth.pose) + world_to_lt = hl2ss_3dcv.world_to_reference(data_depth.pose) @ hl2ss_3dcv.rignode_to_camera(calibration_lt.extrinsics) + world_to_vlc_image = hl2ss_3dcv.world_to_reference(data_vlc.pose) @ hl2ss_3dcv.rignode_to_camera(calibration_vlc.extrinsics) @ hl2ss_3dcv.camera_to_image(calibration_vlc.intrinsics) + world_points = hl2ss_3dcv.transform(lt_points, lt_to_world) + vlc_uv = hl2ss_3dcv.project(world_points, world_to_vlc_image) + color = cv2.remap(color, vlc_uv[:, :, 0], vlc_uv[:, :, 1], cv2.INTER_LINEAR) - mask_uv = hl2ss_3dcv.slice_to_block((pv_uv[:, :, 0] < 0) | (pv_uv[:, :, 0] >= hl2ss.Parameters_RM_VLC.WIDTH) | (pv_uv[:, :, 1] < 0) | (pv_uv[:, :, 1] >= hl2ss.Parameters_RM_VLC.HEIGHT)) + mask_uv = hl2ss_3dcv.slice_to_block((vlc_uv[:, :, 0] < 0) | (vlc_uv[:, :, 0] >= hl2ss.Parameters_RM_VLC.WIDTH) | (vlc_uv[:, :, 1] < 0) | (vlc_uv[:, :, 1] >= hl2ss.Parameters_RM_VLC.HEIGHT)) depth[mask_uv] = 0 + # Convert to Open3D RGBD image ---------------------------------------- color = hl2ss_3dcv.rm_vlc_to_rgb(color) - rgbd = o3d.geometry.RGBDImage.create_from_color_and_depth(o3d.geometry.Image(color), o3d.geometry.Image(depth), depth_scale=1, depth_trunc=max_depth, convert_rgb_to_intensity=False) + color_image = o3d.geometry.Image(color) + depth_image = o3d.geometry.Image(depth) + rgbd = o3d.geometry.RGBDImage.create_from_color_and_depth(color_image, depth_image, depth_scale=1, depth_trunc=max_depth, convert_rgb_to_intensity=False) + + # Compute world to RM Depth Long Throw camera transformation matrix --- depth_world_to_camera = hl2ss_3dcv.world_to_reference(data_depth.pose) @ hl2ss_3dcv.rignode_to_camera(calibration_lt.extrinsics) + # Integrate RGBD and display point cloud ------------------------------ volume.integrate(rgbd, intrinsics_depth, depth_world_to_camera.transpose()) pcd_tmp = volume.extract_point_cloud() @@ -119,9 +137,14 @@ def on_press(key): vis.poll_events() vis.update_renderer() - [sink.detach() for sink in sinks] + # Stop RM VLC and RM Depth Long Throw streams ----------------------------- + sink_vlc.detach() + sink_depth.detach() producer.stop(port) producer.stop(hl2ss.StreamPort.RM_DEPTH_LONGTHROW) + + # Stop keyboard events ---------------------------------------------------- listener.join() + # Show final point cloud -------------------------------------------------- vis.run() diff --git a/viewer/client_pv_microphone.py b/viewer/sample_pv_microphone.py similarity index 100% rename from viewer/client_pv_microphone.py rename to viewer/sample_pv_microphone.py diff --git a/viewer/client_pv_si.py b/viewer/sample_pv_si.py similarity index 100% rename from viewer/client_pv_si.py rename to viewer/sample_pv_si.py diff --git a/viewer/open3d_rgbd.py b/viewer/sample_rgbd.py similarity index 100% rename from viewer/open3d_rgbd.py rename to viewer/sample_rgbd.py diff --git a/viewer/open3d_rgbd_ahat_vlc.py b/viewer/sample_rgbd_ahat_vlc.py similarity index 100% rename from viewer/open3d_rgbd_ahat_vlc.py rename to viewer/sample_rgbd_ahat_vlc.py diff --git a/viewer/open3d_rgbd_pv.py b/viewer/sample_rgbd_pv.py similarity index 100% rename from viewer/open3d_rgbd_pv.py rename to viewer/sample_rgbd_pv.py diff --git a/viewer/open3d_rgbd_rm_vlc.py b/viewer/sample_rgbd_rm_vlc.py similarity index 100% rename from viewer/open3d_rgbd_rm_vlc.py rename to viewer/sample_rgbd_rm_vlc.py diff --git a/viewer/client_rm_vlc_si.py b/viewer/sample_rm_vlc_si.py similarity index 100% rename from viewer/client_rm_vlc_si.py rename to viewer/sample_rm_vlc_si.py diff --git a/viewer/opencv_stereo_rectify.py b/viewer/sample_stereo_rectify.py similarity index 100% rename from viewer/opencv_stereo_rectify.py rename to viewer/sample_stereo_rectify.py diff --git a/viewer/opencv_stereo_rectify_pv.py b/viewer/sample_stereo_rectify_pv.py similarity index 100% rename from viewer/opencv_stereo_rectify_pv.py rename to viewer/sample_stereo_rectify_pv.py diff --git a/viewer/demo_video.py b/viewer/sample_video.py similarity index 100% rename from viewer/demo_video.py rename to viewer/sample_video.py diff --git a/viewer/open3d_viewer_si.py b/viewer/sample_viewer_si.py similarity index 100% rename from viewer/open3d_viewer_si.py rename to viewer/sample_viewer_si.py diff --git a/viewer/simple_player.py b/viewer/simple_player.py index 8633ec16..5eac2489 100644 --- a/viewer/simple_player.py +++ b/viewer/simple_player.py @@ -59,13 +59,18 @@ def display_pv(data): cv2.imshow('vlc', data.payload.image) cv2.waitKey(1) -microphone_buffer = np.empty((1,0), dtype=np.float32) -def display_microphone(data): + +def display_microphone_aac(data): global microphone_buffer print(f'Samples at time {data.timestamp}') microphone_buffer = np.hstack((microphone_buffer, hl2ss_utilities.microphone_planar_to_packed(data.payload))) +def display_microphone_raw(data): + global microphone_buffer + print(f'Samples at time {data.timestamp}') + microphone_buffer = np.hstack((microphone_buffer, data.payload)) + def display_si(data): si = hl2ss.unpack_si(data.payload) @@ -112,6 +117,33 @@ def display_si(data): else: print('No right hand data') +def display_eet(data): + data.payload = hl2ss.unpack_eet(data.payload) + + # See + # https://learn.microsoft.com/en-us/windows/mixed-reality/develop/native/extended-eye-tracking-native + # for details + + print(f'Tracking status at time {data.timestamp}') + print('Pose') + print(data.pose) + print(f'Calibration valid: {data.payload.calibration_valid}') + print(f'Combined eye gaze: Valid={data.payload.combined_ray_valid} Origin={data.payload.combined_ray.origin} Direction={data.payload.combined_ray.direction}') + print(f'Left eye gaze: Valid={data.payload.left_ray_valid} Origin={data.payload.left_ray.origin} Direction={data.payload.left_ray.direction}') + print(f'Right eye gaze: Valid={data.payload.right_ray_valid} Origin={data.payload.right_ray.origin} Direction={data.payload.right_ray.direction}') + + # "...not supported by HoloLens 2 at this time" + print(f'Left eye openness: Valid={data.payload.left_openness_valid} Value={data.payload.left_openness}') + print(f'Right eye openness: Valid={data.payload.right_openness_valid} Value={data.payload.right_openness}') + print(f'Vergence distance: Valid={data.payload.vergence_distance_valid} Value={data.payload.vergence_distance}') + + + +filename = os.path.join(path, f'{hl2ss.get_port_name(port)}.bin') +reader = hl2ss_io.create_rd(True, filename, hl2ss.ChunkSize.SINGLE_TRANSFER, 'bgr24') +reader.open() + +microphone_buffer = np.empty((1,0), dtype=np.int16 if (reader.header.profile == hl2ss.AudioProfile.RAW) else np.float32) display_method = { hl2ss.StreamPort.RM_VLC_LEFTFRONT : display_vlc, @@ -124,14 +156,11 @@ def display_si(data): hl2ss.StreamPort.RM_IMU_GYROSCOPE : display_imu, hl2ss.StreamPort.RM_IMU_MAGNETOMETER : display_imu, hl2ss.StreamPort.PERSONAL_VIDEO : display_pv, - hl2ss.StreamPort.MICROPHONE : display_microphone, - hl2ss.StreamPort.SPATIAL_INPUT : display_si + hl2ss.StreamPort.MICROPHONE : display_microphone_raw if (reader.header.profile == hl2ss.AudioProfile.RAW) else display_microphone_aac, + hl2ss.StreamPort.SPATIAL_INPUT : display_si, + hl2ss.StreamPort.EXTENDED_EYE_TRACKER : display_eet, } -filename = os.path.join(path, f'{hl2ss.get_port_name(port)}.bin') -reader = hl2ss_io.create_rd(True, filename, hl2ss.ChunkSize.SINGLE_TRANSFER, 'bgr24') -reader.open() - while (True): data = reader.read() if (data is None): @@ -140,8 +169,13 @@ def display_si(data): reader.close() +if (reader.header.port != hl2ss.StreamPort.MICROPHONE): + quit() + +print(reader.header.profile) + p = pyaudio.PyAudio() -stream = p.open(format=pyaudio.paFloat32, channels=hl2ss.Parameters_MICROPHONE.CHANNELS, rate=hl2ss.Parameters_MICROPHONE.SAMPLE_RATE, output=True) +stream = p.open(format=pyaudio.paInt16 if (reader.header.profile == hl2ss.AudioProfile.RAW) else pyaudio.paFloat32, channels=hl2ss.Parameters_MICROPHONE.CHANNELS, rate=hl2ss.Parameters_MICROPHONE.SAMPLE_RATE, output=True) stream.start_stream() stream.write(microphone_buffer.tobytes()) stream.stop_stream() diff --git a/viewer/simple_recorder.py b/viewer/simple_recorder.py index 7c35b8ca..8118b832 100644 --- a/viewer/simple_recorder.py +++ b/viewer/simple_recorder.py @@ -1,5 +1,4 @@ -import multiprocessing as mp import time import os import hl2ss @@ -22,34 +21,41 @@ #hl2ss.StreamPort.RM_IMU_MAGNETOMETER, hl2ss.StreamPort.PERSONAL_VIDEO, hl2ss.StreamPort.MICROPHONE, - #hl2ss.StreamPort.SPATIAL_INPUT + #hl2ss.StreamPort.SPATIAL_INPUT, + #hl2ss.StreamPort.EXTENDED_EYE_TRACKER, ] # RM VLC parameters vlc_mode = hl2ss.StreamMode.MODE_1 -vlc_profile = hl2ss.VideoProfile.H264_BASE -vlc_bitrate = 1*1024*1024 +vlc_profile = hl2ss.VideoProfile.H264_MAIN +vlc_bitrate = 2*1024*1024 # RM Depth AHAT parameters -ahat_mode = hl2ss.StreamMode.MODE_1 -ahat_profile = hl2ss.VideoProfile.H264_BASE +ahat_mode = hl2ss.StreamMode.MODE_1 +ahat_profile = hl2ss.VideoProfile.H264_MAIN ahat_bitrate = 8*1024*1024 # RM Depth Long Throw parameters -lt_mode = hl2ss.StreamMode.MODE_1 +lt_mode = hl2ss.StreamMode.MODE_1 lt_filter = hl2ss.PngFilterMode.Paeth # RM IMU parameters imu_mode = hl2ss.StreamMode.MODE_1 # PV parameters -pv_mode = hl2ss.StreamMode.MODE_1 -pv_width = 760 -pv_height = 428 +pv_mode = hl2ss.StreamMode.MODE_1 +pv_width = 760 +pv_height = 428 pv_framerate = 30 -pv_profile = hl2ss.VideoProfile.H264_BASE -pv_bitrate = 1*1024*1024 -pv_format = 'bgr24' +pv_profile = hl2ss.VideoProfile.H264_MAIN +pv_bitrate = 1*1024*1024 +pv_format = 'bgr24' + +# Microphone parameters +microphone_profile = hl2ss.AudioProfile.AAC_24000 + +# EET parameters +eet_fps = 90 # Maximum number of frames in buffer buffer_elements = 300 @@ -68,8 +74,9 @@ producer.configure_rm_imu(host, hl2ss.StreamPort.RM_IMU_GYROSCOPE, hl2ss.ChunkSize.RM_IMU_GYROSCOPE, imu_mode) producer.configure_rm_imu(host, hl2ss.StreamPort.RM_IMU_MAGNETOMETER, hl2ss.ChunkSize.RM_IMU_MAGNETOMETER, imu_mode) producer.configure_pv(False, host, hl2ss.StreamPort.PERSONAL_VIDEO, hl2ss.ChunkSize.PERSONAL_VIDEO, pv_mode, pv_width, pv_height, pv_framerate, pv_profile, pv_bitrate, pv_format) - producer.configure_microphone(False, host, hl2ss.StreamPort.MICROPHONE, hl2ss.ChunkSize.MICROPHONE, hl2ss.AudioProfile.AAC_24000) + producer.configure_microphone(False, host, hl2ss.StreamPort.MICROPHONE, hl2ss.ChunkSize.MICROPHONE, microphone_profile) producer.configure_si(host, hl2ss.StreamPort.SPATIAL_INPUT, hl2ss.ChunkSize.SPATIAL_INPUT) + producer.configure_eet(host, hl2ss.StreamPort.EXTENDED_EYE_TRACKER, hl2ss.ChunkSize.EXTENDED_EYE_TRACKER, eet_fps) for port in ports: producer.initialize(port, buffer_elements) diff --git a/viewer/unity_demo_cube.py b/viewer/unity_demo_cube.py index 48512993..498a2701 100644 --- a/viewer/unity_demo_cube.py +++ b/viewer/unity_demo_cube.py @@ -6,7 +6,7 @@ from pynput import keyboard import hl2ss -import rus +import hl2ss_rus # Settings -------------------------------------------------------------------- @@ -45,15 +45,15 @@ def on_press(key): key = 0 -display_list = rus.command_buffer() +display_list = hl2ss_rus.command_buffer() display_list.begin_display_list() # Begin command sequence display_list.remove_all() # Remove all objects that were created remotely -display_list.create_primitive(rus.PrimitiveType.Cube) # Create a cube, server will return its id -display_list.set_target_mode(rus.TargetMode.UseLast) # Set server to use the last created object as target, this avoids waiting for the id of the cube +display_list.create_primitive(hl2ss_rus.PrimitiveType.Cube) # Create a cube, server will return its id +display_list.set_target_mode(hl2ss_rus.TargetMode.UseLast) # Set server to use the last created object as target, this avoids waiting for the id of the cube display_list.set_world_transform(key, position, rotation, scale) # Set the world transform of the cube display_list.set_color(key, rgba) # Set the color of the cube -display_list.set_active(key, rus.ActiveState.Active) # Make the cube visible -display_list.set_target_mode(rus.TargetMode.UseID) # Restore target mode +display_list.set_active(key, hl2ss_rus.ActiveState.Active) # Make the cube visible +display_list.set_target_mode(hl2ss_rus.TargetMode.UseID) # Restore target mode display_list.end_display_list() # End command sequence ipc.push(display_list) # Send commands to server results = ipc.pull(display_list) # Get results from server @@ -76,7 +76,7 @@ def on_press(key): position[2] = z - display_list = rus.command_buffer() + display_list = hl2ss_rus.command_buffer() display_list.begin_display_list() display_list.set_world_transform(key, position, rotation, scale) display_list.set_color(key, [z, 0, 1-z, 1-z]) # Semi-transparency is supported @@ -84,7 +84,7 @@ def on_press(key): ipc.push(display_list) results = ipc.pull(display_list) -command_buffer = rus.command_buffer() +command_buffer = hl2ss_rus.command_buffer() command_buffer.remove(key) # Destroy cube ipc.push(command_buffer) results = ipc.pull(command_buffer) diff --git a/viewer/unity_demo_hud.py b/viewer/unity_demo_hud.py index 3e501205..06e183a0 100644 --- a/viewer/unity_demo_hud.py +++ b/viewer/unity_demo_hud.py @@ -7,7 +7,7 @@ import threading import hl2ss -import rus +import hl2ss_rus # Settings -------------------------------------------------------------------- @@ -57,15 +57,15 @@ def on_press(key): key = 0 -display_list = rus.command_buffer() +display_list = hl2ss_rus.command_buffer() display_list.begin_display_list() # Begin command sequence display_list.remove_all() # Remove all objects that were created remotely -display_list.create_primitive(rus.PrimitiveType.Quad) # Create a quad, server will return its id -display_list.set_target_mode(rus.TargetMode.UseLast) # Set server to use the last created object as target, this avoids waiting for the id of the quad +display_list.create_primitive(hl2ss_rus.PrimitiveType.Quad) # Create a quad, server will return its id +display_list.set_target_mode(hl2ss_rus.TargetMode.UseLast) # Set server to use the last created object as target, this avoids waiting for the id of the quad display_list.set_local_transform(key, position, rotation, scale) # Set the local transform of the cube display_list.set_texture(key, texture) # Set the texture of the quad -display_list.set_active(key, rus.ActiveState.Active) # Make the quad visible -display_list.set_target_mode(rus.TargetMode.UseID) # Restore target mode +display_list.set_active(key, hl2ss_rus.ActiveState.Active) # Make the quad visible +display_list.set_target_mode(hl2ss_rus.TargetMode.UseID) # Restore target mode display_list.end_display_list() # End command sequence ipc.push(display_list) # Send commands to server results = ipc.pull(display_list) # Get results from server @@ -73,7 +73,7 @@ def on_press(key): stop_event.wait() -command_buffer = rus.command_buffer() +command_buffer = hl2ss_rus.command_buffer() command_buffer.remove(key) # Destroy quad ipc.push(command_buffer) results = ipc.pull(command_buffer) diff --git a/viewer/unity_demo_stickers.py b/viewer/unity_demo_stickers.py index 87bebaf8..ddd2cde6 100644 --- a/viewer/unity_demo_stickers.py +++ b/viewer/unity_demo_stickers.py @@ -13,8 +13,7 @@ import hl2ss_imshow import hl2ss import hl2ss_3dcv -import hl2ss_utilities -import rus +import hl2ss_rus import numpy as np # Settings -------------------------------------------------------------------- @@ -40,6 +39,9 @@ enable = True trigger = False +def clamp(v, min, max): + return min if (v < min) else max if (v > max) else v + def on_press(key): global enable global trigger @@ -67,7 +69,7 @@ def on_release(key): key = 0 -command_buffer = rus.command_buffer() +command_buffer = hl2ss_rus.command_buffer() command_buffer.remove_all() ipc.push(command_buffer) results = ipc.pull(command_buffer) @@ -137,7 +139,7 @@ def on_release(key): canonical_normal = np.array([0, 0, -1]).reshape((1, 3)) # Normal that looks at the camera when the camera transform is the identity axis = np.cross(canonical_normal, normal) axis = axis / np.linalg.norm(axis) - angle = np.arccos(hl2ss_utilities.clamp(np.dot(canonical_normal, normal), -1, 1)) + angle = np.arccos(clamp(np.dot(canonical_normal, normal), -1, 1)) # Convert axis-angle rotation to quaternion cos = np.cos(angle / 2) @@ -146,9 +148,9 @@ def on_release(key): rotation = [axis[0,0] * sin, axis[0,1] * sin, axis[0,2] * sin, cos] # Add quad to Unity scene - display_list = rus.command_buffer() + display_list = hl2ss_rus.command_buffer() display_list.begin_display_list() # Begin sequence - display_list.create_primitive(rus.PrimitiveType.Quad) # Create quad, returns id which can be used to modify its properties + display_list.create_primitive(hl2ss_rus.PrimitiveType.Quad) # Create quad, returns id which can be used to modify its properties display_list.set_target_mode(1) # Set server to use the last created object as target (this avoids waiting for the id) display_list.set_world_transform(key, centroid, rotation, scale) # Set the quad's world transform display_list.set_texture(key, image) # Set the quad's texture @@ -164,7 +166,7 @@ def on_release(key): print('Created quad with id {iid}'.format(iid=key)) -command_buffer = rus.command_buffer() +command_buffer = hl2ss_rus.command_buffer() command_buffer.remove_all() ipc.push(command_buffer) diff --git a/viewer/unity_demo_text.py b/viewer/unity_demo_text.py index 18181492..023b1d21 100644 --- a/viewer/unity_demo_text.py +++ b/viewer/unity_demo_text.py @@ -7,7 +7,7 @@ import threading import hl2ss -import rus +import hl2ss_rus # Settings -------------------------------------------------------------------- @@ -50,15 +50,15 @@ def on_press(key): key = 0 -display_list = rus.command_buffer() +display_list = hl2ss_rus.command_buffer() display_list.begin_display_list() # Begin command sequence display_list.remove_all() # Remove all objects that were created remotely display_list.create_text() # Create text object, server will return its id -display_list.set_target_mode(rus.TargetMode.UseLast) # Set server to use the last created object as target, this avoids waiting for the id of the text object +display_list.set_target_mode(hl2ss_rus.TargetMode.UseLast) # Set server to use the last created object as target, this avoids waiting for the id of the text object display_list.set_text(key, font_size, rgba, text) # Set text display_list.set_world_transform(key, position, rotation, [1, 1, 1]) # Set the world transform of the text object -display_list.set_active(key, rus.ActiveState.Active) # Make the text object visible -display_list.set_target_mode(rus.TargetMode.UseID) # Restore target mode +display_list.set_active(key, hl2ss_rus.ActiveState.Active) # Make the text object visible +display_list.set_target_mode(hl2ss_rus.TargetMode.UseID) # Restore target mode display_list.end_display_list() # End command sequence ipc.push(display_list) # Send commands to server results = ipc.pull(display_list) # Get results from server @@ -68,7 +68,7 @@ def on_press(key): stop_event.wait() -command_buffer = rus.command_buffer() +command_buffer = hl2ss_rus.command_buffer() command_buffer.remove(key) # Destroy text object ipc.push(command_buffer) results = ipc.pull(command_buffer) diff --git a/viewer/unity_test_twin_mesh.py b/viewer/unity_test_twin_mesh.py deleted file mode 100644 index 44a5983e..00000000 --- a/viewer/unity_test_twin_mesh.py +++ /dev/null @@ -1,141 +0,0 @@ - -import multiprocessing as mp -import numpy as np -import cv2 -import hl2ss -import hl2ss_mp -import hl2ss_3dcv -import open3d as o3d -import rus - -host = '192.168.1.7' -calibration_path = '../calibration' - -width = 640 -height = 360 -framerate = 30 -profile = hl2ss.VideoProfile.H265_MAIN -bitrate = hl2ss.get_video_codec_bitrate(width, height, framerate, 1/142) -buffer_length = 10 - -# Integration parameters -voxel_length = 1/32 -sdf_trunc = 0.04 -max_depth = 4.0 - -if __name__ == '__main__': - hl2ss.start_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO) - - calibration_lt = hl2ss_3dcv.get_calibration_rm(host, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, calibration_path) - - uv2xy = hl2ss_3dcv.compute_uv2xy(calibration_lt.intrinsics, hl2ss.Parameters_RM_DEPTH_LONGTHROW.WIDTH, hl2ss.Parameters_RM_DEPTH_LONGTHROW.HEIGHT) - xy1, scale = hl2ss_3dcv.rm_depth_compute_rays(uv2xy, calibration_lt.scale) - - volume = o3d.pipelines.integration.ScalableTSDFVolume(voxel_length=voxel_length, sdf_trunc=sdf_trunc, color_type=o3d.pipelines.integration.TSDFVolumeColorType.RGB8) - intrinsics_depth = o3d.camera.PinholeCameraIntrinsic(hl2ss.Parameters_RM_DEPTH_LONGTHROW.WIDTH, hl2ss.Parameters_RM_DEPTH_LONGTHROW.HEIGHT, calibration_lt.intrinsics[0, 0], calibration_lt.intrinsics[1, 1], calibration_lt.intrinsics[2, 0], calibration_lt.intrinsics[2, 1]) - - producer = hl2ss_mp.producer() - producer.configure_pv(True, host, hl2ss.StreamPort.PERSONAL_VIDEO, hl2ss.ChunkSize.PERSONAL_VIDEO, hl2ss.StreamMode.MODE_1, width, height, framerate, profile, bitrate, 'rgb24') - producer.configure_rm_depth_longthrow(True, host, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, hl2ss.ChunkSize.RM_DEPTH_LONGTHROW, hl2ss.StreamMode.MODE_1, hl2ss.PngFilterMode.Paeth) - producer.initialize(hl2ss.StreamPort.PERSONAL_VIDEO, framerate * buffer_length) - producer.initialize(hl2ss.StreamPort.RM_DEPTH_LONGTHROW, hl2ss.Parameters_RM_DEPTH_LONGTHROW.FPS * buffer_length) - producer.start(hl2ss.StreamPort.PERSONAL_VIDEO) - producer.start(hl2ss.StreamPort.RM_DEPTH_LONGTHROW) - - manager = mp.Manager() - consumer = hl2ss_mp.consumer() - sink_pv = consumer.create_sink(producer, hl2ss.StreamPort.PERSONAL_VIDEO, manager, None) - sink_depth = consumer.create_sink(producer, hl2ss.StreamPort.RM_DEPTH_LONGTHROW, manager, ...) - sink_pv.get_attach_response() - sink_depth.get_attach_response() - - pv_intrinsics = hl2ss.create_pv_intrinsics_placeholder() - - while (True): - sink_depth.acquire() - - _, data_lt = sink_depth.get_most_recent_frame() - if ((data_lt is None) or (not hl2ss.is_valid_pose(data_lt.pose))): - continue - - _, data_pv = sink_pv.get_nearest(data_lt.timestamp) - if ((data_pv is None) or (not hl2ss.is_valid_pose(data_pv.pose))): - continue - - break - - sink_pv.detach() - sink_depth.detach() - producer.stop(hl2ss.StreamPort.PERSONAL_VIDEO) - producer.stop(hl2ss.StreamPort.RM_DEPTH_LONGTHROW) - - depth = hl2ss_3dcv.rm_depth_undistort(data_lt.payload.depth, calibration_lt.undistort_map) - depth = hl2ss_3dcv.rm_depth_normalize(depth, scale) - color = data_pv.payload.image - pv_intrinsics = hl2ss.update_pv_intrinsics(pv_intrinsics, data_pv.payload.focal_length, data_pv.payload.principal_point) - - lt_points = hl2ss_3dcv.rm_depth_to_points(xy1, depth) - lt_to_world = hl2ss_3dcv.camera_to_rignode(calibration_lt.extrinsics) @ hl2ss_3dcv.reference_to_world(data_lt.pose) - world_to_lt = hl2ss_3dcv.world_to_reference(data_lt.pose) @ hl2ss_3dcv.rignode_to_camera(calibration_lt.extrinsics) - world_to_pv_image = hl2ss_3dcv.world_to_reference(data_pv.pose) @ hl2ss_3dcv.camera_to_image(pv_intrinsics) - world_points = hl2ss_3dcv.transform(lt_points, lt_to_world) - pv_uv = hl2ss_3dcv.project(world_points, world_to_pv_image) - color = cv2.remap(color, pv_uv[:, :, 0], pv_uv[:, :, 1], cv2.INTER_LINEAR) - - mask_uv = hl2ss_3dcv.slice_to_block((pv_uv[:, :, 0] < 0) | (pv_uv[:, :, 0] >= width) | (pv_uv[:, :, 1] < 0) | (pv_uv[:, :, 1] >= height)) - depth[mask_uv] = 0 - - color_image = o3d.geometry.Image(color) - depth_image = o3d.geometry.Image(depth) - rgbd = o3d.geometry.RGBDImage.create_from_color_and_depth(color_image, depth_image, depth_scale=1, depth_trunc=max_depth, convert_rgb_to_intensity=False) - - depth_world_to_camera = hl2ss_3dcv.world_to_reference(data_lt.pose) @ hl2ss_3dcv.rignode_to_camera(calibration_lt.extrinsics) - - volume.integrate(rgbd, intrinsics_depth, depth_world_to_camera.transpose()) - - mesh = volume.extract_triangle_mesh() - vertices = np.asarray(mesh.vertices) - tris = np.asarray(mesh.triangles) - l_vertices = vertices @ np.array([[1, 0, 0], [0, 1, 0], [0, 0, -1]], dtype=np.float32) - for i in range(0, tris.shape[0]): - a = tris[i, 0] - b = tris[i, 1] - c = tris[i, 2] - tris[i, 0] = c - tris[i, 1] = b - tris[i, 2] = a - mesh.vertices = o3d.utility.Vector3dVector(l_vertices) - mesh.triangles = o3d.utility.Vector3iVector(tris) - - print(f'Mesh triangles {tris.shape}') - - o3d.io.write_triangle_mesh('./data/mesh.obj', mesh) - - with open('./data/mesh.obj', mode='a') as file: - file.write('# end of file') - - with open('./data/mesh.obj', mode='rb') as file: - mesh_bytes = file.read() - - ipc_umq = hl2ss.ipc_umq(host, hl2ss.IPCPort.UNITY_MESSAGE_QUEUE) - ipc_umq.open() - - display_list = rus.command_buffer() - display_list.remove_all() - display_list.load_mesh(mesh_bytes) - - ipc_umq.push(display_list) - results = ipc_umq.pull(display_list) - key = results[1] - - display_list = rus.command_buffer() - display_list.set_world_transform(key, [0, 0, 0], [0, 0, 0, 1], [1, 1, 1]) - ipc_umq.push(display_list) - ipc_umq.pull(display_list) - - ipc_umq.close() - - print(f'Created object with id {key}') - - o3d.visualization.draw_geometries([mesh]) -