EET recorder, raw IO, cleanup

viewer/client_eet.py

@@ -1,6 +1,5 @@
 #------------------------------------------------------------------------------
 # This script receives extended eye tracking data from the HoloLens.
-# See https://learn.microsoft.com/en-us/windows/mixed-reality/develop/native/extended-eye-tracking-native
 #------------------------------------------------------------------------------
 
 import hl2ss
@@ -25,26 +24,21 @@
 data = client.get_next_packet()
 data.payload = hl2ss.unpack_eet(data.payload)
 
-print(data.timestamp)
-print(data.payload._reserved)
-
-print(data.payload.combined_ray.origin)
-print(data.payload.combined_ray.direction)
-print(data.payload.left_ray.origin)
-print(data.payload.left_ray.direction)
-print(data.payload.right_ray.origin)
-print(data.payload.right_ray.direction)
-print(data.payload.left_openness)
-print(data.payload.right_openness)
-print(data.payload.vergence_distance)
-
-print(data.payload.calibration_valid)
-print(data.payload.combined_ray_valid)
-print(data.payload.left_ray_valid)
-print(data.payload.right_ray_valid)
-print(data.payload.left_openness_valid)
-print(data.payload.right_openness_valid)
-print(data.payload.vergence_distance_valid)
+# See
+# https://learn.microsoft.com/en-us/windows/mixed-reality/develop/native/extended-eye-tracking-native
+# for details
+
+print(f'Tracking status at time {data.timestamp}')
+print('Pose')
 print(data.pose)
+print(f'Calibration valid: {data.payload.calibration_valid}')
+print(f'Combined eye gaze: Valid={data.payload.combined_ray_valid} Origin={data.payload.combined_ray.origin} Direction={data.payload.combined_ray.direction}')
+print(f'Left eye gaze: Valid={data.payload.left_ray_valid} Origin={data.payload.left_ray.origin} Direction={data.payload.left_ray.direction}')
+print(f'Right eye gaze: Valid={data.payload.right_ray_valid} Origin={data.payload.right_ray.origin} Direction={data.payload.right_ray.direction}')
+
+# "...not supported by HoloLens 2 at this time"
+print(f'Left eye openness: Valid={data.payload.left_openness_valid} Value={data.payload.left_openness}')
+print(f'Right eye openness: Valid={data.payload.right_openness_valid} Value={data.payload.right_openness}')
+print(f'Vergence distance: Valid={data.payload.vergence_distance_valid} Value={data.payload.vergence_distance}')
 
 client.close()

viewer/client_microphone.py

@@ -1,8 +1,8 @@
 #------------------------------------------------------------------------------
-# This script receives AAC encoded microphone audio from the HoloLens and
-# plays it. The main thread receives the data, decodes it, and puts the decoded
-# audio samples in a queue. A second thread gets the samples from the queue and
-# plays them. Audio stream configuration is fixed to 2 channels, 48000 Hz.
+# This script receives microphone audio from the HoloLens and plays it. The 
+# main thread receives the data, decodes it, and puts the decoded audio samples
+# in a queue. A second thread gets the samples from the queue and plays them.
+# Audio stream configuration is fixed to 2 channels, 48000 Hz.
 # Press esc to stop.
 #------------------------------------------------------------------------------
 
@@ -27,12 +27,15 @@
 
 #------------------------------------------------------------------------------
 
+# RAW format is s16 packed, AAC decoded format is f32 planar
+audio_format = pyaudio.paInt16 if (profile == hl2ss.AudioProfile.RAW) else pyaudio.paFloat32
 enable = True
 
 def pcmworker(pcmqueue):
     global enable
+    global audio_format
     p = pyaudio.PyAudio()
-    stream = p.open(format=pyaudio.paFloat32, channels=hl2ss.Parameters_MICROPHONE.CHANNELS, rate=hl2ss.Parameters_MICROPHONE.SAMPLE_RATE, output=True)
+    stream = p.open(format=audio_format, channels=hl2ss.Parameters_MICROPHONE.CHANNELS, rate=hl2ss.Parameters_MICROPHONE.SAMPLE_RATE, output=True)
     stream.start_stream()
     while (enable):
         stream.write(pcmqueue.get())
@@ -55,7 +58,8 @@ def on_press(key):
 
 while (enable): 
     data = client.get_next_packet()
-    audio = hl2ss_utilities.microphone_planar_to_packed(data.payload)
+    # RAW format is s16 packed, AAC decoded format is f32 planar
+    audio = hl2ss_utilities.microphone_planar_to_packed(data.payload) if (profile != hl2ss.AudioProfile.RAW) else data.payload
     pcmqueue.put(audio.tobytes())
 
 client.close()

viewer/client_pv.py

@@ -1,11 +1,11 @@
 #------------------------------------------------------------------------------
-# This script receives encoded video from the HoloLens front RGB camera and
-# plays it. The camera support various resolutions and framerates. See
-# etc/hl2_capture_formats.txt for a list of supported formats. The default
-# configuration is 1080p 30 FPS. The stream supports three operating modes:
-# 0) video, 1) video + camera pose, 2) query calibration (single transfer).
-# Press esc to stop. Note that the ahat stream cannot be used while the pv
-# subsystem is on.
+# This script receives video from the HoloLens front RGB camera and plays it.
+# The camera supports various resolutions and framerates. See
+# https://github.com/jdibenes/hl2ss/blob/main/etc/pv_configurations.txt
+# for a list of supported formats. The default configuration is 1080p 30 FPS. 
+# The stream supports three operating modes: 0) video, 1) video + camera pose, 
+# 2) query calibration (single transfer).
+# Press esc to stop.
 #------------------------------------------------------------------------------
 
 from pynput import keyboard
@@ -38,9 +38,15 @@
 
 # Encoded stream average bits per second
 # Must be > 0
-bitrate = 5*1024*1024
+bitrate = hl2ss.get_video_codec_bitrate(width, height, framerate, hl2ss.get_video_codec_default_factor(profile))
 
 # Decoded format
+# Options include:
+# 'bgr24'
+# 'rgb24'
+# 'bgra'
+# 'rgba'
+# 'gray8'
 decoded_format = 'bgr24'
 
 #------------------------------------------------------------------------------
@@ -50,11 +56,13 @@
 if (mode == hl2ss.StreamMode.MODE_2):
     data = hl2ss.download_calibration_pv(host, port, width, height, framerate)
     print('Calibration')
-    print(data.focal_length)
-    print(data.principal_point)
-    print(data.radial_distortion)
-    print(data.tangential_distortion)
+    print(f'Focal length: {data.focal_length}')
+    print(f'Principal point: {data.principal_point}')
+    print(f'Radial distortion: {data.radial_distortion}')
+    print(f'Tangential distortion: {data.tangential_distortion}')
+    print('Projection')
     print(data.projection)
+    print('Intrinsics')
     print(data.intrinsics)
 else:
     enable = True
@@ -72,12 +80,12 @@ def on_press(key):
 
     while (enable):
         data = client.get_next_packet()
+
         print('Pose at time {ts}'.format(ts=data.timestamp))
         print(data.pose)
-        print('Focal length')
-        print(data.payload.focal_length)
-        print('Principal point')
-        print(data.payload.principal_point)
+        print(f'Focal length: {data.payload.focal_length}')
+        print(f'Principal point: {data.payload.principal_point}')
+
         cv2.imshow('Video', data.payload.image)
         cv2.waitKey(1)
 

viewer/client_rc.py

@@ -64,17 +64,19 @@
 client.open()
 
 version = client.get_application_version()
-print('Installed version {v0}.{v1}.{v2}.{v3}'.format(v0=version[0], v1=version[1], v2=version[2], v3=version[3]))
+print(f'Installed version {version[0]}.{version[1]}.{version[2]}.{version[3]}')
 
-utc_offset = client.get_utc_offset(32) # Add this offset to timestamps to convert to utc
-print('QPC timestamp to UTC offset is {offset} hundreds of nanoseconds'.format(offset=utc_offset))
+# Add this offset to timestamps to convert to utc (Windows FILETIME)
+utc_offset = client.get_utc_offset(32)
+print(f'QPC timestamp to UTC offset is {utc_offset} hundreds of nanoseconds')
 
 client.set_hs_marker_state(marker_state)
 
 # PV camera configuration
 pv_status = client.get_pv_subsystem_status()
-print('PV subsystem is {status}'.format(status=('On' if pv_status else 'Off')))
+print(f'PV subsystem is {("On" if (pv_status) else "Off")}')
 
+# Ignored if PV subsystem is Off
 client.set_pv_focus(focus_mode, auto_focus_range, manual_focus_distance, focus_value, driver_fallback)
 client.set_pv_video_temporal_denoising(video_temporal_denoising)
 client.set_pv_white_balance_preset(white_balance_preset)

viewer/client_rm_depth_ahat.py

@@ -1,11 +1,10 @@
 #------------------------------------------------------------------------------
-# This script receives encoded video from the HoloLens depth camera in ahat
-# mode and plays it. The resolution is 512x512 @ 45 FPS. The stream supports 
-# three operating modes: 0) video, 1) video + rig pose, 2) query calibration
-# (single transfer). Press esc to stop. Depth and AB data are scaled for
-# visibility. Note that 1) the ahat stream cannot be used while the pv
-# subsystem is on, and 2) the ahat and long throw streams cannot be used
-# simultaneously.
+# This script receives video from the HoloLens depth camera in ahat mode and 
+# plays it. The resolution is 512x512 @ 45 FPS. The stream supports three 
+# operating modes: 0) video, 1) video + rig pose, 2) query calibration (single 
+# transfer). Depth and AB data are scaled for visibility. The ahat and long 
+# throw streams cannot be used simultaneously.
+# Press esc to stop.
 #------------------------------------------------------------------------------
 
 from pynput import keyboard
@@ -41,11 +40,15 @@
 if (mode == hl2ss.StreamMode.MODE_2):
     data = hl2ss.download_calibration_rm_depth_ahat(host, port)
     print('Calibration data')
-    print(data.uv2xy.shape)
+    print('Image point to unit plane')
+    print(data.uv2xy)
+    print('Extrinsics')
     print(data.extrinsics)
-    print(data.scale)
-    print(data.alias)
-    print(data.undistort_map.shape)
+    print(f'Scale: {data.scale}')
+    print(f'Alias: {data.alias}')
+    print('Undistort map')
+    print(data.undistort_map)
+    print('Intrinsics (undistorted only)')
     print(data.intrinsics)
     quit()
 
@@ -64,10 +67,10 @@ def on_press(key):
 
 while (enable):
     data = client.get_next_packet()
-    print('Pose at time {ts}'.format(ts=data.timestamp))
+    print(f'Pose at time {data.timestamp}')
     print(data.pose)
-    cv2.imshow('Depth', data.payload.depth / np.max(data.payload.depth)) # Normalized for visibility
-    cv2.imshow('AB', data.payload.ab / np.max(data.payload.ab)) # Normalized for visibility
+    cv2.imshow('Depth', data.payload.depth / np.max(data.payload.depth)) # Scaled for visibility
+    cv2.imshow('AB', data.payload.ab / np.max(data.payload.ab)) # Scaled for visibility
     cv2.waitKey(1)
 
 client.close()