Hand positions saved when recording a ghost on HoloLens

Assets/MirageXR/Common/Scripts/CombinedEditor/ActionDetailView/Annotation Editors/GhostRecorder.cs

@@ -1,19 +1,22 @@
+using Microsoft.MixedReality.Toolkit;
+using Microsoft.MixedReality.Toolkit.Input;
+using Microsoft.MixedReality.Toolkit.Utilities;
+using MirageXR;
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Xml.Serialization;
-using Microsoft.MixedReality.Toolkit.Utilities;
-using MirageXR;
 using UnityEngine;
-using Microsoft.MixedReality.Toolkit.Input;
 
 public class GhostRecorder
 {
     public bool IsRecording => _isRecording;
     public GhostDataFrame LastFrame => _lastFrame;
 
+    private IMixedRealityHandJointService handJointService;
+
     private readonly List<GhostDataFrame> _ghostFrames = new List<GhostDataFrame>();
 
     private GhostDataFrame _lastFrame;
@@ -90,6 +93,8 @@ public void Start(Transform anchor, Transform camera, float? cooldown = null)
         _ghostFrames.Clear();
         _isRecording = true;
 
+        handJointService = CoreServices.GetInputSystemDataProvider<IMixedRealityHandJointService>();
+
         StartAsync();
     }
 
@@ -137,18 +142,66 @@ private void RecordFrame()
             lowerSpine = CreateLocalPose(_anchor, GetLowerSpinPosition(_cameraTransform), cameraRotation),
         };
 
+        var zeroPose = new Pose{position = Vector3.zero,rotation = Quaternion.identity};
+
         if (InputRayUtils.TryGetHandRay(Handedness.Right, out var rightHandRay))
         {
             var rotation = Quaternion.LookRotation(rightHandRay.direction, Vector3.up);
             _lastFrame.rightHand = CreateLocalPose(_anchor, rightHandRay.origin, rotation);
+
+            if (handJointService != null)
+            {
+                _lastFrame.rightThumbTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.ThumbTip, Handedness.Right));
+                _lastFrame.rightIndexTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.IndexTip, Handedness.Right));
+                _lastFrame.rightMiddleTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.MiddleTip, Handedness.Right));
+                _lastFrame.rightRingTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.RingTip, Handedness.Right));
+                _lastFrame.rightPinkyTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.PinkyTip, Handedness.Right));
+
+
+                Debug.Log("right index " + _lastFrame.rightIndexTip.position);
+                Debug.Log("right thumb " + _lastFrame.rightThumbTip.position);
+                Debug.Log("right Pinky " + _lastFrame.rightPinkyTip.position);
+
+            }
+            else
+            {
+                _lastFrame.rightThumbTip = zeroPose;
+                _lastFrame.rightIndexTip = zeroPose;
+                _lastFrame.rightMiddleTip = zeroPose;
+                _lastFrame.rightRingTip = zeroPose;
+                _lastFrame.rightPinkyTip = zeroPose;
+            }
         }
 
         if (InputRayUtils.TryGetHandRay(Handedness.Left, out var leftHandRay))
         {
             var rotation = Quaternion.LookRotation(leftHandRay.direction, Vector3.up);
             _lastFrame.leftHand = CreateLocalPose(_anchor, leftHandRay.origin, rotation);
+
+            if (handJointService != null)
+            {
+                _lastFrame.leftThumbTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.ThumbTip, Handedness.Left));
+                _lastFrame.leftIndexTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.IndexTip, Handedness.Left));
+                _lastFrame.leftMiddleTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.MiddleTip, Handedness.Left));
+                _lastFrame.leftRingTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.RingTip, Handedness.Left));
+                _lastFrame.leftPinkyTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.PinkyTip, Handedness.Left));
+
+                Debug.Log("Left index " + _lastFrame.leftIndexTip.position);
+                Debug.Log("Left thumb " + _lastFrame.leftThumbTip.position);
+                Debug.Log("Left Pinky " + _lastFrame.leftPinkyTip.position);
+            }
+            else
+            {
+                _lastFrame.leftThumbTip = zeroPose;
+                _lastFrame.leftIndexTip = zeroPose;
+                _lastFrame.leftMiddleTip = zeroPose;
+                _lastFrame.leftRingTip = zeroPose;
+                _lastFrame.leftPinkyTip = zeroPose;
+            }
         }
 
+
+
         _ghostFrames.Add(_lastFrame);
     }
 
@@ -161,6 +214,15 @@ private static Pose CreateLocalPose(Transform anchor, Vector3 position, Quaterni
         };
     }
 
+    private static Pose CreateLocalPose(Transform anchor, Transform transform)
+    {
+        return new Pose
+        {
+            position = anchor.InverseTransformPoint(transform.position),
+            rotation = Quaternion.Inverse(anchor.parent.localRotation) * transform.rotation,
+        };
+    }    
+
     private static Vector3 GetHeadPosition(Transform camera)
     {
         return camera.position;
@@ -185,4 +247,5 @@ private static Vector3 GetLowerSpinPosition(Transform camera)
     {
         return camera.position + (camera.up * -0.45f);
     }
+
 }

Assets/MirageXR/Common/Scripts/DataModel/GhostDataFrame.cs

@@ -11,5 +11,18 @@ public struct GhostDataFrame
         public Pose rightHand;
         public Pose upperSpine;
         public Pose lowerSpine;
+
+        public Pose leftThumbTip;
+        public Pose leftIndexTip;
+        public Pose leftMiddleTip;
+        public Pose leftRingTip;
+        public Pose leftPinkyTip;
+
+        public Pose rightThumbTip;
+        public Pose rightIndexTip;
+        public Pose rightMiddleTip;
+        public Pose rightRingTip;
+        public Pose rightPinkyTip;
+
     }
 }