From be7d0d30718a1316ddb31444a5d1e9ffcd0a1b1e Mon Sep 17 00:00:00 2001 From: robhillman97 Date: Tue, 15 Aug 2023 16:42:00 +0100 Subject: [PATCH 1/2] position of the tips of each fingers are now saved with the rest of ghost data --- .../Annotation Editors/GhostRecorder.cs | 69 ++++++++++++++++++- .../Scripts/DataModel/GhostDataFrame.cs | 13 ++++ 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/Assets/MirageXR/Common/Scripts/CombinedEditor/ActionDetailView/Annotation Editors/GhostRecorder.cs b/Assets/MirageXR/Common/Scripts/CombinedEditor/ActionDetailView/Annotation Editors/GhostRecorder.cs index e868374bb..dc509165e 100644 --- a/Assets/MirageXR/Common/Scripts/CombinedEditor/ActionDetailView/Annotation Editors/GhostRecorder.cs +++ b/Assets/MirageXR/Common/Scripts/CombinedEditor/ActionDetailView/Annotation Editors/GhostRecorder.cs @@ -1,19 +1,22 @@ +using Microsoft.MixedReality.Toolkit; +using Microsoft.MixedReality.Toolkit.Input; +using Microsoft.MixedReality.Toolkit.Utilities; +using MirageXR; using System; using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; using System.Xml.Serialization; -using Microsoft.MixedReality.Toolkit.Utilities; -using MirageXR; using UnityEngine; -using Microsoft.MixedReality.Toolkit.Input; public class GhostRecorder { public bool IsRecording => _isRecording; public GhostDataFrame LastFrame => _lastFrame; + private IMixedRealityHandJointService handJointService; + private readonly List _ghostFrames = new List(); private GhostDataFrame _lastFrame; @@ -90,6 +93,8 @@ public void Start(Transform anchor, Transform camera, float? cooldown = null) _ghostFrames.Clear(); _isRecording = true; + handJointService = CoreServices.GetInputSystemDataProvider(); + StartAsync(); } @@ -137,18 +142,66 @@ private void RecordFrame() lowerSpine = CreateLocalPose(_anchor, GetLowerSpinPosition(_cameraTransform), cameraRotation), }; + var zeroPose = new Pose{position = Vector3.zero,rotation = Quaternion.identity}; + if (InputRayUtils.TryGetHandRay(Handedness.Right, out var rightHandRay)) { var rotation = Quaternion.LookRotation(rightHandRay.direction, Vector3.up); _lastFrame.rightHand = CreateLocalPose(_anchor, rightHandRay.origin, rotation); + + if (handJointService != null) + { + _lastFrame.rightThumbTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.ThumbTip, Handedness.Right)); + _lastFrame.rightIndexTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.IndexTip, Handedness.Right)); + _lastFrame.rightMiddleTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.MiddleTip, Handedness.Right)); + _lastFrame.rightRingTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.RingTip, Handedness.Right)); + _lastFrame.rightPinkyTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.PinkyTip, Handedness.Right)); + + + Debug.Log("right index " + _lastFrame.rightIndexTip.position); + Debug.Log("right thumb " + _lastFrame.rightThumbTip.position); + Debug.Log("right Pinky " + _lastFrame.rightPinkyTip.position); + + } + else + { + _lastFrame.rightThumbTip = zeroPose; + _lastFrame.rightIndexTip = zeroPose; + _lastFrame.rightMiddleTip = zeroPose; + _lastFrame.rightRingTip = zeroPose; + _lastFrame.rightPinkyTip = zeroPose; + } } if (InputRayUtils.TryGetHandRay(Handedness.Left, out var leftHandRay)) { var rotation = Quaternion.LookRotation(leftHandRay.direction, Vector3.up); _lastFrame.leftHand = CreateLocalPose(_anchor, leftHandRay.origin, rotation); + + if (handJointService != null) + { + _lastFrame.leftThumbTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.ThumbTip, Handedness.Left)); + _lastFrame.leftIndexTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.IndexTip, Handedness.Left)); + _lastFrame.leftMiddleTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.MiddleTip, Handedness.Left)); + _lastFrame.leftRingTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.RingTip, Handedness.Left)); + _lastFrame.leftPinkyTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.PinkyTip, Handedness.Left)); + + Debug.Log("Left index " + _lastFrame.leftIndexTip.position); + Debug.Log("Left thumb " + _lastFrame.leftThumbTip.position); + Debug.Log("Left Pinky " + _lastFrame.leftPinkyTip.position); + } + else + { + _lastFrame.leftThumbTip = zeroPose; + _lastFrame.leftIndexTip = zeroPose; + _lastFrame.leftMiddleTip = zeroPose; + _lastFrame.leftRingTip = zeroPose; + _lastFrame.leftPinkyTip = zeroPose; + } } + + _ghostFrames.Add(_lastFrame); } @@ -161,6 +214,15 @@ private static Pose CreateLocalPose(Transform anchor, Vector3 position, Quaterni }; } + private static Pose CreateLocalPose(Transform anchor, Transform transform) + { + return new Pose + { + position = anchor.InverseTransformPoint(transform.position), + rotation = Quaternion.Inverse(anchor.parent.localRotation) * transform.rotation, + }; + } + private static Vector3 GetHeadPosition(Transform camera) { return camera.position; @@ -185,4 +247,5 @@ private static Vector3 GetLowerSpinPosition(Transform camera) { return camera.position + (camera.up * -0.45f); } + } \ No newline at end of file diff --git a/Assets/MirageXR/Common/Scripts/DataModel/GhostDataFrame.cs b/Assets/MirageXR/Common/Scripts/DataModel/GhostDataFrame.cs index 2fbd42e45..97f5ee677 100644 --- a/Assets/MirageXR/Common/Scripts/DataModel/GhostDataFrame.cs +++ b/Assets/MirageXR/Common/Scripts/DataModel/GhostDataFrame.cs @@ -11,5 +11,18 @@ public struct GhostDataFrame public Pose rightHand; public Pose upperSpine; public Pose lowerSpine; + + public Pose leftThumbTip; + public Pose leftIndexTip; + public Pose leftMiddleTip; + public Pose leftRingTip; + public Pose leftPinkyTip; + + public Pose rightThumbTip; + public Pose rightIndexTip; + public Pose rightMiddleTip; + public Pose rightRingTip; + public Pose rightPinkyTip; + } } \ No newline at end of file From 202bd79e43745905275671766e7bbe9ac3df3d65 Mon Sep 17 00:00:00 2001 From: robhillman97 Date: Tue, 17 Oct 2023 10:03:43 +0100 Subject: [PATCH 2/2] removed debugs as requested in #1632 --- .../ActionDetailView/Annotation Editors/GhostRecorder.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Assets/MirageXR/Common/Scripts/CombinedEditor/ActionDetailView/Annotation Editors/GhostRecorder.cs b/Assets/MirageXR/Common/Scripts/CombinedEditor/ActionDetailView/Annotation Editors/GhostRecorder.cs index dc509165e..19adc8710 100644 --- a/Assets/MirageXR/Common/Scripts/CombinedEditor/ActionDetailView/Annotation Editors/GhostRecorder.cs +++ b/Assets/MirageXR/Common/Scripts/CombinedEditor/ActionDetailView/Annotation Editors/GhostRecorder.cs @@ -156,12 +156,6 @@ private void RecordFrame() _lastFrame.rightMiddleTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.MiddleTip, Handedness.Right)); _lastFrame.rightRingTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.RingTip, Handedness.Right)); _lastFrame.rightPinkyTip = CreateLocalPose(_anchor, handJointService.RequestJointTransform(TrackedHandJoint.PinkyTip, Handedness.Right)); - - - Debug.Log("right index " + _lastFrame.rightIndexTip.position); - Debug.Log("right thumb " + _lastFrame.rightThumbTip.position); - Debug.Log("right Pinky " + _lastFrame.rightPinkyTip.position); - } else {