Skip to content

Commit 8fa17da

Browse files
keveleighkeveleigh
authored
Add support for XR_MSFT_hand_tracking_mesh and XR_ANDROID_hand_mesh (#993)
* Initial drop * Iterate * Add UNITY_OPENXR_PRESENT * Update PlatformHandMeshVisualizer.cs * Update PlatformHandMeshVisualizer.cs * Docs update * Move prefabs into own folder Move back * Update PlatformHandMeshVisualizer.cs * Remove temp hand mesh prefabs * Add hand mesh manager * Iterate * Iterate * Update PlatformHandMeshVisualizer.cs * Add AXR hand mesh support stash Update OpenXRHandsSubsystem.cs * Update PlatformHandMeshVisualizer.cs * Update PlatformHandMeshVisualizer.cs Update PlatformHandMeshVisualizer.cs * Update PlatformHandMeshVisualizer.cs * Ensure the bounds are recalculated Otherwise, they'll stay vaguely at the origin on some platforms, which causes Unity to clip them as you look away from the origin Update PlatformHandMeshVisualizer.cs * Ensure we transform from the playspace pose * Update CHANGELOG.md * Update PlatformHandMeshVisualizer.cs * Update PlatformHandMeshVisualizer.cs * Update shader to use wrist position for fade, if given * Iterate back to sphere * Iterate shaders again Update RiggedHand-Main.mat * Iterate shaders again * Update to include outline * Update material defaults * Iterate to using Unity's API instead of Google's Update PlatformHandMeshVisualizer.cs * Update AndroidXRConfig.cs * Update HandMeshVisualizer.cs * Don't update more than once per frame * Update PlatformHandMeshVisualizer.cs * Revert "Iterate to using Unity's API instead of Google's" This reverts commit 71c1f40. This reverts commit d53c546. This reverts commit 62d1781. This reverts commit c196e26. * Update PlatformHandMeshVisualizer.cs * Use input action references for tracked state * Only query the fallback if needed and we're focused Some tracking state actions will be disabled based on the focus state, and we want to respect that * Reapply "Iterate to using Unity's API instead of Google's" This reverts commit d5e8004. * Update Hands version * Some optimizations and improvements * Another iteration * Revert "Reapply "Iterate to using Unity's API instead of Google's"" * Mark the hand mesh as dynamic * Lock the Android XR package to 1.0.0 for now * Iterate visualizers * Revert "Lock the Android XR package to 1.0.0 for now" This reverts commit 45c742f. * Update PlatformHandMeshVisualizer.cs * Update HandMeshVisualizer.cs * Reapply "Reapply "Iterate to using Unity's API instead of Google's"" This reverts commit 43c86af. * Update PlatformHandMeshVisualizer.cs * Update PlatformHandMeshVisualizer.cs * Fix stray mesh when untracked * Don't overwrite the actual tracking state This introduced a bug in the XRI3 migration, because the old base class (ActionBasedController) read this value from the action every frame. The new base class (TrackedPoseDriver) only reads the events when the action state changes.
1 parent 6bd4df9 commit 8fa17da

22 files changed

+893
-287
lines changed

‎UnityProjects/MRTKDevTemplate/Assets/Scripts/Editor/AndroidXRConfig.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,16 @@ public static void InstallPackages()
2424
return;
2525
}
2626

27-
Debug.Log("Adding com.unity.xr.androidxr-openxr and com.google.xr.extensions...");
28-
request = Client.AddAndRemove(new[] { "com.unity.xr.androidxr-openxr", "https://github.com/android/android-xr-unity-package.git" });
27+
Debug.Log("Adding the Unity OpenXR Android XR package...");
28+
request = Client.AddAndRemove(new[] { "com.unity.xr.androidxr-openxr" });
2929
EditorApplication.update += Progress;
3030
}
3131

3232
private static void Progress()
3333
{
3434
if (request.IsCompleted)
3535
{
36-
Debug.Log($"Package install request complete ({request.Status})");
36+
Debug.Log($"Package install request complete ({request.Status}).");
3737
EditorApplication.update -= Progress;
3838
request = null;
3939
}

‎UnityProjects/MRTKDevTemplate/Packages/packages-lock.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -264,12 +264,13 @@
264264
"url": "https://packages.unity.com"
265265
},
266266
"com.unity.xr.hands": {
267-
"version": "1.3.0",
267+
"version": "1.6.0",
268268
"depth": 1,
269269
"source": "registry",
270270
"dependencies": {
271271
"com.unity.modules.xr": "1.0.0",
272272
"com.unity.inputsystem": "1.3.0",
273+
"com.unity.mathematics": "1.2.6",
273274
"com.unity.xr.core-utils": "2.2.0",
274275
"com.unity.xr.management": "4.0.1"
275276
},
@@ -389,7 +390,7 @@
389390
"com.unity.inputsystem": "1.6.1",
390391
"com.unity.xr.arfoundation": "5.0.5",
391392
"com.unity.xr.core-utils": "2.1.0",
392-
"com.unity.xr.hands": "1.3.0",
393+
"com.unity.xr.hands": "1.6.0",
393394
"com.unity.xr.interaction.toolkit": "3.0.4",
394395
"org.mixedrealitytoolkit.core": "4.0.0"
395396
}

‎org.mixedrealitytoolkit.input/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
1212

1313
* Added `MRTKFocusFeature` to provide XrSession focus info to MRTK components. [PR #1057](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/1057)
1414
* Added input action focus handling to disable controller/hand tracked state when the XrSession goes out of focus. [PR #1057](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/1057)
15+
* Added support for XR_MSFT_hand_tracking_mesh and XR_ANDROID_hand_mesh on compatible runtimes. [PR #993](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/993)
1516

1617
## [4.0.0-pre.2] - 2025-12-05
1718

‎org.mixedrealitytoolkit.input/Controllers/HandModel.cs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public Transform ModelPrefab
5656
/// the hand model prefab when implementing <see cref="ISelectInputVisualizer"/>.
5757
/// </summary>
5858
public XRInputButtonReader SelectInput => selectInput;
59-
59+
6060
#endregion Associated hand select values
6161

6262
/// <summary>
@@ -88,9 +88,13 @@ protected virtual void Start()
8888
Debug.Assert(selectInput != null, $"The Select Input reader for {name} is not set and will not be used with the instantiated hand model.");
8989

9090
// Set the select input reader for the model if it implements ISelectInputVisualizer
91-
if (selectInput != null && model != null && model.TryGetComponent(out ISelectInputVisualizer selectInputVisualizer))
91+
if (selectInput != null && model != null)
9292
{
93-
selectInputVisualizer.SelectInput = selectInput;
93+
ISelectInputVisualizer[] selectInputVisualizers = model.GetComponentsInChildren<ISelectInputVisualizer>();
94+
foreach (ISelectInputVisualizer selectInputVisualizer in selectInputVisualizers)
95+
{
96+
selectInputVisualizer.SelectInput = selectInput;
97+
}
9498
}
9599
}
96100

‎org.mixedrealitytoolkit.input/Subsystems/Hands/UnityHandsSubsystem.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -263,8 +263,7 @@ private XRHand GetTrackedHand()
263263
}
264264
}
265265

266-
XRHand hand = HandNode == XRNode.LeftHand ? xrHandSubsystem.leftHand : xrHandSubsystem.rightHand;
267-
return hand;
266+
return HandNode == XRNode.LeftHand ? xrHandSubsystem.leftHand : xrHandSubsystem.rightHand;
268267
}
269268
}
270269

‎org.mixedrealitytoolkit.input/Visualizers/HandMeshVisualizer.cs‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
// Copyright (c) Mixed Reality Toolkit Contributors
2+
// Licensed under the BSD 3-Clause
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using UnityEngine;
7+
using UnityEngine.XR;
8+
using UnityEngine.XR.Interaction.Toolkit;
9+
using UnityEngine.XR.Interaction.Toolkit.Inputs.Readers;
10+
11+
namespace MixedReality.Toolkit.Input
12+
{
13+
public abstract class HandMeshVisualizer : MonoBehaviour, ISelectInputVisualizer
14+
{
15+
[SerializeField]
16+
[Tooltip("The XRNode on which this hand is located.")]
17+
private XRNode handNode = XRNode.LeftHand;
18+
19+
/// <summary> The XRNode on which this hand is located. </summary>
20+
public XRNode HandNode { get => handNode; set => handNode = value; }
21+
22+
[SerializeField]
23+
[Tooltip("When true, this visualizer will render rigged hands even on XR devices " +
24+
"with transparent displays or with passthrough enabled. When false, the rigged hands will only render " +
25+
"on devices with opaque displays. This behavior uses XRDisplaySubsystem.displayOpaque.")]
26+
private bool showHandsOnTransparentDisplays;
27+
28+
/// <summary>
29+
/// When true, this visualizer will render rigged hands even on XR devices with transparent displays or with passthrough enabled.
30+
/// When false, the rigged hands will only render on devices with opaque displays.
31+
/// </summary>
32+
/// <remarks>
33+
/// This behavior uses <see cref="XRDisplaySubsystem.displayOpaque"/>.
34+
/// Usually, it's recommended not to show hand visualization on transparent displays as it can
35+
/// distract from the user's real hands, and cause a "double image" effect that can be disconcerting.
36+
/// </remarks>
37+
public bool ShowHandsOnTransparentDisplays
38+
{
39+
get => showHandsOnTransparentDisplays;
40+
set => showHandsOnTransparentDisplays = value;
41+
}
42+
43+
[SerializeField]
44+
[Tooltip("Name of the shader property used to drive pinch-amount-based visual effects. " +
45+
"Generally, maps to something like a glow or an outline color!")]
46+
private string pinchAmountMaterialProperty = "_PinchAmount";
47+
48+
[SerializeField]
49+
[Tooltip("The input reader used when pinch selecting an interactable.")]
50+
private XRInputButtonReader selectInput = new XRInputButtonReader("Select");
51+
52+
#region ISelectInputVisualizer implementation
53+
54+
/// <summary>
55+
/// Input reader used when pinch selecting an interactable.
56+
/// </summary>
57+
public XRInputButtonReader SelectInput
58+
{
59+
get => selectInput;
60+
set => SetInputProperty(ref selectInput, value);
61+
}
62+
63+
#endregion ISelectInputVisualizer implementation
64+
65+
// The property block used to modify the pinch amount property on the material
66+
private MaterialPropertyBlock propertyBlock = null;
67+
68+
// Scratch list for checking for the presence of display subsystems.
69+
private readonly List<XRDisplaySubsystem> displaySubsystems = new List<XRDisplaySubsystem>();
70+
71+
// The XRController that is used to determine the pinch strength (i.e., select value!)
72+
[Obsolete("This field has been deprecated in version 4.0.0 and will be removed in a future version. Use the SelectInput property instead.")]
73+
private XRBaseController controller;
74+
75+
/// <summary>
76+
/// The list of button input readers used by this interactor. This interactor will automatically enable or disable direct actions
77+
/// if that mode is used during <see cref="OnEnable"/> and <see cref="OnDisable"/>.
78+
/// </summary>
79+
/// <seealso cref="XRInputButtonReader.EnableDirectActionIfModeUsed"/>
80+
/// <seealso cref="XRInputButtonReader.DisableDirectActionIfModeUsed"/>
81+
private readonly List<XRInputButtonReader> buttonReaders = new List<XRInputButtonReader>();
82+
83+
/// <summary>
84+
/// Whether this visualizer currently has a loaded and visible hand mesh or not.
85+
/// </summary>
86+
protected internal bool IsRendering => HandRenderer != null && HandRenderer.enabled;
87+
88+
/// <summary>
89+
/// The renderer for this visualizer, to use to visualize the pinch amount.
90+
/// </summary>
91+
protected abstract Renderer HandRenderer { get; }
92+
93+
/// <summary>
94+
/// A Unity event function that is called when an enabled script instance is being loaded.
95+
/// </summary>
96+
protected virtual void Awake()
97+
{
98+
propertyBlock = new MaterialPropertyBlock();
99+
buttonReaders.Add(selectInput);
100+
}
101+
102+
/// <summary>
103+
/// A Unity event function that is called when the script component has been enabled.
104+
/// </summary>
105+
protected virtual void OnEnable()
106+
{
107+
buttonReaders.ForEach(reader => reader?.EnableDirectActionIfModeUsed());
108+
109+
// Ensure hand is not visible until we can update position first time.
110+
HandRenderer.enabled = false;
111+
112+
Debug.Assert(handNode == XRNode.LeftHand || handNode == XRNode.RightHand,
113+
$"HandVisualizer has an invalid XRNode ({handNode})!");
114+
}
115+
116+
/// <summary>
117+
/// A Unity event function that is called when the script component has been disabled.
118+
/// </summary>
119+
protected virtual void OnDisable()
120+
{
121+
buttonReaders.ForEach(reader => reader?.DisableDirectActionIfModeUsed());
122+
123+
// Disable the rigged hand renderer when this component is disabled
124+
HandRenderer.enabled = false;
125+
}
126+
127+
/// <summary>
128+
/// Helper method for setting an input property.
129+
/// </summary>
130+
/// <param name="property">The <see langword="ref"/> to the field.</param>
131+
/// <param name="value">The new value being set.</param>
132+
/// <remarks>
133+
/// If the application is playing, this method will also enable or disable directly embedded input actions
134+
/// serialized by the input if that mode is used. It will also add or remove the input from the list of button inputs
135+
/// to automatically manage enabling and disabling direct actions with this behavior.
136+
/// </remarks>
137+
/// <seealso cref="buttonReaders"/>
138+
protected void SetInputProperty(ref XRInputButtonReader property, XRInputButtonReader value)
139+
{
140+
if (value == null)
141+
{
142+
Debug.LogError("Setting XRInputButtonReader property to null is disallowed and has therefore been ignored.");
143+
return;
144+
}
145+
146+
if (Application.isPlaying && property != null)
147+
{
148+
buttonReaders?.Remove(property);
149+
property.DisableDirectActionIfModeUsed();
150+
}
151+
152+
property = value;
153+
154+
if (Application.isPlaying)
155+
{
156+
buttonReaders?.Add(property);
157+
if (isActiveAndEnabled)
158+
{
159+
property.EnableDirectActionIfModeUsed();
160+
}
161+
}
162+
}
163+
164+
protected virtual bool ShouldRenderHand()
165+
{
166+
if (displaySubsystems.Count == 0)
167+
{
168+
SubsystemManager.GetSubsystems(displaySubsystems);
169+
}
170+
171+
// Are we running on an XR display and it happens to be transparent?
172+
// Probably shouldn't be showing rigged hands! (Users can
173+
// specify showHandsOnTransparentDisplays if they disagree.)
174+
if (displaySubsystems.Count > 0 &&
175+
displaySubsystems[0].running &&
176+
!displaySubsystems[0].displayOpaque &&
177+
!showHandsOnTransparentDisplays)
178+
{
179+
return false;
180+
}
181+
182+
// All checks out!
183+
return true;
184+
}
185+
186+
protected virtual void UpdateHandMaterial()
187+
{
188+
if (HandRenderer == null)
189+
{
190+
return;
191+
}
192+
193+
// Update the hand material
194+
float pinchAmount = TryGetSelectionValue(out float selectionValue) ? Mathf.Pow(selectionValue, 2.0f) : 0;
195+
HandRenderer.GetPropertyBlock(propertyBlock);
196+
propertyBlock.SetFloat(pinchAmountMaterialProperty, pinchAmount);
197+
HandRenderer.SetPropertyBlock(propertyBlock);
198+
}
199+
200+
/// <summary>
201+
/// Try to obtain the tracked devices selection value from the provided input reader.
202+
/// </summary>
203+
/// <remarks>
204+
/// For backwards compatibility, this method will also attempt to get the selection amount from a
205+
/// legacy XRI controller if the input reader is not set.
206+
/// </remarks>
207+
private bool TryGetSelectionValue(out float value)
208+
{
209+
if (selectInput != null && selectInput.TryReadValue(out value))
210+
{
211+
return true;
212+
}
213+
214+
bool success = false;
215+
value = 0.0f;
216+
217+
#pragma warning disable CS0618 // XRBaseController is obsolete
218+
if (controller == null)
219+
{
220+
controller = GetComponentInParent<XRBaseController>();
221+
}
222+
if (controller != null)
223+
{
224+
value = controller.selectInteractionState.value;
225+
success = true;
226+
}
227+
#pragma warning restore CS0618 // XRBaseController is obsolete
228+
229+
return success;
230+
}
231+
}
232+
}

‎org.mixedrealitytoolkit.input/Visualizers/HandMeshVisualizer.cs.meta‎

Lines changed: 11 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎org.mixedrealitytoolkit.input/Visualizers/PlatformHandVisualizer.meta‎

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)