CHANGE: Merging touch events (#1392)

Author: jimon

Assets/Tests/InputSystem/APIVerificationTests.cs

@@ -794,6 +794,10 @@ public class TrackedPoseDriver : UnityEngine.MonoBehaviour
     [Property("Exclusions", @"1.0.0
         public static event System.Action<UnityEngine.InputSystem.LowLevel.InputEventPtr, UnityEngine.InputSystem.InputDevice> onEvent;
     ")]
+    // Mouse and Touchscreen implement internal IEventMerger interface
+    [Property("Exclusions", @"1.0.0
+        public class Touchscreen : UnityEngine.InputSystem.Pointer, UnityEngine.InputSystem.LowLevel.IInputStateCallbackReceiver
+    ")]
     [ScopedExclusionProperty("1.0.0", "UnityEngine.InputSystem.Editor", "public sealed class InputControlPathEditor : System.IDisposable", "public void OnGUI(UnityEngine.Rect rect);")]
     public void API_MinorVersionsHaveNoBreakingChanges()
     {

Assets/Tests/InputSystem/CoreTests_Events.cs

@@ -2328,8 +2328,39 @@ public void Events_CanQueueEventsFromWithinEventProcessing_WithEventMergingSetTo
         Assert.That(mouse.position.ReadValue(), Is.EqualTo(new Vector2(numMouseEventsQueued, numMouseEventsQueued * 2)));
         Assert.That(mouse.delta.ReadValue(), Is.EqualTo(new Vector2(numMouseEventsQueued, numMouseEventsQueued)));
         Assert.That(numMouseEventsReceived, Is.EqualTo(mergeRedundantEvents ? 1 : numMouseEventsQueued));
+    }
+
+    [Test]
+    [Category("Events")]
+    public void Events_QueuedToDifferentDevices_AreNotMergedTogether()
+    {
+        var mouse1 = InputSystem.AddDevice<Mouse>();
+        var mouse2 = InputSystem.AddDevice<Mouse>();
+
+        var numMouse1EventsReceived = 0;
+        var numMouse2EventsReceived = 0;
+        InputSystem.onEvent +=
+            (eventPtr, device) =>
+        {
+            if (device == mouse1)
+                ++numMouse1EventsReceived;
+            if (device == mouse2)
+                ++numMouse2EventsReceived;
+        };
+
+        InputSystem.QueueStateEvent(mouse1, new MouseState { position = new Vector2(1, 2)});
+        InputSystem.QueueStateEvent(mouse1, new MouseState { position = new Vector2(2, 3)});
+        InputSystem.QueueStateEvent(mouse2, new MouseState { position = new Vector2(3, 4)});
+        InputSystem.QueueStateEvent(mouse2, new MouseState { position = new Vector2(4, 5)});
+        InputSystem.QueueStateEvent(mouse2, new MouseState { position = new Vector2(5, 6)});
+
+        InputSystem.Update();
+
+        Assert.That(mouse1.position.ReadValue(), Is.EqualTo(new Vector2(2, 3)));
+        Assert.That(mouse2.position.ReadValue(), Is.EqualTo(new Vector2(5, 6)));
 
-        InputSystem.settings.disableRedundantEventsMerging = false;
+        Assert.That(numMouse1EventsReceived, Is.EqualTo(1));
+        Assert.That(numMouse2EventsReceived, Is.EqualTo(1));
     }
 
     [Test]

Assets/Tests/InputSystem/Plugins/EnhancedTouchTests.cs

@@ -441,19 +441,40 @@ public void EnhancedTouch_DeltasInActiveTouchesAccumulateAndReset()
     // Thus we don't want accumulation and resetting (which again are frame-to-frame kind of mechanics).
     [Test]
     [Category("EnhancedTouch")]
-    public void EnhancedTouch_DeltasInTouchHistoryDoNotAccumulateAndReset()
+    [TestCase(false)]
+    [TestCase(true)]
+    public void EnhancedTouch_DeltasInTouchHistoryDoNotAccumulateAndReset_WithEventMergingSetTo(bool mergeRedundantEvents)
     {
+        InputSystem.settings.disableRedundantEventsMerging = !mergeRedundantEvents;
+
         BeginTouch(1, new Vector2(0.123f, 0.234f), queueEventOnly: true);
         MoveTouch(1, new Vector2(0.234f, 0.345f), queueEventOnly: true);
         MoveTouch(1, new Vector2(0.345f, 0.456f), queueEventOnly: true);
+        MoveTouch(1, new Vector2(0.456f, 0.567f), queueEventOnly: true);
 
         InputSystem.Update();
 
-        Assert.That(Touch.activeFingers[0].touchHistory[0].delta,
-            Is.EqualTo(new Vector2(0.111f, 0.111f)).Using(Vector2EqualityComparer.Instance));
-        Assert.That(Touch.activeFingers[0].touchHistory[1].delta,
-            Is.EqualTo(new Vector2(0.111f, 0.111f)).Using(Vector2EqualityComparer.Instance));
-        Assert.That(Touch.activeFingers[0].touchHistory[2].delta,
+        Assert.That(Touch.activeFingers[0].touchHistory.Count, Is.EqualTo(mergeRedundantEvents ? 3 : 4));
+
+        if (mergeRedundantEvents)
+        {
+            // Event merging adds deltas inside
+            Assert.That(Touch.activeFingers[0].touchHistory[0].delta,
+                Is.EqualTo(new Vector2(0.222f, 0.222f)).Using(Vector2EqualityComparer.Instance));
+            Assert.That(Touch.activeFingers[0].touchHistory[1].delta,
+                Is.EqualTo(new Vector2(0.111f, 0.111f)).Using(Vector2EqualityComparer.Instance));
+        }
+        else
+        {
+            Assert.That(Touch.activeFingers[0].touchHistory[0].delta,
+                Is.EqualTo(new Vector2(0.222f, 0.222f)).Using(Vector2EqualityComparer.Instance));
+            Assert.That(Touch.activeFingers[0].touchHistory[1].delta,
+                Is.EqualTo(new Vector2(0.111f, 0.111f)).Using(Vector2EqualityComparer.Instance));
+            Assert.That(Touch.activeFingers[0].touchHistory[2].delta,
+                Is.EqualTo(new Vector2(0.111f, 0.111f)).Using(Vector2EqualityComparer.Instance));
+        }
+
+        Assert.That(Touch.activeFingers[0].touchHistory.Last().delta,
             Is.EqualTo(new Vector2()).Using(Vector2EqualityComparer.Instance));
     }
 
@@ -501,8 +522,12 @@ public void EnhancedTouch_CanGetStartPositionAndTimeOfTouch()
 
     [Test]
     [Category("EnhancedTouch")]
-    public void EnhancedTouch_CanAccessHistoryOfTouch()
+    [TestCase(false)]
+    [TestCase(true)]
+    public void EnhancedTouch_CanAccessHistoryOfTouch_WithEventMergingSetTo(bool mergeRedundantEvents)
     {
+        InputSystem.settings.disableRedundantEventsMerging = !mergeRedundantEvents;
+
         // Noise. This one shouldn't show up in the history.
         BeginTouch(2, new Vector2(0.111f, 0.222f), queueEventOnly: true);
         EndTouch(2, new Vector2(0.111f, 0.222f), queueEventOnly: true);
@@ -514,6 +539,7 @@ public void EnhancedTouch_CanAccessHistoryOfTouch()
         runtime.currentTime = 0.987;
         MoveTouch(1, new Vector2(0.234f, 0.345f), queueEventOnly: true);
         MoveTouch(1, new Vector2(0.345f, 0.456f), queueEventOnly: true);
+        MoveTouch(1, new Vector2(0.456f, 0.567f), queueEventOnly: true);
         BeginTouch(3, new Vector2(0.666f, 0.666f), queueEventOnly: true);
         BeginTouch(4, new Vector2(0.777f, 0.777f), queueEventOnly: true);
         EndTouch(4, new Vector2(0.888f, 0.888f), queueEventOnly: true);
@@ -523,18 +549,22 @@ public void EnhancedTouch_CanAccessHistoryOfTouch()
         Assert.That(Touch.activeTouches, Has.Count.EqualTo(3));
 
         Assert.That(Touch.activeTouches[0].touchId, Is.EqualTo(1));
-        Assert.That(Touch.activeTouches[0].history, Has.Count.EqualTo(2));
+        Assert.That(Touch.activeTouches[0].history, Has.Count.EqualTo(mergeRedundantEvents ? 2 : 3));
         Assert.That(Touch.activeTouches[0].history, Has.All.Property("finger").SameAs(Touch.activeTouches[0].finger));
-        Assert.That(Touch.activeTouches[0].history[0].phase, Is.EqualTo(TouchPhase.Moved));
-        Assert.That(Touch.activeTouches[0].history[1].phase, Is.EqualTo(TouchPhase.Began));
-        Assert.That(Touch.activeTouches[0].history[0].time, Is.EqualTo(0.987));
-        Assert.That(Touch.activeTouches[0].history[1].time, Is.EqualTo(0.876));
-        Assert.That(Touch.activeTouches[0].history[0].startTime, Is.EqualTo(0.876));
-        Assert.That(Touch.activeTouches[0].history[1].startTime, Is.EqualTo(0.876));
-        Assert.That(Touch.activeTouches[0].history[0].startScreenPosition,
-            Is.EqualTo(new Vector2(0.123f, 0.234f)).Using(Vector2EqualityComparer.Instance));
-        Assert.That(Touch.activeTouches[0].history[1].startScreenPosition,
+        var beganIndex = mergeRedundantEvents ? 1 : 2;
+        Assert.That(Touch.activeTouches[0].history[beganIndex].phase, Is.EqualTo(TouchPhase.Began));
+        Assert.That(Touch.activeTouches[0].history[beganIndex].time, Is.EqualTo(0.876));
+        Assert.That(Touch.activeTouches[0].history[beganIndex].startTime, Is.EqualTo(0.876));
+        Assert.That(Touch.activeTouches[0].history[beganIndex].startScreenPosition,
             Is.EqualTo(new Vector2(0.123f, 0.234f)).Using(Vector2EqualityComparer.Instance));
+        for (int index = 0; index < (mergeRedundantEvents ? 1 : 2); ++index)
+        {
+            Assert.That(Touch.activeTouches[0].history[index].phase, Is.EqualTo(TouchPhase.Moved));
+            Assert.That(Touch.activeTouches[0].history[index].time, Is.EqualTo(0.987));
+            Assert.That(Touch.activeTouches[0].history[index].startTime, Is.EqualTo(0.876));
+            Assert.That(Touch.activeTouches[0].history[index].startScreenPosition,
+                Is.EqualTo(new Vector2(0.123f, 0.234f)).Using(Vector2EqualityComparer.Instance));
+        }
 
         Assert.That(Touch.activeTouches[1].touchId, Is.EqualTo(3));
         Assert.That(Touch.activeTouches[1].history, Is.Empty);

Packages/com.unity.inputsystem/CHANGELOG.md

@@ -30,7 +30,7 @@ however, it has to be formatted properly to pass verification tests.
 - Fixed InvalidOperationException when opening a preset created from a .inputactions asset ([case 1199544](https://issuetracker.unity3d.com/issues/input-system-properties-are-not-visible-and-invalidoperationexception-is-thrown-on-selecting-inputactionimporter-preset-asset)).
 - Fixed a problem arising when combining InputSystemUIInputModule and PlayInput with SendMessage or BroadcastMessage callback behavior on the same game object or hierarchy which is an ambiguous input setup. This fix eliminates callbacks into InputSystemUIInputModule. Related to ([1343712](https://issuetracker.unity3d.com/issues/input-system-ui-components-lags-when-using-input-system-ui-input-module-together-with-player-input-component)).
 - Fixed inconsistent usage of `ENABLE_PROFILER` define together with `Profiler.BeginSample`/`Profiler.EndSample` by removing `ENABLE_PROFILER` macro check because `BeginSample`/`EndSample` are already conditional with `[Conditional("ENABLE_PROFILER")]` ([case 1350139](https://issuetracker.unity3d.com/issues/inconsistent-enable-profiler-scripting-defines-in-inputmanager-dot-cs-when-using-profiler-dot-beginssample-and-profiler-dot-endsample)).
-- Remediated majority of performance issues with high frequency mice (>=1kHz poll rates) in release mode by merging consecutive mouse move events together ([case 1281266](https://issuetracker.unity3d.com/issues/many-input-events-when-using-1000hz-mouse)).
+- Remediated majority of performance issues with high frequency mice (>=1kHz poll rates) in release mode by merging consecutive mouse move events together ([case 1281266](https://issuetracker.unity3d.com/issues/many-input-events-when-using-1000hz-mouse)), see the events documentation for more information.
 - Fixed `InputEventTrace` replays skipping over empty frames and thus causing playback to happen too fast.
 - Fixed `"Pointer should have exited all objects before being removed"` error when changing screen orientation on mobile.
 - Controls such as mouse positions are no longer reset when focus is lost.
@@ -85,6 +85,7 @@ however, it has to be formatted properly to pass verification tests.
        }
    ");
    ```
+- Improved performance of `Touchscreen` by merging consecutive touch move events together. See the events documentation for more information.
 
 #### Actions
 

Packages/com.unity.inputsystem/Documentation~/Events.md

@@ -9,6 +9,7 @@
     * [Reading state events](#reading-state-events)
     * [Creating events](#creating-events)
     * [Capturing events](#capturing-events)
+* [Merging of events](#merging-of-events)
 
 The Input System is event-driven. All input is delivered as events, and you can generate custom input by injecting events. You can also observe all source input by listening in on the events flowing through the system.
 
@@ -231,3 +232,45 @@ controller.PlayAllFramesOnyByOne();
 // Replay events in a way that tries to simulate original event timing.
 controller.PlayAllEventsAccordingToTimestamps();
 ```
+
+## Merging of events
+
+Input system uses event mering to reduce amount of events required to be processed.
+This greatly improves performance when working with high refresh rate devices like 8000 Hz mice, touchscreens and others.
+
+For example let's take a stream of 7 mouse events coming in the same update:
+
+```
+
+Mouse       Mouse       Mouse       Mouse       Mouse       Mouse       Mouse
+Event no1   Event no2   Event no3   Event no4   Event no5   Event no6   Event no7
+Time 1      Time 2      Time 3      Time 4      Time 5      Time 6      Time 7
+Pos(10,20)  Pos(12,21)  Pos(13,23)  Pos(14,24)  Pos(16,25)  Pos(17,27)  Pos(18,28)
+Delta(1,1)  Delta(2,1)  Delta(1,2)  Delta(1,1)  Delta(2,1)  Delta(1,2)  Delta(1,1)
+BtnLeft(0)  BtnLeft(0)  BtnLeft(0)  BtnLeft(1)  BtnLeft(1)  BtnLeft(1)  BtnLeft(1)
+```
+
+To reduce workload we can skip events that are not encoding button state changes:
+
+```
+                        Mouse       Mouse                               Mouse
+                        Time 3      Time 4                              Time 7
+                        Event no3   Event no4                           Event no7
+                        Pos(13,23)  Pos(14,24)                          Pos(18,28)
+                        Delta(3,3)  Delta(1,1)                          Delta(4,4)
+                        BtnLeft(0)  BtnLeft(1)                          BtnLeft(1)
+```
+
+In that case we combine no1, no2, no3 together into no3 and accumulate the delta,
+then we keep no4 because it stores the transition from button unpressed to button pressed,
+and it's important to keep the exact timestamp of such transition.
+Later we combine no5, no6, no7 together into no7 because it is the last event in the update.
+
+Currently this approach is implemented for:
+- `FastMouse`, combines events unless `buttons` or `clickCount` differ in `MouseState`.
+- `Touchscreen`, combines events unless `touchId`, `phaseId` or `flags` differ in `TouchState`.
+
+You can disable merging of events by:
+```
+InputSystem.settings.disableRedundantEventsMerging = true;
+```

Packages/com.unity.inputsystem/InputSystem/Devices/IEventMerger.cs

@@ -4,6 +4,6 @@ namespace UnityEngine.InputSystem
 {
     internal interface IEventMerger
     {
-        bool MergeForward(InputEventPtr currentEvent, InputEventPtr nextEvent);
+        bool MergeForward(InputEventPtr currentEventPtr, InputEventPtr nextEventPtr);
     }
 }

Packages/com.unity.inputsystem/InputSystem/Devices/InputDevice.cs

@@ -620,14 +620,15 @@ internal enum DeviceFlags
             HasStateCallbacks = 1 << 1,
             HasControlsWithDefaultState = 1 << 2,
             HasDontResetControls = 1 << 3,
+            HasEventMerger = 1 << 4,
 
-            Remote = 1 << 4, // It's a local mirror of a device from a remote player connection.
-            Native = 1 << 5, // It's a device created from data surfaced by NativeInputRuntime.
+            Remote = 1 << 5, // It's a local mirror of a device from a remote player connection.
+            Native = 1 << 6, // It's a device created from data surfaced by NativeInputRuntime.
 
-            DisabledInFrontend = 1 << 6, // Explicitly disabled on the managed side.
-            DisabledInRuntime = 1 << 7, // Disabled in the native runtime.
-            DisabledWhileInBackground = 1 << 8, // Disabled while the player is running in the background.
-            DisabledStateHasBeenQueriedFromRuntime = 1 << 9, // Whether we have fetched the current enable/disable state from the runtime.
+            DisabledInFrontend = 1 << 7, // Explicitly disabled on the managed side.
+            DisabledInRuntime = 1 << 8, // Disabled in the native runtime.
+            DisabledWhileInBackground = 1 << 9, // Disabled while the player is running in the background.
+            DisabledStateHasBeenQueriedFromRuntime = 1 << 10, // Whether we have fetched the current enable/disable state from the runtime.
 
             CanRunInBackground = 1 << 11,
             CanRunInBackgroundHasBeenQueried = 1 << 12,
@@ -756,7 +757,7 @@ internal bool hasControlsWithDefaultState
 
         internal bool hasDontResetControls
         {
-            get => (m_DeviceFlags & DeviceFlags.HasDontResetControls) != 0;
+            get => (m_DeviceFlags & DeviceFlags.HasDontResetControls) == DeviceFlags.HasDontResetControls;
             set
             {
                 if (value)
@@ -768,7 +769,7 @@ internal bool hasDontResetControls
 
         internal bool hasStateCallbacks
         {
-            get => (m_DeviceFlags & DeviceFlags.HasStateCallbacks) != 0;
+            get => (m_DeviceFlags & DeviceFlags.HasStateCallbacks) == DeviceFlags.HasStateCallbacks;
             set
             {
                 if (value)
@@ -778,6 +779,18 @@ internal bool hasStateCallbacks
             }
         }
 
+        internal bool hasEventMerger
+        {
+            get => (m_DeviceFlags & DeviceFlags.HasEventMerger) == DeviceFlags.HasEventMerger;
+            set
+            {
+                if (value)
+                    m_DeviceFlags |= DeviceFlags.HasEventMerger;
+                else
+                    m_DeviceFlags &= ~DeviceFlags.HasEventMerger;
+            }
+        }
+
         internal void AddDeviceUsage(InternedString usage)
         {
             var controlUsageCount = m_UsageToControl.LengthSafe();

Packages/com.unity.inputsystem/InputSystem/Devices/Precompiled/FastMouse.partial.cs

@@ -49,7 +49,7 @@ void IInputStateCallbackReceiver.OnStateEvent(InputEventPtr eventPtr)
             OnStateEvent(eventPtr);
         }
 
-        public unsafe bool MergeForward(InputEventPtr currentEventPtr, InputEventPtr nextEventPtr)
+        internal static unsafe bool MergeForward(InputEventPtr currentEventPtr, InputEventPtr nextEventPtr)
         {
             if (currentEventPtr.type != StateEvent.Type || nextEventPtr.type != StateEvent.Type)
                 return false;
@@ -71,5 +71,10 @@ public unsafe bool MergeForward(InputEventPtr currentEventPtr, InputEventPtr nex
             nextState->scroll += currentState->scroll;
             return true;
         }
+
+        bool IEventMerger.MergeForward(InputEventPtr currentEventPtr, InputEventPtr nextEventPtr)
+        {
+            return MergeForward(currentEventPtr, nextEventPtr);
+        }
     }
 }

Packages/com.unity.inputsystem/InputSystem/Devices/Touchscreen.cs

@@ -489,7 +489,7 @@ public enum TouchPhase
     /// </remarks>
     [InputControlLayout(stateType = typeof(TouchscreenState), isGenericTypeOfDevice = true)]
     [Scripting.Preserve]
-    public class Touchscreen : Pointer, IInputStateCallbackReceiver
+    public class Touchscreen : Pointer, IInputStateCallbackReceiver, IEventMerger
     {
         /// <summary>
         /// Synthetic control that has the data for the touch that is deemed the "primary" touch at the moment.
@@ -953,6 +953,33 @@ unsafe bool IInputStateCallbackReceiver.GetStateOffsetForEvent(InputControl cont
             return true;
         }
 
+        internal static unsafe bool MergeForward(InputEventPtr currentEventPtr, InputEventPtr nextEventPtr)
+        {
+            if (currentEventPtr.type != StateEvent.Type || nextEventPtr.type != StateEvent.Type)
+                return false;
+
+            var currentEvent = StateEvent.FromUnchecked(currentEventPtr);
+            var nextEvent = StateEvent.FromUnchecked(nextEventPtr);
+
+            if (currentEvent->stateFormat != TouchState.Format || nextEvent->stateFormat != TouchState.Format)
+                return false;
+
+            var currentState = (TouchState*)currentEvent->state;
+            var nextState = (TouchState*)nextEvent->state;
+
+            if (currentState->touchId != nextState->touchId || currentState->phaseId != nextState->phaseId || currentState->flags != nextState->flags)
+                return false;
+
+            nextState->delta += currentState->delta;
+
+            return true;
+        }
+
+        bool IEventMerger.MergeForward(InputEventPtr currentEventPtr, InputEventPtr nextEventPtr)
+        {
+            return MergeForward(currentEventPtr, nextEventPtr);
+        }
+
         // We can only detect taps on touch *release*. At which point it acts like a button that triggers and releases
         // in one operation.
         private static void TriggerTap(TouchControl control, ref TouchState state, InputEventPtr eventPtr)