From 6db67fe2d31b2add61de6a63b2ec0f3f4b086a84 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sun, 15 Feb 2026 07:01:51 +0100 Subject: [PATCH 1/7] Increase the threshold when running performance tests on macOS in CI --- .../src/suite/performance.vscode.test.ts | 25 ++++++++++++++----- packages/node-common/src/isOS.ts | 4 +++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index e79d3cd21f..c0923e11df 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -9,16 +9,18 @@ import { openNewEditor, runCursorlessAction } from "@cursorless/vscode-common"; import assert from "assert"; import * as vscode from "vscode"; import { endToEndTestSetup } from "../endToEndTestSetup"; +import { isCI } from "../isCI"; +import { isMac } from "@cursorless/node-common"; const testData = generateTestData(100); - -const smallThresholdMs = 100; -const largeThresholdMs = 600; -const xlThresholdMs = 800; +const multiplier = calculateMultiplier(); +const smallThresholdMs = 50 * multiplier; +const largeThresholdMs = 300 * multiplier; +const thresholds = [smallThresholdMs, largeThresholdMs]; type ModifierType = "containing" | "previous" | "every"; -suite("Performance", async function () { +suite(`Performance ${thresholds.join("/")}ms`, async function () { endToEndTestSetup(this); let previousTitle = ""; @@ -111,7 +113,7 @@ suite("Performance", async function () { test( "Select surroundingPair.any with multiple cursors", asyncSafety(() => - selectWithMultipleCursors(xlThresholdMs, { + selectWithMultipleCursors(largeThresholdMs, { type: "surroundingPair", delimiter: "any", }), @@ -255,3 +257,14 @@ function generateTestData(n: number): string { ); return JSON.stringify(obj, null, 2); } + +function calculateMultiplier() { + if (isCI()) { + // The GitHub test runner for macOS is very slow, so we increase the thresholds for macOS in CI. + if (isMac()) { + return 4; + } + return 2; + } + return 1; +} diff --git a/packages/node-common/src/isOS.ts b/packages/node-common/src/isOS.ts index 58d74d69c6..fb3fe64699 100644 --- a/packages/node-common/src/isOS.ts +++ b/packages/node-common/src/isOS.ts @@ -7,3 +7,7 @@ export function isWindows() { export function isLinux() { return os.platform() === "linux"; } + +export function isMac() { + return os.platform() === "darwin"; +} From fcc5efb980145ef78a6cea58f78d36cbcfe2e1e4 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sun, 15 Feb 2026 07:24:36 +0100 Subject: [PATCH 2/7] Reintroduce xl threshold --- .../src/suite/performance.vscode.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index c0923e11df..d00f9d24c2 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -16,11 +16,12 @@ const testData = generateTestData(100); const multiplier = calculateMultiplier(); const smallThresholdMs = 50 * multiplier; const largeThresholdMs = 300 * multiplier; -const thresholds = [smallThresholdMs, largeThresholdMs]; +const xlThresholdMs = 400 * multiplier; +const thresholds = [smallThresholdMs, largeThresholdMs, xlThresholdMs]; type ModifierType = "containing" | "previous" | "every"; -suite(`Performance ${thresholds.join("/")}ms`, async function () { +suite(`Performance ${thresholds.join("/")} ms`, async function () { endToEndTestSetup(this); let previousTitle = ""; @@ -113,7 +114,7 @@ suite(`Performance ${thresholds.join("/")}ms`, async function () { test( "Select surroundingPair.any with multiple cursors", asyncSafety(() => - selectWithMultipleCursors(largeThresholdMs, { + selectWithMultipleCursors(xlThresholdMs, { type: "surroundingPair", delimiter: "any", }), @@ -199,7 +200,7 @@ async function testPerformanceCallback( const duration = Math.round(performance.now() - start); - console.log(` ${duration} ms`); + console.log(` ${duration} / ${thresholdMs} ms`); assert.ok( duration < thresholdMs, From d5448c8db6bbb9ea05698ea5283d28da2ddf657e Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sun, 15 Feb 2026 07:27:03 +0100 Subject: [PATCH 3/7] allow duration equal to threshold --- .../cursorless-vscode-e2e/src/suite/performance.vscode.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index d00f9d24c2..3887fa44f2 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -203,7 +203,7 @@ async function testPerformanceCallback( console.log(` ${duration} / ${thresholdMs} ms`); assert.ok( - duration < thresholdMs, + duration <= thresholdMs, `Duration ${duration}ms exceeds threshold ${thresholdMs}ms`, ); } From ff1c68dd0946e70117fa7f2428e39da62e8e7685 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sun, 15 Feb 2026 08:06:45 +0100 Subject: [PATCH 4/7] run only performance test for debug reasons --- .../src/suite/performance.vscode.test.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 3887fa44f2..5376dbfec3 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -16,12 +16,12 @@ const testData = generateTestData(100); const multiplier = calculateMultiplier(); const smallThresholdMs = 50 * multiplier; const largeThresholdMs = 300 * multiplier; -const xlThresholdMs = 400 * multiplier; -const thresholds = [smallThresholdMs, largeThresholdMs, xlThresholdMs]; +const thresholds = [smallThresholdMs, largeThresholdMs]; type ModifierType = "containing" | "previous" | "every"; -suite(`Performance ${thresholds.join("/")} ms`, async function () { +// eslint-disable-next-line mocha/no-exclusive-tests +suite.only(`Performance ${thresholds.join("/")} ms`, async function () { endToEndTestSetup(this); let previousTitle = ""; @@ -114,7 +114,7 @@ suite(`Performance ${thresholds.join("/")} ms`, async function () { test( "Select surroundingPair.any with multiple cursors", asyncSafety(() => - selectWithMultipleCursors(xlThresholdMs, { + selectWithMultipleCursors(largeThresholdMs, { type: "surroundingPair", delimiter: "any", }), @@ -260,10 +260,13 @@ function generateTestData(n: number): string { } function calculateMultiplier() { + // The GitHub test runner is generally slower than running tests locally, so + // we increase the thresholds in CI. We do this for all platforms, but + // especially for macOS since the GitHub test runner for macOS is particularly + // slow. if (isCI()) { - // The GitHub test runner for macOS is very slow, so we increase the thresholds for macOS in CI. if (isMac()) { - return 4; + return 3; } return 2; } From 48e80f4ecd194fda36330f69156f6561147e68c4 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sun, 15 Feb 2026 08:18:34 +0100 Subject: [PATCH 5/7] update --- .../src/suite/performance.vscode.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 5376dbfec3..76fe10020f 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -16,7 +16,8 @@ const testData = generateTestData(100); const multiplier = calculateMultiplier(); const smallThresholdMs = 50 * multiplier; const largeThresholdMs = 300 * multiplier; -const thresholds = [smallThresholdMs, largeThresholdMs]; +const xlThresholdMs = 400 * multiplier; +const thresholds = [smallThresholdMs, largeThresholdMs, xlThresholdMs]; type ModifierType = "containing" | "previous" | "every"; @@ -114,7 +115,7 @@ suite.only(`Performance ${thresholds.join("/")} ms`, async function () { test( "Select surroundingPair.any with multiple cursors", asyncSafety(() => - selectWithMultipleCursors(largeThresholdMs, { + selectWithMultipleCursors(xlThresholdMs, { type: "surroundingPair", delimiter: "any", }), From 57dd7678ed73a39043d383b3fbd9ee52e0e31167 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sun, 15 Feb 2026 08:25:14 +0100 Subject: [PATCH 6/7] Re enable all tests --- .../cursorless-vscode-e2e/src/suite/performance.vscode.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 76fe10020f..5c2dd40667 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -21,8 +21,7 @@ const thresholds = [smallThresholdMs, largeThresholdMs, xlThresholdMs]; type ModifierType = "containing" | "previous" | "every"; -// eslint-disable-next-line mocha/no-exclusive-tests -suite.only(`Performance ${thresholds.join("/")} ms`, async function () { +suite(`Performance ${thresholds.join("/")} ms`, async function () { endToEndTestSetup(this); let previousTitle = ""; From cae426b8159c24ed23519d82fcf85bc5830f87d9 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sun, 15 Feb 2026 08:30:16 +0100 Subject: [PATCH 7/7] Update comment --- .../cursorless-vscode-e2e/src/suite/performance.vscode.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 5c2dd40667..806f34864f 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -26,7 +26,7 @@ suite(`Performance ${thresholds.join("/")} ms`, async function () { let previousTitle = ""; - // Before each test, print the test title. This is done we have the test + // Before each test, print the test title. This is done so we have the test // title before the test run time / duration. this.beforeEach(function () { const title = this.currentTest!.title;