Core: Create config.testIds to expose all available test ids

kltan · Kean Tan · commit bf39c182a1cf · 2020-01-08T15:15:44.000-08:00
What this does

Currently each testId is calculated as each test is executed.
Config.testIds pre-hashes all the loaded tests on demand and returns all
testIds.

How is this helpful

Currently test runners can get access to test modules via config.modules
and partition the modules based on some distribution strategy for running
the tests in parallel. This is fine but it can lead to unbalanced
distribution as some modules have a lot of tests and some modules are
testing integration (longer running) test instead of unit test.

Partitioning the tests by modules caused some instance of test runners
to run way longer than others as some heavier running modules might be
clumped together in the same instance.

To mitigate and minimize the unbalanced test runs, the proposed solution
is to partition based on each individual test with testId. The test
runners can now access testIds beforehand and can distribute those ids
with the testId url param. This will minimize the test execution time
descrepencies between each individual instance of the test runners.
diff --git a/Gruntfile.js b/Gruntfile.js
@@ -165,6 +165,7 @@ module.exports = function( grunt ) {
 						"test/reporter-html/xhtml-single-testid.xhtml",
 						"test/reporter-urlparams.html",
 						"test/moduleId.html",
+						"test/testIds.html",
 						"test/onerror/inside-test.html",
 						"test/onerror/outside-test.html",
 						"test/only.html",
diff --git a/docs/config/QUnit.config.md b/docs/config/QUnit.config.md
@@ -101,6 +101,10 @@ Enabling this option will cause tests to fail, if they don't call `expect()` at
 
 This property allows QUnit to run specific tests identified by the hashed version of their module name and test name. You can specify one or multiple tests to run.
 
+### `QUnit.config.testIds` (array) | default: `array`
+
+This property allows QUnit to return all hashed testIds of all loaded tests.
+
 ### `QUnit.config.testTimeout` (number) | default: `undefined`
 
 Specify a global timeout in milliseconds after which all tests will fail with an appropriate message. Useful when async tests aren't finishing, to prevent the testrunner getting stuck. Set to something high, e.g. 30000 (30 seconds) to avoid slow tests to time out by accident.
diff --git a/package-lock.json b/package-lock.json
diff --git a/src/core/config.js b/src/core/config.js
@@ -1,5 +1,5 @@
 import { window, localSessionStorage } from "../globals";
-import { extend } from "./utilities";
+import { extend, generateHash } from "./utilities";
 
 /**
  * Config object: Maintain internal state
@@ -58,7 +58,22 @@ const config = {
 	callbacks: {},
 
 	// The storage module to use for reordering tests
-	storage: localSessionStorage
+	storage: localSessionStorage,
+
+	// Get the testIds for all tests, value is read only
+	get testIds() {
+		var i, j, ml, tl, testIds = [];
+
+		// Loop through all modules and tests and generate testIds
+		for ( i = 0, ml = config.modules.length; i < ml; i++ ) {
+			for ( j = 0, tl = config.modules[ i ].tests.length; j < tl; j++ ) {
+				testIds.push( generateHash( config.modules[ i ].name,
+					config.modules[ i ].tests[ j ].name ) );
+			}
+		}
+		return testIds;
+	}
+
 };
 
 // take a predefined QUnit.config and extend the defaults
diff --git a/test/testIds.html b/test/testIds.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<meta charset="UTF-8">
+		<title>QUnit ModuleId Test Suite</title>
+		<link rel="stylesheet" href="../dist/qunit.css">
+		<script src="../dist/qunit.js"></script>
+		<script src="testIds.js"></script>
+	</head>
+	<body>
+		<div id="qunit"></div>
+	</body>
+</html>
diff --git a/test/testIds.js b/test/testIds.js
@@ -0,0 +1,5 @@
+QUnit.module( "QUnit.config.testIds" );
+
+QUnit.test( "testIds should return all testIds", function( assert ) {
+	assert.deepEqual( QUnit.config.testIds, [ "01e80961" ] );
+} );
