diff --git a/packages/core/src/util.ts b/packages/core/src/util.ts index 35e6aa1..cfe17d4 100644 --- a/packages/core/src/util.ts +++ b/packages/core/src/util.ts @@ -52,6 +52,17 @@ export class Util { return locator; } + static shuffle(array: Array) + { + let currentIndex = array.length, randomIndex; + while (currentIndex != 0) { + randomIndex = Math.floor(Math.random() * currentIndex); + currentIndex--; + [array[currentIndex], array[randomIndex]] = [ + array[randomIndex], array[currentIndex]]; + } + return array; + } /** * Loads blocked tests by reading a file at path specified by env var BLOCK_TESTS_FILE. * This works on the assumption that the format of JSON stored in file is in the following format: diff --git a/packages/core/src/validations.ts b/packages/core/src/validations.ts index dda4a6c..e102e7f 100644 --- a/packages/core/src/validations.ts +++ b/packages/core/src/validations.ts @@ -38,6 +38,9 @@ export class Validations { if (process.env.BUILD_ID === undefined) { throw new ValidationException("'BUILD_ID' is not set in environment variables.") } + if (process.env.SHUFFLE_TEST === undefined) { + throw new ValidationException("'SHUFFLE_TEST' is not set in environment variables.") + } } } diff --git a/packages/jasmine-runner/src/jasmine-runner.ts b/packages/jasmine-runner/src/jasmine-runner.ts index 6428b2d..07f3aeb 100644 --- a/packages/jasmine-runner/src/jasmine-runner.ts +++ b/packages/jasmine-runner/src/jasmine-runner.ts @@ -59,7 +59,8 @@ class JasmineRunner implements TestRunner { } } - async execute(testFilesGlob: string | string[], config: string, locators: string[] = []): Promise { + async execute(testFilesGlob: string | string[], shuffleTest: boolean, config: string, + locators: string[] = []): Promise { const testLocators = new Set(locators) const blockTestLocators = new Set() const entityIdFilenameMap = new Map(); @@ -89,7 +90,7 @@ class JasmineRunner implements TestRunner { const rootSuite = jasmineObj.env.topSuite(); const specIdsToRun: number[] = []; - this.fetchSpecIdsToRun(rootSuite, specIdsToRun, entityIdFilenameMap, + this.fetchSpecIdsToRunAndDetermineOrder(rootSuite, specIdsToRun, shuffleTest, entityIdFilenameMap, testLocators, blockTestLocators); if (specIdsToRun.length == 0) { @@ -127,6 +128,7 @@ class JasmineRunner implements TestRunner { const orgID = process.env.ORG_ID as ID; const repoID = process.env.REPO_ID as ID; const commitID = process.env.COMMIT_ID as ID; + const shuffleTest = JSON.parse(process.env.SHUFFLE_TEST as string) as boolean; Validations.validateExecutionEnv(argv); const testFilesGlob = argv.pattern as string | string[]; const locatorFile = argv.locatorFile as string; @@ -140,11 +142,11 @@ class JasmineRunner implements TestRunner { ); if (locatorFile) { const locators = Util.getLocatorsFromFile(locatorFile) - const result = await this.execute(testFilesGlob, argv.config, locators) + const result = await this.execute(testFilesGlob, shuffleTest, argv.config, locators) executionResults.push(result) } else { // run all tests if locator file is not present - const result = await this.execute(testFilesGlob, argv.config) + const result = await this.execute(testFilesGlob, shuffleTest, argv.config) executionResults.push(result) } return executionResults; @@ -265,21 +267,26 @@ class JasmineRunner implements TestRunner { } } } - - private fetchSpecIdsToRun( + + private fetchSpecIdsToRunAndDetermineOrder( currentSuite: jasmine.Suite, specIdsToRun: number[], + shuffleTest: boolean, entityIdFilenameMap: Map, testLocators: Set, blockTestLocators: Set, ancestorTitles: string[] = [], ) { - for (const child of currentSuite.children) { + let currentSuiteChildren = currentSuite.children; + if (shuffleTest){ + currentSuiteChildren = Util.shuffle(currentSuite.children); + } + for (const child of currentSuiteChildren) { if ((child as jasmine.Suite).children !== undefined) { // child is a TestSuite const childSuite = child as jasmine.Suite; ancestorTitles.push(child.description); - this.fetchSpecIdsToRun(childSuite, specIdsToRun, entityIdFilenameMap, + this.fetchSpecIdsToRunAndDetermineOrder(childSuite, specIdsToRun, shuffleTest, entityIdFilenameMap, testLocators, blockTestLocators, ancestorTitles); ancestorTitles.pop(); } else { diff --git a/packages/mocha-runner/src/mocha-runner.ts b/packages/mocha-runner/src/mocha-runner.ts index e62ef8f..159304f 100644 --- a/packages/mocha-runner/src/mocha-runner.ts +++ b/packages/mocha-runner/src/mocha-runner.ts @@ -91,13 +91,14 @@ class MochaRunner implements TestRunner { return result; } - async execute(testFilesGlob: string | string[], locators: string[] = []): Promise { + async execute(testFilesGlob: string | string[], shuffleTest: boolean, + locators: string [] = []): Promise { const mocha = this.createMochaInstance() const testRunTask = new Task(); this._testlocator = new Set(locators); - this.extendNativeRunner(); + this.extendNativeRunner(shuffleTest); mocha.reporter(require.resolve("./mocha-reporter")); let testFilesToProcess: Set = new Set(); @@ -140,6 +141,7 @@ class MochaRunner implements TestRunner { const orgID = process.env.ORG_ID as ID; const repoID = process.env.REPO_ID as ID; const commitID = process.env.COMMIT_ID as ID; + const shuffleTest = JSON.parse(process.env.SHUFFLE_TEST as string) as boolean; Validations.validateExecutionEnv(argv); const testFilesGlob = argv.pattern as string | string[]; const locatorFile = argv.locatorFile as string; @@ -153,11 +155,11 @@ class MochaRunner implements TestRunner { if (locatorFile) { const locators = Util.getLocatorsFromFile(locatorFile); - const result = await this.execute(testFilesGlob, locators); + const result = await this.execute(testFilesGlob, shuffleTest, locators); executionResults.push(result); } else { // run all tests if locator file is not present - const result = await this.execute(testFilesGlob) + const result = await this.execute(testFilesGlob, shuffleTest) executionResults.push(result) } return executionResults; @@ -219,19 +221,22 @@ class MochaRunner implements TestRunner { } } - private extendNativeRunner() { + private extendNativeRunner(shuffleTest: boolean) { // eslint-disable-next-line @typescript-eslint/no-this-alias const _self = this; // This is the hook point where we can randomizee, specify order // or do any sort of stuffs with suites and tests Mocha.Runner.prototype.run = function (fn: ((failures: number) => void)) { - _self.filterSpecs(this.suite); + _self.filterSpecsAndDetermineOrder(this.suite, shuffleTest); return originalRun.call(this, fn); }; } - private filterSpecs(suite: Mocha.Suite) { + private filterSpecsAndDetermineOrder(suite: Mocha.Suite, shuffleTest:boolean) { if (suite.tests) { + if(shuffleTest){ + suite.tests = Util.shuffle(suite.tests) as Mocha.Test[]; + } const filteredTests: Mocha.Test[] = []; for (const test of suite.tests) { const filename = test.file ?? ""; @@ -271,9 +276,12 @@ class MochaRunner implements TestRunner { } } } - suite.tests = filteredTests; + suite.tests = filteredTests; } if (suite.suites) { + if (shuffleTest){ + suite.suites = Util.shuffle(suite.suites) as Mocha.Suite[]; + } for (const childSuite of suite.suites) { const filename = childSuite.file ?? ""; const parentSuites: string[] = []; @@ -289,7 +297,7 @@ class MochaRunner implements TestRunner { ); this._blockedSuites.push(suiteResult); } - this.filterSpecs(childSuite); + this.filterSpecsAndDetermineOrder(childSuite, shuffleTest); } } }