Revert "Skip costly tests" (#35197)

* Revert "Skip costly tests"

* fix package.json format

Author: sandersn

Gulpfile.js

@@ -477,7 +477,6 @@ task("runtests-parallel").flags = {
     "   --workers=<number>": "The number of parallel workers to use.",
     "   --timeout=<ms>": "Overrides the default test timeout.",
     "   --built": "Compile using the built version of the compiler.",
-    "   --skipPercent=<number>": "Skip expensive tests with <percent> chance to miss an edit. Default 5%.",
     "   --shards": "Total number of shards running tests (default: 1)",
     "   --shardId": "1-based ID of this shard (default: 1)",
 };

package.json

@@ -92,7 +92,6 @@
         "prex": "^0.4.3",
         "q": "latest",
         "remove-internal": "^2.9.2",
-        "simple-git": "^1.113.0",
         "source-map-support": "latest",
         "through2": "latest",
         "travis-fold": "latest",
@@ -117,8 +116,7 @@
         "lint:ci": "gulp lint --ci",
         "lint:compiler": "gulp lint-compiler",
         "lint:scripts": "gulp lint-scripts",
-        "setup-hooks": "node scripts/link-hooks.js",
-        "update-costly-tests": "node scripts/costly-tests.js"
+        "setup-hooks": "node scripts/link-hooks.js"
     },
     "browser": {
         "fs": false,

scripts/build/options.js

@@ -14,7 +14,6 @@ module.exports = minimist(process.argv.slice(2), {
         "ru": "runners", "runner": "runners",
         "r": "reporter",
         "c": "colors", "color": "colors",
-        "skip-percent": "skipPercent",
         "skippercent": "skipPercent",
         "w": "workers",
         "f": "fix"

scripts/build/tests.js

@@ -31,7 +31,6 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode,
     const inspect = cmdLineOptions.inspect;
     const runners = cmdLineOptions.runners;
     const light = cmdLineOptions.light;
-    const skipPercent = process.env.CI === "true" ? 0 : cmdLineOptions.skipPercent;
     const stackTraceLimit = cmdLineOptions.stackTraceLimit;
     const testConfigFile = "test.config";
     const failed = cmdLineOptions.failed;
@@ -65,8 +64,8 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode,
         testTimeout = 400000;
     }
 
-    if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || skipPercent !== undefined || shards || shardId) {
-        writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed, shards, shardId);
+    if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || shards || shardId) {
+        writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed, shards, shardId);
     }
 
     const colors = cmdLineOptions.colors;
@@ -161,7 +160,6 @@ exports.cleanTestDirs = cleanTestDirs;
  * @param {string} tests
  * @param {string} runners
  * @param {boolean} light
- * @param {string} skipPercent
  * @param {string} [taskConfigsFolder]
  * @param {string | number} [workerCount]
  * @param {string} [stackTraceLimit]
@@ -170,12 +168,11 @@ exports.cleanTestDirs = cleanTestDirs;
  * @param {number | undefined} [shards]
  * @param {number | undefined} [shardId]
  */
-function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed, shards, shardId) {
+function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed, shards, shardId) {
     const testConfigContents = JSON.stringify({
         test: tests ? [tests] : undefined,
         runners: runners ? runners.split(",") : undefined,
         light,
-        skipPercent,
         workerCount,
         stackTraceLimit,
         taskConfigsFolder,

scripts/costly-tests.js

@@ -11,7 +11,7 @@ namespace Harness.Parallel.Host {
         const isatty = tty.isatty(1) && tty.isatty(2);
         const path = require("path") as typeof import("path");
         const { fork } = require("child_process") as typeof import("child_process");
-        const { statSync, readFileSync } = require("fs") as typeof import("fs");
+        const { statSync } = require("fs") as typeof import("fs");
 
         // NOTE: paths for module and types for FailedTestReporter _do not_ line up due to our use of --outFile for run.js
         const FailedTestReporter = require(path.resolve(__dirname, "../../scripts/failed-tests")) as typeof import("../../../scripts/failed-tests");
@@ -186,31 +186,6 @@ namespace Harness.Parallel.Host {
             return `tsrunner-${runner}://${test}`;
         }
 
-        function skipCostlyTests(tasks: Task[]) {
-            if (statSync("tests/.test-cost.json")) {
-                const costs = JSON.parse(readFileSync("tests/.test-cost.json", "utf8")) as {
-                    totalTime: number,
-                    totalEdits: number,
-                    data: { name: string, time: number, edits: number, costs: number }[]
-                };
-                let skippedEdits = 0;
-                let skippedTime = 0;
-                const skippedTests = new Set<string>();
-                let i = 0;
-                for (; i < costs.data.length && (skippedEdits / costs.totalEdits) < (skipPercent / 100); i++) {
-                    skippedEdits += costs.data[i].edits;
-                    skippedTime += costs.data[i].time;
-                    skippedTests.add(costs.data[i].name);
-                }
-                console.log(`Skipped ${i} expensive tests; estimated time savings of ${(skippedTime / costs.totalTime * 100).toFixed(2)}% with --skipPercent=${skipPercent.toFixed(2)} chance of missing a test.`);
-                return tasks.filter(t => !skippedTests.has(t.file));
-            }
-            else {
-                console.log("No cost analysis discovered.");
-                return tasks;
-            }
-        }
-
         function startDelayed(perfData: { [testHash: string]: number } | undefined, totalCost: number) {
             console.log(`Discovered ${tasks.length} unittest suites` + (newTasks.length ? ` and ${newTasks.length} new suites.` : "."));
             console.log("Discovering runner-based tests...");
@@ -250,7 +225,6 @@ namespace Harness.Parallel.Host {
             }
             tasks.sort((a, b) => a.size - b.size);
             tasks = tasks.concat(newTasks);
-            tasks = skipCostlyTests(tasks);
             const batchCount = workerCount;
             const packfraction = 0.9;
             const chunkSize = 1000; // ~1KB or 1s for sending batches near the end of a test

src/testRunner/parallel/host.ts

@@ -67,7 +67,6 @@ let runUnitTests: boolean | undefined;
 let stackTraceLimit: number | "full" | undefined;
 let noColors = false;
 let keepFailed = false;
-let skipPercent = 5;
 
 interface TestConfig {
     light?: boolean;
@@ -81,7 +80,6 @@ interface TestConfig {
     noColors?: boolean;
     timeout?: number;
     keepFailed?: boolean;
-    skipPercent?: number;
     shardId?: number;
     shards?: number;
 }
@@ -115,9 +113,6 @@ function handleTestConfig() {
         if (testConfig.keepFailed) {
             keepFailed = true;
         }
-        if (testConfig.skipPercent !== undefined) {
-            skipPercent = testConfig.skipPercent;
-        }
         if (testConfig.shardId) {
             shardId = testConfig.shardId;
         }