acc: kill tests after timeout (#2793)

## Changes
- Kill tests after timeout expires.
- New test configuration: Timeout/TimeoutWindows/TimeoutCloud to set a
timeout for a test with possible overrides for known slow environments
(windows and cloud).

## Why
I saw a few cases where test suite is hanging.

Some tests take a surprisingly long time currently (apps/run-local),
having explicit setting that increases default timeout means we are
tracking those.

## Tests
New acceptance test: selftest/timeout. Manually checked its run time.

Author: denik

acceptance/acceptance_test.go

@@ -368,8 +368,22 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	err = CopyDir(dir, tmpDir, inputs, outputs)
 	require.NoError(t, err)
 
+	timeout := config.Timeout
+
+	if runtime.GOOS == "windows" {
+		if isRunningOnCloud {
+			timeout = max(timeout, config.TimeoutWindows, config.TimeoutCloud)
+		} else {
+			timeout = max(timeout, config.TimeoutWindows)
+		}
+	} else if isRunningOnCloud {
+		timeout = max(timeout, config.TimeoutCloud)
+	}
+
+	ctx, cancelFunc := context.WithTimeout(context.Background(), timeout)
+	defer cancelFunc()
 	args := []string{"bash", "-euo", "pipefail", EntryPointScript}
-	cmd := exec.Command(args[0], args[1:]...)
+	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
 	cmd.Env = os.Environ()
 	cmd.Env = append(cmd.Env, "UNIQUE_NAME="+uniqueName)
 	cmd.Env = append(cmd.Env, "TEST_TMP_DIR="+tmpDir)
@@ -806,15 +820,24 @@ func runWithLog(t *testing.T, cmd *exec.Cmd, out *os.File, tail bool) error {
 	r, w := io.Pipe()
 	cmd.Stdout = w
 	cmd.Stderr = w
+	processErrCh := make(chan error, 1)
+
+	cmd.Cancel = func() error {
+		processErrCh <- errors.New("Test script killed due to a timeout")
+		_ = cmd.Process.Kill()
+		_ = w.Close()
+		return nil
+	}
 
 	start := time.Now()
 	err := cmd.Start()
-	require.NoError(t, err)
+	if err != nil {
+		return err
+	}
 
-	processErrCh := make(chan error, 1)
 	go func() {
 		processErrCh <- cmd.Wait()
-		w.Close()
+		_ = w.Close()
 	}()
 
 	reader := bufio.NewReader(r)

acceptance/bundle/templates/experimental-jobs-as-code/test.toml

@@ -0,0 +1 @@
+TimeoutWindows = '120s'

acceptance/cmd/workspace/apps/run-local/test.toml

@@ -1,4 +1,6 @@
 RecordRequests = false
+Timeout = '1m'
+TimeoutWindows = '5m'
 
 Ignore = [
     '.venv',

acceptance/internal/config.go

@@ -96,6 +96,16 @@ type TestConfig struct {
 
 	// List of keys for which to do string replacement value -> [KEY]. If not set, defaults to true.
 	EnvRepl map[string]bool
+
+	// Maximum amount of time the test is allowed to run, will be killed after that.
+	Timeout time.Duration
+
+	// On windows/local run, max(Timeout, TimeoutWindows) is used for timeout
+	TimeoutWindows time.Duration
+
+	// For cloud tests, max(Timeout, TimeoutCloud) is used for timeout
+	// For cloud+windows tests, max(Timeout, TimeoutWindows, TimeoutCloud) is used for timeout
+	TimeoutCloud time.Duration
 }
 
 type ServerStub struct {

acceptance/selftest/timeout/output.txt

@@ -0,0 +1,4 @@
+
+>>> sleep 2
+
+Error: Test script killed due to a timeout

acceptance/selftest/timeout/script

@@ -0,0 +1,2 @@
+trace sleep 2
+echo "This is never printed"

acceptance/selftest/timeout/test.toml

@@ -0,0 +1,5 @@
+# Manually check that test takes about 0.1s:
+# PASS: TestAccept/selftest/timeout (0.10s)
+Timeout = '100ms'
+TimeoutWindows = '300ms'
+TimeoutCloud = '300ms'

acceptance/test.toml

@@ -2,5 +2,10 @@
 Local = true
 Cloud = false
 
+# default timeouts
+Timeout = '20s'
+TimeoutWindows = '60s'
+TimeoutCloud = '120s'
+
 Env.PYTHONDONTWRITEBYTECODE = "1"
 EnvRepl.PYTHONDONTWRITEBYTECODE = false