acc: add ability to skip tests dynamically (#2894)

## Changes
- If acceptance test outputs "SKIP_TEST <reason>" as last line, test
runner will call t.Skip() on this test.
- Modify "Summary" output to indicate that this about directories, not
test cases (single directory can expand to many test cases; dynamic
skips are not reflected in the output of Summary line).

## Why
See #2876 -- we test all
combinations of default-python template there but not all of them work
in all environments.

## Tests
New selftest.
Testing in practice in #2876

Author: denik

acceptance/acceptance_test.go

@@ -257,7 +257,7 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 		})
 	}
 
-	t.Logf("Summary: %d/%d/%d run/selected/total, %d skipped", selectedDirs-skippedDirs, selectedDirs, totalDirs, skippedDirs)
+	t.Logf("Summary (dirs): %d/%d/%d run/selected/total, %d skipped", selectedDirs-skippedDirs, selectedDirs, totalDirs, skippedDirs)
 
 	return len(testDirs)
 }
@@ -471,7 +471,11 @@ func runTest(t *testing.T,
 	require.NoError(t, err)
 	defer out.Close()
 
-	err = runWithLog(t, cmd, out, tailOutput)
+	skipReason, err := runWithLog(t, cmd, out, tailOutput)
+
+	if skipReason != "" {
+		t.Skip("Skipping based on output: " + skipReason)
+	}
 
 	// Include exit code in output (if non-zero)
 	formatOutput(out, err)
@@ -847,7 +851,7 @@ func isTruePtr(value *bool) bool {
 	return value != nil && *value
 }
 
-func runWithLog(t *testing.T, cmd *exec.Cmd, out *os.File, tail bool) error {
+func runWithLog(t *testing.T, cmd *exec.Cmd, out *os.File, tail bool) (string, error) {
 	r, w := io.Pipe()
 	cmd.Stdout = w
 	cmd.Stderr = w
@@ -863,14 +867,16 @@ func runWithLog(t *testing.T, cmd *exec.Cmd, out *os.File, tail bool) error {
 	start := time.Now()
 	err := cmd.Start()
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	go func() {
 		processErrCh <- cmd.Wait()
 		_ = w.Close()
 	}()
 
+	mostRecentLine := ""
+
 	reader := bufio.NewReader(r)
 	for {
 		line, err := reader.ReadString('\n')
@@ -882,6 +888,7 @@ func runWithLog(t *testing.T, cmd *exec.Cmd, out *os.File, tail bool) error {
 			}
 		}
 		if len(line) > 0 {
+			mostRecentLine = line
 			_, err = out.WriteString(line)
 			require.NoError(t, err)
 		}
@@ -891,7 +898,13 @@ func runWithLog(t *testing.T, cmd *exec.Cmd, out *os.File, tail bool) error {
 		require.NoError(t, err)
 	}
 
-	return <-processErrCh
+	mostRecentLine = strings.TrimRight(mostRecentLine, "\n")
+	skipReason := ""
+	if strings.HasPrefix(mostRecentLine, "SKIP_TEST") {
+		skipReason = mostRecentLine
+	}
+
+	return skipReason, <-processErrCh
 }
 
 func getCloudEnvBase(cloudEnv string) string {

acceptance/selftest/skip/output.txt

@@ -0,0 +1 @@
+output.txt will not be checked for skipped test

acceptance/selftest/skip/script

@@ -0,0 +1 @@
+echo SKIP_TEST testing skip functionality