*: Transition from tap Diagnostic(...) to YAML(...)

Makefile

@@ -40,6 +40,14 @@ clean:
 	rm -f oci-runtime-tool runtimetest *.1 $(VALIDATION_TESTS)
 
 localvalidation:
+	@for EXECUTABLE in runtimetest $(VALIDATION_TESTS); \
+	do \
+		if test ! -x "$${EXECUTABLE}"; \
+		then \
+			echo "missing test executable $${EXECUTABLE}; run 'make runtimetest validation-executables'" >&2; \
+			exit 1; \
+		fi; \
+	done
 	RUNTIME=$(RUNTIME) $(TAP) $(VALIDATION_TESTS)
 
 .PHONY: validation-executables

README.md

@@ -39,15 +39,18 @@ $ npm install tap
 
 ```console
 $ make runtimetest validation-executables
+$ sudo make RUNTIME=runc localvalidation
 RUNTIME=runc tap validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t
 validation/linux_rootfs_propagation_shared.t ........ 18/19
   not ok rootfs propagation
+    error: 'rootfs should be shared, but not'
 
 validation/create.t ................................... 4/4
 validation/default.t ................................ 19/19
 validation/linux_readonly_paths.t ................... 19/19
 validation/linux_masked_paths.t ..................... 18/19
   not ok masked paths
+    error: /masktest should not be readable
 
 validation/mounts.t ................................... 0/1
   Skipped: 1

cmd/runtimetest/main.go

@@ -945,15 +945,15 @@ func run(context *cli.Context) error {
 					} else {
 						t.Fail(v.description)
 					}
-					t.Diagnostic(err.Error())
+					t.YAML(map[string]string{"error": err.Error()})
 				}
 			} else {
 				if e, ok := err.(*rfc2119.Error); ok {
 					t.Ok(e.Level < complianceLevel, v.description)
 				} else {
 					t.Fail(v.description)
 				}
-				t.Diagnostic(err.Error())
+				t.YAML(map[string]string{"error": err.Error()})
 			}
 		}
 	}

validation/create.go

@@ -50,18 +50,24 @@ func main() {
 		r.SetID(c.id)
 		stderr, err := r.Create()
 		t.Ok((err == nil) == c.errExpected, c.err.(*specerror.Error).Err.Err.Error())
-		t.Diagnostic(c.err.(*specerror.Error).Err.Reference)
+		diagnostic := map[string]string{
+			"reference": c.err.(*specerror.Error).Err.Reference,
+		}
 		if err != nil {
-			t.Diagnostic(err.Error())
+			diagnostic["error"] = err.Error()
 		}
 		if len(stderr) > 0 {
-			t.Diagnostic(string(stderr))
+			diagnostic["stderr"] = string(stderr)
 		}
+		t.YAML(diagnostic)
 
 		if err == nil {
 			state, _ := r.State()
-			t.Ok(state.ID == c.id, "")
-			t.Diagnosticf("container PID: %d, state ID: %d", c.id, state.ID)
+			t.Ok(state.ID == c.id, "'state' MUST return the state of a container")
+			t.YAML(map[string]string{
+				"container ID": c.id,
+				"state ID":     state.ID,
+			})
 		}
 	}
 

validation/process_capabilities.go

@@ -9,7 +9,7 @@ import (
 
 func main() {
 	if "linux" != runtime.GOOS {
-		util.Skip("linux-specific process.capabilities test", runtime.GOOS)
+		util.Skip("linux-specific process.capabilities test", map[string]string{"OS": runtime.GOOS})
 		os.Exit(0)
 	}
 

validation/root_readonly_true.go

@@ -9,7 +9,7 @@ import (
 
 func main() {
 	if "windows" == runtime.GOOS {
-		util.Skip("non-Windows root.readonly test", runtime.GOOS)
+		util.Skip("non-Windows root.readonly test", map[string]string{"OS": runtime.GOOS})
 		os.Exit(0)
 	}
 

validation/util/container.go

@@ -12,6 +12,7 @@ import (
 
 	rspecs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
+	"github.com/satori/go.uuid"
 )
 
 // Runtime represents the basic requirement of a container runtime
@@ -63,12 +64,13 @@ func (r *Runtime) Create() (stderr []byte, err error) {
 	//	}
 	cmd := exec.Command(r.RuntimeCommand, args...)
 	cmd.Dir = r.BundleDir
-	r.stdout, err = os.OpenFile(filepath.Join(r.BundleDir, fmt.Sprintf("stdout-%s", r.ID)), os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600)
+	id := uuid.NewV4().String()
+	r.stdout, err = os.OpenFile(filepath.Join(r.BundleDir, fmt.Sprintf("stdout-%s", id)), os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600)
 	if err != nil {
 		return []byte(""), err
 	}
 	cmd.Stdout = r.stdout
-	r.stderr, err = os.OpenFile(filepath.Join(r.BundleDir, fmt.Sprintf("stderr-%s", r.ID)), os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600)
+	r.stderr, err = os.OpenFile(filepath.Join(r.BundleDir, fmt.Sprintf("stderr-%s", id)), os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600)
 	if err != nil {
 		return []byte(""), err
 	}

validation/util/test.go

@@ -42,12 +42,12 @@ func Fatal(err error) {
 }
 
 // Skip skips a full TAP suite.
-func Skip(message string, diagnostic string) {
+func Skip(message string, diagnostic interface{}) {
 	t := tap.New()
 	t.Header(1)
 	t.Skip(1, message)
-	if diagnostic != "" {
-		t.Diagnostic(diagnostic)
+	if diagnostic != nil {
+		t.YAML(diagnostic)
 	}
 }