Merge pull request #1162 from cloudbees-oss/AIENG-255

[AIENG-255] capture output from bazel test runs

Author: kohsuke

launchable/test_runners/bazel.py

@@ -1,6 +1,7 @@
 import json
-import os
 import sys
+import xml.etree.ElementTree as ET
+from os import path
 from pathlib import Path
 from typing import Generator, List
 
@@ -61,15 +62,45 @@ def f(case: TestCase, suite: TestSuite, report_file: str) -> TestPath:
 
         # last path component is the target, the rest is package
         # TODO: does this work correctly when on Windows?
-        path = make_test_path(os.path.dirname(pkgNtarget), os.path.basename(pkgNtarget))
+        tp = make_test_path(path.dirname(pkgNtarget), path.basename(pkgNtarget))
 
         # let the normal path building kicks in
-        path.extend(default_path_builder(case, suite, report_file))
-        return path
+        tp.extend(default_path_builder(case, suite, report_file))
+        return tp
 
     client.path_builder = f
     client.check_timestamp = False
 
+    def parse_func(report: str) -> ET.ElementTree:
+        """
+        test result XML generated by Bazel's java_test rule (and possibly others) do not capture system-out/system-err.
+        The whole thing gets captured separately as test.log file in the same directory as test.xml.
+        This limits our ability to do useful things with data, so we go out of our way to capture it.
+        Ideally Bazel should do this. test.log captures the entire output from the whole session, and therefore
+        it is incapable of splitting log between different test classes.
+        """
+        tree = ET.parse(report)
+        root = tree.getroot()
+        for ts in root.findall('testsuite'):
+            # be defensive -- future/non-Java test rules might capture those
+            if ts.findtext('system-out', '') == "" and ts.findtext('system-err', '') == "":
+                logfile = path.join(path.dirname(report), 'test.log')
+                if path.exists(logfile):
+                    print("Found {}".format(logfile))
+                    for x in ts.findall('system-out'):
+                        ts.remove(x)
+                    with open(logfile, 'r', encoding='utf-8', errors='replace') as log_file:
+                        log = log_file.read()
+
+                    for c in ts.findall('testcase'):
+                        if c.findtext('system-out', '') == "" and c.findtext('system-err', '') == "":
+                            system_out = ET.SubElement(c, 'system-out')
+                            system_out.text = log
+
+        return tree
+
+    client.junitxml_parse_func = parse_func
+
     if build_event_json_files:
         for l in parse_build_event_json(build_event_json_files):
             if l is None:

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test1/test.log

@@ -0,0 +1 @@
+Dummy standard output

tests/data/bazel/bazel-testlogs/src/test/java/com/ninjinkun/mylib_test1/test.xml

@@ -1,7 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <testsuites>
-  <testsuite name='com.ninjinkun.MyLibTest1' timestamp='2020-12-16T03:11:43.526Z' hostname='localhost' tests='1' failures='0' errors='0' time='2.02' package='' id='0'>
+  <testsuite name='com.ninjinkun.MyLibTest1' timestamp='2020-12-16T03:11:43.526Z' hostname='localhost' tests='1' failures='1' errors='0' time='2.02' package='' id='0'>
     <properties />
-    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest1' time='2.02' />
+    <testcase name='testFibonacci' classname='com.ninjinkun.MyLibTest1' time='2.02'>
+      <failure message='Yo!' type='java.lang.Exception'>java.lang.Exception: Yo!
+        at somewhere(Somewhere.java:33)</failure></testcase>
     <system-out />
     <system-err /></testsuite></testsuites>

tests/data/bazel/record_test_result.json

@@ -23,9 +23,9 @@
         { "type": "testcase", "name": "testFibonacci" }
       ],
       "duration": 2.02,
-      "status": 1,
-      "stdout": "",
-      "stderr": "",
+      "status": 0,
+      "stdout": "Dummy standard output\n",
+      "stderr": "java.lang.Exception: Yo!\n        at somewhere(Somewhere.java:33)",
       "data": null
     },
     {

tests/data/bazel/record_test_with_build_event_json_result.json

@@ -9,9 +9,9 @@
         { "type": "testcase", "name": "testFibonacci" }
       ],
       "duration": 2.02,
-      "status": 1,
-      "stdout": "",
-      "stderr": "",
+      "status": 0,
+      "stdout": "Dummy standard output\n",
+      "stderr": "java.lang.Exception: Yo!\n        at somewhere(Somewhere.java:33)",
       "data": null
     },
     {

tests/data/bazel/record_test_with_multiple_build_event_json_result.json

@@ -9,9 +9,9 @@
         { "type": "testcase", "name": "testFibonacci" }
       ],
       "duration": 2.02,
-      "status": 1,
-      "stdout": "",
-      "stderr": "",
+      "status": 0,
+      "stdout": "Dummy standard output\n",
+      "stderr": "java.lang.Exception: Yo!\n        at somewhere(Somewhere.java:33)",
       "data": null
     },
     {