cloudbees-oss · Konboi · Nov 18, 2024 · Nov 14, 2024 · Nov 15, 2024 · Nov 15, 2024
diff --git a/launchable/test_runners/flutter.py b/launchable/test_runners/flutter.py
@@ -0,0 +1,249 @@
+import json
+import pathlib
+from pathlib import Path
+from typing import Dict, Generator, List, Optional
+
+import click
+
+from launchable.commands.record.case_event import CaseEvent
+from launchable.testpath import FilePathNormalizer
+
+from . import launchable
+
+FLUTTER_FILE_EXT = "_test.dart"
+
+FLUTTER_TEST_RESULT_SUCCESS = "success"
+FLUTTER_TEST_RESULT_FAILURE = "error"
+
+
+class TestCase:
+    def __init__(self, id: int, name: str, is_skipped: bool = False):
+        self._id: int = id
+        self._name: str = name
+        self._is_skipped: bool = is_skipped
+        self._status: str = ""
+        self._stdout: str = ""
+        self._stderr: str = ""
+        self._duration_sec: float = 0
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def status(self) -> int:  # status code see: case_event.py
+        if self._is_skipped:
+            return CaseEvent.TEST_SKIPPED
+        elif self._status == FLUTTER_TEST_RESULT_SUCCESS:
+            return CaseEvent.TEST_PASSED
+        elif self._status == FLUTTER_TEST_RESULT_FAILURE:
+            return CaseEvent.TEST_FAILED
+
+        # safe fallback
+        return CaseEvent.TEST_PASSED
+
+    @status.setter
+    def status(self, status: str):
+        self._status = status
+
+    @property
+    def duration(self) -> float:
+        return self._duration_sec
+
+    @duration.setter
+    def duration(self, duration_sec: float):
+        self._duration_sec = duration_sec
+
+    @property
+    def stdout(self) -> str:
+        return self._stdout
+
+    @stdout.setter
+    def stdout(self, stdout: str):
+        self._stdout = stdout
+
+    @property
+    def stderr(self) -> str:
+        return self._stderr
+
+    @stderr.setter
+    def stderr(self, stderr: str):
+        self._stderr = stderr
+
+
+class TestSuite:
+    def __init__(self, id: int, platform: str, path: str):
+        self._id = id
+        self._platform = platform
+        self._path = path
+        self._test_cases: Dict[int, TestCase] = {}
+
+    def _get_test_case(self, id: int) -> Optional[TestCase]:
+        return self._test_cases.get(id)
+
+
+class ReportParser:
+    def __init__(self, file_path_normalizer: FilePathNormalizer):
+        self.file_path_normalizer = file_path_normalizer
+        self._suites: Dict[int, TestSuite] = {}
+
+    def _get_suite(self, suite_id: int) -> Optional[TestSuite]:
+        return self._suites.get(suite_id)
+
+    def _get_test(self, test_id: int) -> Optional[TestCase]:
+        if test_id is None:
+            return None
+
+        for s in self._suites.values():
+            c = s._get_test_case(test_id)
+            if c is not None:
+                return c
+
+        return None
+
+    def _events(self) -> List:
+        events = []
+        for s in self._suites.values():
+            for c in s._test_cases.values():
+                events.append(CaseEvent.create(
+                    test_path=[
+                        {"type": "file", "name": pathlib.Path(self.file_path_normalizer.relativize(s._path)).as_posix()},
+                        {"type": "testcase", "name": c.name}],
+                    duration_secs=c.duration,
+                    status=c.status,
+                    stdout=c.stdout,
+                    stderr=c.stderr,
+                ))
+
+        return events
+
+    def parse_func(self, report_file: str) -> Generator[CaseEvent, None, None]:
+        # TODO: Support cases that include information about `flutter pub get`
+        # see detail: https://github.com/launchableinc/examples/actions/runs/11884312142/job/33112309450
+        if not pathlib.Path(report_file).exists():
+            click.echo(click.style("Error: Report file not found: {}".format(report_file), fg='red'), err=True)
+            return
+
+        with open(report_file, "r") as ndjson:
+            try:
+                for j in ndjson:
+                    if not j.strip():
+                        continue
+                    try:
+                        data = json.loads(j)
+                        self._parse_json(data)
+                    except json.JSONDecodeError:
+                        click.echo(
+                            click.style("Error: Invalid JSON format: {}. Skip load this line".format(j), fg='yellow'), err=True)
+                        continue
+            except Exception as e:
+                click.echo(
+                    click.style("Error: Failed to parse the report file: {} : {}".format(report_file, e), fg='red'), err=True)
+                return
+
+        for event in self._events():
+            yield event
+
+    def _parse_json(self, data: Dict):
+        if not isinstance(data, Dict):
+            # Note: array sometimes comes in but won't use it
+            return
+
+        data_type = data.get("type")
+        if data_type is None:
+            return
+        elif data_type == "suite":
+            suite_data = data.get("suite")
+            if suite_data is None:
+                # it's might be invalid suite data
+                return
+
+            suite_id = suite_data.get("id")
+            if self._get_suite(suite_data.get("id")) is not None:
+                # already recorded
+                return
+
+            self._suites[suite_id] = TestSuite(suite_id, suite_data.get("platform"), suite_data.get("path"))
+        elif data_type == "testStart":
+            test_data = data.get("test")
+
+            if test_data is None:
+                # it's might be invalid test data
+                return
+            if test_data.get("line") is None:
+                # Still set up test case, should skip
+                return
+
+            suite_id = test_data.get("suiteID")
+            suite = self._get_suite(suite_id)
+
+            if suite_id is None or suite is None:
+                click.echo(click.style(
+                    "Warning: Cannot find a parent test suite (id: {}). So won't send test result of {}".format(
+                        suite_id, test_data.get("name")), fg='yellow'), err=True)
+                return
+
+            test_id = test_data.get("id")
+            test = self._get_test(test_id)
+            if test is not None:
+                # already recorded
+                return
+
+            name = test_data.get("name")
+            metadata = test_data.get("metadata", {})
+            is_skipped = metadata.get("skip", False)
+            suite._test_cases[test_id] = TestCase(test_id, name, is_skipped)
+
+        elif data_type == "testDone":
+            test_id = data.get("testID")
+            test = self._get_test(test_id)
+
+            if test is None:
+                return
+
+            test.status = data.get("result", "success")  # safe fallback
+            duration_msec = data.get("time", 0)
+            test.duration = duration_msec / 1000  # to sec
+
+        elif data_type == "error":
+            test_id = data.get("testID")
+            test = self._get_test(test_id)
+            if test is None:
+                click.echo(click.style(
+                    "Warning: Cannot find a test (id: {}). So we skip update stderr".format(test_id), fg='yellow'),
+                    err=True)
+                return
+            test.stderr += ("\n" if test.stderr else "") + data.get("error", "")
+
+        elif data_type == "print":
+            # It's difficult to identify the "Retry" case because Flutter reports it with the same test ID
+            # So we won't handle it at the moment.
+            test_id = data.get("testID")
+            test = self._get_test(test_id)
+            if test is None:
+                click.echo(click.style(
+                    "Warning: Cannot find a test (id: {}). So we skip update stdout".format(test_id), fg='yellow'),
+                    err=True)
+                return
+
+            test.stdout += ("\n" if test.stdout else "") + data.get("message", "")
+
+        else:
+            return
+
+
+@click.argument('reports', required=True, nargs=-1)
+@launchable.record.tests
+def record_tests(client, reports):
+    file_path_normalizer = FilePathNormalizer(base_path=client.base_path, no_base_path_inference=client.no_base_path_inference)
+    client.parse_func = ReportParser(file_path_normalizer).parse_func
+
+    launchable.CommonRecordTestImpls.load_report_files(client=client, source_roots=reports)
+
+
+subset = launchable.CommonSubsetImpls(__name__).scan_files('*.dart')
+split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset()
diff --git a/tests/data/flutter/record_test_result.json b/tests/data/flutter/record_test_result.json
@@ -0,0 +1,61 @@
+{
+  "events": [
+    {
+      "type": "case",
+      "testPath": [
+        {
+          "type": "file",
+          "name": "test/skip_widget_test.dart"
+        },
+        {
+          "type": "testcase",
+          "name": "Counter increments smoke skip test"
+        }
+      ],
+      "duration": 1.562,
+      "status": 2,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    },
+    {
+      "type": "case",
+      "testPath": [
+        {
+          "type": "file",
+          "name": "test/failure_widget_test.dart"
+        },
+        {
+          "type": "testcase",
+          "name": "Counter increments smoke failure test"
+        }
+      ],
+      "duration": 2.046,
+      "status": 0,
+      "stdout": "\u2550\u2550\u2561 EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK \u255e\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\nThe following TestFailure was thrown running a test:\nExpected: exactly one matching candidate\n  Actual: _TextWidgetFinder:<Found 0 widgets with text \"2\": []>\n   Which: means none were found but one was expected\n\nWhen the exception was thrown, this was the stack:\n#4      main.<anonymous closure> (file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart:30:5)\n<asynchronous suspension>\n#5      testWidgets.<anonymous closure>.<anonymous closure> (package:flutter_test/src/widget_tester.dart:189:15)\n<asynchronous suspension>\n#6      TestWidgetsFlutterBinding._runTestBody (package:flutter_test/src/binding.dart:1032:5)\n<asynchronous suspension>\n<asynchronous suspension>\n(elided one frame from package:stack_trace)\n\nThis was caught by the test expectation on the following line:\n  file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart line 30\nThe test description was:\n  Counter increments smoke failure test\n\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550",
+      "stderr": "Test failed. See exception logs above.\nThe test description was: Counter increments smoke failure test",
+      "data": null
+    },
+    {
+      "type": "case",
+      "testPath": [
+        {
+          "type": "file",
+          "name": "test/widget_test.dart"
+        },
+        {
+          "type": "testcase",
+          "name": "Counter increments smoke test"
+        }
+      ],
+      "duration": 1.998,
+      "status": 1,
+      "stdout": "",
+      "stderr": "",
+      "data": null
+    }
+  ],
+  "testRunner": "flutter",
+  "group": "",
+  "noBuild": false
+}
diff --git a/tests/data/flutter/report.json b/tests/data/flutter/report.json
@@ -0,0 +1,29 @@
+{"protocolVersion":"0.1.1","runnerVersion":"1.25.7","pid":30535,"type":"start","time":0}
+{"suite":{"id":0,"platform":"vm","path":"/Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/skip_widget_test.dart"},"type":"suite","time":0}
+{"test":{"id":1,"name":"loading /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/skip_widget_test.dart","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":0}
+{"suite":{"id":2,"platform":"vm","path":"/Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart"},"type":"suite","time":2}
+{"test":{"id":3,"name":"loading /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart","suiteID":2,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":2}
+{"suite":{"id":4,"platform":"vm","path":"/Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/widget_test.dart"},"type":"suite","time":3}
+{"test":{"id":5,"name":"loading /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/widget_test.dart","suiteID":4,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":3}
+{"count":3,"time":3,"type":"allSuites"}
+
+[{"event":"test.startedProcess","params":{"vmServiceUri":null,"observatoryUri":null}}]
+
+[{"event":"test.startedProcess","params":{"vmServiceUri":null,"observatoryUri":null}}]
+
+[{"event":"test.startedProcess","params":{"vmServiceUri":null,"observatoryUri":null}}]
+{"testID":5,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1558}
+{"group":{"id":6,"suiteID":4,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":null,"column":null,"url":null},"type":"group","time":1560}
+{"test":{"id":7,"name":"Counter increments smoke test","suiteID":4,"groupIDs":[6],"metadata":{"skip":false,"skipReason":null},"line":171,"column":5,"url":"package:flutter_test/src/widget_tester.dart","root_line":14,"root_column":3,"root_url":"file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/widget_test.dart"},"type":"testStart","time":1560}
+{"testID":3,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1561}
+{"group":{"id":8,"suiteID":2,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":null,"column":null,"url":null},"type":"group","time":1561}
+{"test":{"id":9,"name":"Counter increments smoke failure test","suiteID":2,"groupIDs":[8],"metadata":{"skip":false,"skipReason":null},"line":171,"column":5,"url":"package:flutter_test/src/widget_tester.dart","root_line":14,"root_column":3,"root_url":"file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart"},"type":"testStart","time":1561}
+{"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1562}
+{"group":{"id":10,"suiteID":0,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":null,"column":null,"url":null},"type":"group","time":1562}
+{"test":{"id":11,"name":"Counter increments smoke skip test","suiteID":0,"groupIDs":[10],"metadata":{"skip":true,"skipReason":null},"line":171,"column":5,"url":"package:flutter_test/src/widget_tester.dart","root_line":14,"root_column":3,"root_url":"file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/skip_widget_test.dart"},"type":"testStart","time":1562}
+{"testID":11,"result":"success","skipped":true,"hidden":false,"type":"testDone","time":1562}
+{"testID":7,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":1998}
+{"testID":9,"messageType":"print","message":"══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════\nThe following TestFailure was thrown running a test:\nExpected: exactly one matching candidate\n  Actual: _TextWidgetFinder:<Found 0 widgets with text \"2\": []>\n   Which: means none were found but one was expected\n\nWhen the exception was thrown, this was the stack:\n#4      main.<anonymous closure> (file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart:30:5)\n<asynchronous suspension>\n#5      testWidgets.<anonymous closure>.<anonymous closure> (package:flutter_test/src/widget_tester.dart:189:15)\n<asynchronous suspension>\n#6      TestWidgetsFlutterBinding._runTestBody (package:flutter_test/src/binding.dart:1032:5)\n<asynchronous suspension>\n<asynchronous suspension>\n(elided one frame from package:stack_trace)\n\nThis was caught by the test expectation on the following line:\n  file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart line 30\nThe test description was:\n  Counter increments smoke failure test\n════════════════════════════════════════════════════════════════════════════════════════════════════","type":"print","time":2042}
+{"testID":9,"error":"Test failed. See exception logs above.\nThe test description was: Counter increments smoke failure test","stackTrace":"","isFailure":false,"type":"error","time":2044}
+{"testID":9,"result":"error","skipped":false,"hidden":false,"type":"testDone","time":2046}
+{"success":false,"type":"done","time":2052}