V1 sync

smart_tests/commands/verify.py

@@ -7,13 +7,13 @@
 import click
 
 import smart_tests.args4p.typer as typer
-from smart_tests.utils.commands import Command
-from smart_tests.utils.env_keys import TOKEN_KEY
 from smart_tests.utils.tracking import Tracking, TrackingClient
 
 from .. import args4p
 from ..app import Application
-from ..utils.authentication import get_org_workspace
+from ..utils.authentication import ensure_org_workspace, get_org_workspace
+from ..utils.commands import Command
+from ..utils.env_keys import TOKEN_KEY
 from ..utils.java import get_java_command
 from ..utils.smart_tests_client import SmartTestsClient
 from ..utils.typer_types import emoji
@@ -82,18 +82,8 @@ def verify(app_instance: Application):
     click.echo("Java command: " + repr(java))
     click.echo("smart-tests version: " + repr(version))
 
-    if org is None or workspace is None:
-        msg = (
-            "Could not identify Smart Tests organization/workspace. "
-            "Please confirm if you set SMART_TESTS_TOKEN or SMART_TESTS_ORGANIZATION and SMART_TESTS_WORKSPACE "
-            "environment variables"
-        )
-        tracking_client.send_error_event(
-            event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
-            stack_trace=msg
-        )
-        click.secho(msg, fg='red', err=True)
-        raise typer.Exit(1)
+    # raise an error here after we print out the basic diagnostics if LAUNCHABLE_TOKEN is not set.
+    ensure_org_workspace()
 
     try:
         res = client.request("get", "verification")

smart_tests/test_runners/file.py

@@ -2,19 +2,9 @@
 # The most bare-bone versions of the test runner support
 #
 
-from ..commands.subset import Subset
 from . import smart_tests
 
-
-@smart_tests.subset
-def subset(client: Subset):
-    # read lines as test file names
-    for t in client.stdin():
-        client.test_path(t.rstrip("\n"))
-
-    client.run()
-
-
+subset = smart_tests.CommonSubsetImpls(__name__).scan_stdin()
 record_tests = smart_tests.CommonRecordTestImpls(__name__).file_profile_report_files()
 smart_tests.CommonDetectFlakesImpls(__name__).detect_flakes()
 # split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset()

smart_tests/test_runners/jasmine.py

@@ -27,13 +27,7 @@ def record_tests(
     client.run()
 
 
-@smart_tests.subset
-def subset(client):
-    # read lines as test file names
-    for t in client.stdin():
-        client.test_path(t.rstrip("\n"))
-
-    client.run()
+subset = smart_tests.CommonSubsetImpls(__name__).scan_stdin()
 
 
 class JSONReportParser:

smart_tests/test_runners/karma.py

@@ -0,0 +1,167 @@
+# This runner only supports recording tests
+# For subsetting, use 'ng' test runner instead
+# It's possible to use 'karma' runner for recording, and 'ng' runner for subsetting, for the same test session
+import json
+from typing import Annotated, Dict, Generator, List
+
+import click
+
+from ..args4p import typer
+from ..commands.record.case_event import CaseEvent
+from ..testpath import TestPath
+from . import smart_tests
+
+
+@smart_tests.subset
+def subset(client, _with: Annotated[str | None, typer.Option(
+        '--with', help="Specify 'ng' to use the Angular test runner for subsetting")] = None, ):
+    # TODO: implement the --with ng option
+
+    # read lines as test file names
+    for t in client.stdin():
+        client.test_path(t.rstrip("\n"))
+
+    client.run()
+
+
+@smart_tests.record.tests
+def record_tests(client,
+                 reports: Annotated[List[str], typer.Argument(multiple=True, help="Test report files to process")],
+                 ):
+    client.parse_func = JSONReportParser(client).parse_func
+
+    for r in reports:
+        client.report(r)
+
+    client.run()
+
+
+class JSONReportParser:
+    """
+    Sample Karma report format:
+    {
+      "browsers": {...},
+      "result": {
+        "24461741": [
+          {
+            "fullName": "path/to/spec.ts should do something",
+            "description": "should do something",
+            "id": "spec0",
+            "log": [],
+            "skipped": false,
+            "disabled": false,
+            "pending": false,
+            "success": true,
+            "suite": [
+              "path/to/spec.ts"
+            ],
+            "time": 92,
+            "executedExpectationsCount": 1,
+            "passedExpectations": [...],
+            "properties": null
+          }
+        ]
+      },
+      "summary": {...}
+    }
+    """
+
+    def __init__(self, client):
+        self.client = client
+
+    def parse_func(self, report_file: str) -> Generator[Dict, None, None]:  # type: ignore
+        data: Dict
+        with open(report_file, 'r') as json_file:
+            try:
+                data = json.load(json_file)
+            except Exception:
+                click.echo(
+                    click.style("Error: Failed to load Json report file: {}".format(report_file), fg='red'), err=True)
+                return
+
+        if not self._validate_report_format(data):
+            click.echo(
+                "Error: {} does not appear to be valid Karma report format. "
+                "Make sure you are using karma-json-reporter or a compatible reporter.".format(
+                    report_file), err=True)
+            return
+
+        results = data.get("result", {})
+        for browser_id, specs in results.items():
+            if isinstance(specs, list):
+                for event in self._parse_specs(specs):
+                    yield event
+
+    def _validate_report_format(self, data: Dict) -> bool:
+        if not isinstance(data, dict):
+            return False
+
+        if "result" not in data:
+            return False
+
+        results = data.get("result", {})
+        if not isinstance(results, dict):
+            return False
+
+        for browser_id, specs in results.items():
+            if not isinstance(specs, list):
+                return False
+
+            for spec in specs:
+                if not isinstance(spec, dict):
+                    return False
+                # Check for required fields
+                if "suite" not in spec or "time" not in spec:
+                    return False
+                # Field suite should have at least one element (filename)
+                suite = spec.get("suite", [])
+                if not isinstance(suite, list) or len(suite) == 0:
+                    return False
+
+        return True
+
+    def _parse_specs(self, specs: List[Dict]) -> List[Dict]:
+        events: List[Dict] = []
+
+        for spec in specs:
+            # TODO:
+            # In NextWorld, test filepaths are included in the suite tag
+            # But generally in a Karma test report, a suite tag can be any string
+            # For the time being let's get filepaths from the suite tag,
+            # until we find a standard way to include filepaths in the test reports
+            suite = spec.get("suite", [])
+            filename = suite[0] if suite else ""
+
+            test_path: TestPath = [
+                self.client.make_file_path_component(filename),
+                {"type": "testcase", "name": spec.get("fullName", spec.get("description", ""))}
+            ]
+
+            duration_msec = spec.get("time", 0)
+            status = self._case_event_status_from_spec(spec)
+            stderr = self._parse_stderr(spec)
+
+            events.append(CaseEvent.create(
+                test_path=test_path,
+                duration_secs=duration_msec / 1000 if duration_msec else 0,
+                status=status,
+                stderr=stderr
+            ))
+
+        return events
+
+    def _case_event_status_from_spec(self, spec: Dict) -> int:
+        if spec.get("skipped", False) or spec.get("disabled", False) or spec.get("pending", False):
+            return CaseEvent.TEST_SKIPPED
+
+        if spec.get("success", False):
+            return CaseEvent.TEST_PASSED
+        else:
+            return CaseEvent.TEST_FAILED
+
+    def _parse_stderr(self, spec: Dict) -> str:
+        log_messages = spec.get("log", [])
+        if not log_messages:
+            return ""
+
+        return "\n".join(str(msg) for msg in log_messages if msg)

smart_tests/test_runners/ng.py

@@ -0,0 +1,22 @@
+from . import smart_tests
+
+
+@smart_tests.subset
+def subset(client):
+    """
+    Input format example:
+        src/app/feature/feature.component.spec.ts
+        src/app/service/service.service.spec.ts
+
+    Output format: --include=<path> format that can be passed to ng test
+    Example:
+        --include=src/app/feature/feature.component.spec.ts --include=src/app/service/service.service.spec.ts
+    """
+    for t in client.stdin():
+        path = t.strip()
+        if path:
+            client.test_path(path)
+
+    client.formatter = lambda x: "--include={}".format(x[0]['name'])
+    client.separator = " "
+    client.run()

smart_tests/test_runners/playwright.py

@@ -13,7 +13,6 @@
 from ..args4p.exceptions import BadCmdLineException
 from ..commands.record.case_event import CaseEvent, CaseEventGenerator
 from ..commands.record.tests import RecordTests
-from ..commands.subset import Subset
 from ..testpath import TestPath
 from . import smart_tests
 
@@ -63,13 +62,7 @@ def path_builder(case: TestCase, suite: TestSuite,
     client.run()
 
 
-@smart_tests.subset
-def subset(client: Subset):
-    # read lines as test file names
-    for t in client.stdin():
-        client.test_path(t.rstrip("\n"))
-
-    client.run()
+subset = smart_tests.CommonSubsetImpls(__name__).scan_stdin()
 
 
 class JSONReportParser:

smart_tests/test_runners/prove.py

@@ -7,7 +7,6 @@
 
 from ..args4p.exceptions import BadCmdLineException
 from ..commands.record.tests import RecordTests
-from ..commands.subset import Subset
 from ..testpath import TestPath
 from . import smart_tests
 
@@ -19,13 +18,7 @@ def remove_leading_number_and_dash(input_string: str) -> str:
     return result
 
 
-@smart_tests.subset
-def subset(client: Subset):
-    # read lines as test file names
-    for t in client.stdin():
-        client.test_path(t.rstrip("\n"))
-
-    client.run()
+subset = smart_tests.CommonSubsetImpls(__name__).scan_stdin()
 
 
 @smart_tests.record.tests

smart_tests/test_runners/smart_tests.py

@@ -75,11 +75,27 @@ class CommonSubsetImpls:
     def __init__(self, module_name):
         self.cmdname = cmdname(module_name)
 
-    def scan_files(self, pattern):
+    def scan_stdin(self):
+        """
+        Historical implementation of the files profile that's also used elsewhere.
+        Reads test one line at a time from stdin. Consider this implementation deprecated.
+        Newer test runners are advised to use scan_files() without the pattern argument.
+        """
+        def subset(client):
+            # read lines as test file names
+            for t in client.stdin():
+                client.test_path(t.rstrip("\n"))
+            client.run()
+
+        return wrap(subset, subset_cmd, self.cmdname)
+
+    def scan_files(self, pattern=None):
         """
         Suitable for test runners that use files as unit of tests where file names follow a naming pattern.
 
         :param pattern: file masks that identify test files, such as '*_spec.rb'
+                        for test runners that do not have natural file naming conventions, pass in None,
+                        so that the implementation will refuse to accept directories.
         """
         def subset(
             client: Subset,
@@ -92,6 +108,8 @@ def subset(
             # client type: Optimize in def lauchable.commands.subset.subset
             def parse(fname: str):
                 if os.path.isdir(fname):
+                    if pattern is None:
+                        raise click.UsageError(f'{fname} is a directory, but expecting a file or GLOB')
                     client.scan(fname, '**/' + pattern)
                 elif fname == '@-':
                     # read stdin

smart_tests/test_runners/vitest.py

@@ -4,7 +4,6 @@
 import smart_tests.args4p.typer as typer
 
 from ..commands.record.tests import RecordTests
-from ..commands.subset import Subset
 from . import smart_tests
 
 
@@ -40,10 +39,4 @@ def parse_func(report: str) -> ET.ElementTree:
     smart_tests.CommonRecordTestImpls.load_report_files(client=client, source_roots=reports)
 
 
-@smart_tests.subset
-def subset(client: Subset):
-    # read lines as test file names
-    for t in client.stdin():
-        client.test_path(t.rstrip("\n"))
-
-    client.run()
+subset = smart_tests.CommonSubsetImpls(__name__).scan_stdin()

smart_tests/utils/authentication.py

@@ -10,14 +10,19 @@
 
 
 def get_org_workspace():
+    '''
+    Returns (org,ws) tuple from LAUNCHABLE_TOKEN, or (None,None) if not found.
+    Use ensure_org_workspace() if this is supposed to be an error condition
+    '''
     token = get_token()
     if token:
         try:
             _, user, _ = token.split(":", 2)
             org, workspace = user.split("/", 1)
             return org, workspace
         except ValueError:
-            return None, None
+            click.secho("Invalid value in LAUNCHABLE_TOKEN environment variable.", fg="red")
+            raise typer.Exit(1)
 
     return os.getenv(ORGANIZATION_KEY), os.getenv(WORKSPACE_KEY)