From cf9bb3c2e83d737c0b25957966ff331e97398a90 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Fri, 24 Oct 2025 17:26:18 +0900 Subject: [PATCH 1/6] Support CodeceptJS profile --- launchable/test_runners/codeceptjs.py | 28 ++++ tests/test_runners/test_codeceptjs.py | 198 ++++++++++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100644 launchable/test_runners/codeceptjs.py create mode 100644 tests/test_runners/test_codeceptjs.py diff --git a/launchable/test_runners/codeceptjs.py b/launchable/test_runners/codeceptjs.py new file mode 100644 index 000000000..199a50b62 --- /dev/null +++ b/launchable/test_runners/codeceptjs.py @@ -0,0 +1,28 @@ +import json +from typing import List + +import click + +from ..testpath import TestPath +from . import launchable + + +@launchable.subset +def subset(client): + def handler(output: List[TestPath], rests: List[TestPath]): + if client.rest: + with open(client.rest, "w+", encoding="utf-8") as f: + f.write(json.dumps({"tests": [client.formatter(t) for t in rests]})) + if output: + click.echo(json.dumps({"tests": [client.formatter(t) for t in output]})) + + # read lines as test file names + for t in client.stdin(): + client.test_path(t.rstrip("\n")) + client.output_handler = handler + + client.run() + + +record_tests = launchable.CommonRecordTestImpls(__name__).report_files() +split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() diff --git a/tests/test_runners/test_codeceptjs.py b/tests/test_runners/test_codeceptjs.py new file mode 100644 index 000000000..aea6bc027 --- /dev/null +++ b/tests/test_runners/test_codeceptjs.py @@ -0,0 +1,198 @@ +import json +import os +import tempfile +from pathlib import Path +from unittest import mock + +import responses # type: ignore + +from launchable.utils.http_client import get_base_url +from tests.cli_test_case import CliTestCase + + +class CodeceptjsTest(CliTestCase): + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset(self): + """Test basic subset functionality with multiple test files""" + pipe = "test/example_test.js\ntest/login_test.js\n" + + # Mock the subset API response to return test paths + mock_response = { + "testPaths": [ + [{"type": "file", "name": "test/example_test.js"}], + [{"type": "file", "name": "test/login_test.js"}], + ], + "rest": [], + "subsettingId": 456, + } + + responses.replace( + responses.POST, + "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace + ), + json=mock_response, + status=200, + ) + + result = self.cli( + "subset", + "--target", + "10%", + "--session", + self.session, + "codeceptjs", + input=pipe, + ) + self.assert_success(result) + + # Verify the output is valid JSON + output = result.output.strip() + output_json = json.loads(output) + self.assertEqual(output_json["tests"], ['test/example_test.js', 'test/login_test.js']) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_with_rest(self): + """Test subset functionality with --rest option to save remaining tests""" + pipe = "test/example_test.js\ntest/other_test.js\n" + + # Mock the subset API response with both testPaths and rest + mock_response = { + "testPaths": [ + [{"type": "file", "name": "test/example_test.js"}], + ], + "rest": [ + [{"type": "file", "name": "test/other_test.js"}], + ], + "subsettingId": 456, + } + + responses.replace( + responses.POST, + "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace + ), + json=mock_response, + status=200, + ) + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" + ) as rest_file: + rest_file_path = rest_file.name + + try: + result = self.cli( + "subset", + "--target", + "50%", + "--session", + self.session, + "--rest", + rest_file_path, + "codeceptjs", + input=pipe, + ) + self.assert_success(result) + + # Verify the output is valid JSON with "tests" key + output = result.output.strip() + output_json = json.loads(output) + self.assertEqual(output_json["tests"], ['test/example_test.js']) + + # Verify rest file was created and contains valid JSON + self.assertTrue( + Path(rest_file_path).exists(), "Rest file should be created" + ) + with open(rest_file_path, "r") as f: + rest_json = json.load(f) + self.assertEqual(rest_json["tests"], ['test/other_test.js']) + finally: + # Cleanup + if Path(rest_file_path).exists(): + Path(rest_file_path).unlink() + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_with_single_test(self): + """Test subset functionality with a single test file""" + pipe = "test/single_test.js\n" + + # Mock the subset API response + mock_response = { + "testPaths": [ + [{"type": "file", "name": "test/single_test.js"}], + ], + "rest": [], + "subsettingId": 456, + } + + responses.replace( + responses.POST, + "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace + ), + json=mock_response, + status=200, + ) + + result = self.cli( + "subset", + "--target", + "10%", + "--session", + self.session, + "codeceptjs", + input=pipe, + ) + self.assert_success(result) + + # Verify the output is valid JSON + output = result.output.strip() + output_json = json.loads(output) + self.assertEqual(output_json["tests"], ['test/single_test.js']) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_strips_newlines(self): + """Test that subset properly strips newlines from test paths""" + # Test with various newline formats + pipe = "test/example_test.js\n\ntest/login_test.js\r\ntest/signup_test.js\n" + + # Mock the subset API response + mock_response = { + "testPaths": [ + [{"type": "file", "name": "test/example_test.js"}], + [{"type": "file", "name": "test/login_test.js"}], + [{"type": "file", "name": "test/signup_test.js"}], + ], + "rest": [], + "subsettingId": 456, + } + + responses.replace( + responses.POST, + "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace + ), + json=mock_response, + status=200, + ) + + result = self.cli( + "subset", + "--target", + "10%", + "--session", + self.session, + "codeceptjs", + input=pipe, + ) + self.assert_success(result) + + # Verify the output is valid JSON + output = result.output.strip() + output_json = json.loads(output) + self.assertEqual(output_json["tests"], ['test/example_test.js', 'test/login_test.js', 'test/signup_test.js']) From aa3bbd55ac83c905e54e7049d832b5f328e69433 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Mon, 27 Oct 2025 11:12:41 +0900 Subject: [PATCH 2/6] Add test cases for recording CodeceptJS test results --- tests/data/codeceptjs/codeceptjs-result.xml | 18 ++++++++ tests/data/codeceptjs/record_test_result.json | 45 +++++++++++++++++++ tests/test_runners/test_codeceptjs.py | 30 ++++++++++--- 3 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 tests/data/codeceptjs/codeceptjs-result.xml create mode 100644 tests/data/codeceptjs/record_test_result.json diff --git a/tests/data/codeceptjs/codeceptjs-result.xml b/tests/data/codeceptjs/codeceptjs-result.xml new file mode 100644 index 000000000..c08455332 --- /dev/null +++ b/tests/data/codeceptjs/codeceptjs-result.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + diff --git a/tests/data/codeceptjs/record_test_result.json b/tests/data/codeceptjs/record_test_result.json new file mode 100644 index 000000000..b875c94be --- /dev/null +++ b/tests/data/codeceptjs/record_test_result.json @@ -0,0 +1,45 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "Visit homepage" + }, + { + "type": "testcase", + "name": "Basic Tests: Visit homepage" + } + ], + "duration": 0.194, + "status": 0, + "stdout": "", + "stderr": "expected web application to include \"Test Pagee\"\n\n + expected - actual\n\n -Test Page Welcome\n +Test Pagee\n ", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "Visit form page" + }, + { + "type": "testcase", + "name": "Basic Tests: Visit form page" + } + ], + "duration": 0.127, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + } + ], + "testRunner": "codeceptjs", + "group": "", + "noBuild": false, + "flavors": [], + "testSuite": "" +} diff --git a/tests/test_runners/test_codeceptjs.py b/tests/test_runners/test_codeceptjs.py index aea6bc027..e9796e1b9 100644 --- a/tests/test_runners/test_codeceptjs.py +++ b/tests/test_runners/test_codeceptjs.py @@ -11,6 +11,21 @@ class CodeceptjsTest(CliTestCase): + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_test_codeceptjs(self): + result = self.cli( + "record", + "tests", + "--session", + self.session, + "codeceptjs", + str(self.test_files_dir.joinpath("codeceptjs-result.xml")), + ) + + self.assert_success(result) + self.assert_record_tests_payload("record_test_result.json") + @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_subset(self): @@ -50,7 +65,9 @@ def test_subset(self): # Verify the output is valid JSON output = result.output.strip() output_json = json.loads(output) - self.assertEqual(output_json["tests"], ['test/example_test.js', 'test/login_test.js']) + self.assertEqual( + output_json["tests"], ["test/example_test.js", "test/login_test.js"] + ) @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) @@ -100,7 +117,7 @@ def test_subset_with_rest(self): # Verify the output is valid JSON with "tests" key output = result.output.strip() output_json = json.loads(output) - self.assertEqual(output_json["tests"], ['test/example_test.js']) + self.assertEqual(output_json["tests"], ["test/example_test.js"]) # Verify rest file was created and contains valid JSON self.assertTrue( @@ -108,7 +125,7 @@ def test_subset_with_rest(self): ) with open(rest_file_path, "r") as f: rest_json = json.load(f) - self.assertEqual(rest_json["tests"], ['test/other_test.js']) + self.assertEqual(rest_json["tests"], ["test/other_test.js"]) finally: # Cleanup if Path(rest_file_path).exists(): @@ -152,7 +169,7 @@ def test_subset_with_single_test(self): # Verify the output is valid JSON output = result.output.strip() output_json = json.loads(output) - self.assertEqual(output_json["tests"], ['test/single_test.js']) + self.assertEqual(output_json["tests"], ["test/single_test.js"]) @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) @@ -195,4 +212,7 @@ def test_subset_strips_newlines(self): # Verify the output is valid JSON output = result.output.strip() output_json = json.loads(output) - self.assertEqual(output_json["tests"], ['test/example_test.js', 'test/login_test.js', 'test/signup_test.js']) + self.assertEqual( + output_json["tests"], + ["test/example_test.js", "test/login_test.js", "test/signup_test.js"], + ) From e330ce310d4a1fe5b9c470c8228c1057a0ed71c4 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Mon, 27 Oct 2025 12:24:36 +0900 Subject: [PATCH 3/6] Refactor --- launchable/test_runners/codeceptjs.py | 13 ++++++- launchable/test_runners/file.py | 30 ++-------------- launchable/test_runners/launchable.py | 35 +++++++++++++++++++ tests/data/codeceptjs/record_test_result.json | 16 ++++++--- 4 files changed, 62 insertions(+), 32 deletions(-) diff --git a/launchable/test_runners/codeceptjs.py b/launchable/test_runners/codeceptjs.py index 199a50b62..e25ecdff8 100644 --- a/launchable/test_runners/codeceptjs.py +++ b/launchable/test_runners/codeceptjs.py @@ -24,5 +24,16 @@ def handler(output: List[TestPath], rests: List[TestPath]): client.run() -record_tests = launchable.CommonRecordTestImpls(__name__).report_files() +@click.argument("reports", required=True, nargs=-1) +@launchable.record.tests +def record_tests(client, reports): + client.path_builder = launchable.CommonRecordTestImpls.create_file_path_builder( + client + ) + + for r in reports: + client.report(r) + client.run() + + split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() diff --git a/launchable/test_runners/file.py b/launchable/test_runners/file.py index 3b9a11549..1d7377c86 100644 --- a/launchable/test_runners/file.py +++ b/launchable/test_runners/file.py @@ -2,9 +2,7 @@ # The most bare-bone versions of the test runner support # import click -from junitparser import TestCase, TestSuite # type: ignore -from ..testpath import TestPath from . import launchable @@ -20,31 +18,9 @@ def subset(client): @click.argument('reports', required=True, nargs=-1) @launchable.record.tests def record_tests(client, reports): - def path_builder(case: TestCase, suite: TestSuite, - report_file: str) -> TestPath: - """path builder that puts the file name first, which is consistent with the subset command""" - def find_filename(): - """look for what looks like file names from test reports""" - for e in [case, suite]: - for a in ["file", "filepath"]: - filepath = e._elem.attrib.get(a) - if filepath: - return filepath - return None # failing to find a test name - - filepath = find_filename() - if not filepath: - raise click.ClickException( - "No file name found in %s" % report_file) - - # default test path in `subset` expects to have this file name - test_path = [client.make_file_path_component(filepath)] - if suite.name: - test_path.append({"type": "testsuite", "name": suite.name}) - if case.name: - test_path.append({"type": "testcase", "name": case.name}) - return test_path - client.path_builder = path_builder + client.path_builder = launchable.CommonRecordTestImpls.create_file_path_builder( + client + ) for r in reports: client.report(r) diff --git a/launchable/test_runners/launchable.py b/launchable/test_runners/launchable.py index eb8cb2331..a44da3936 100644 --- a/launchable/test_runners/launchable.py +++ b/launchable/test_runners/launchable.py @@ -4,12 +4,15 @@ import types import click +from junitparser import TestCase, TestSuite # type: ignore from launchable.commands.detect_flakes import detect_flakes as detect_flakes_cmd from launchable.commands.record.tests import tests as record_tests_cmd from launchable.commands.split_subset import split_subset as split_subset_cmd from launchable.commands.subset import subset as subset_cmd +from ..testpath import TestPath + def cmdname(m): """figure out the sub-command name from a test runner function""" @@ -113,6 +116,38 @@ def record_tests(client, source_roots): return wrap(record_tests, record_tests_cmd, self.cmdname) + @staticmethod + def create_file_path_builder(client): + """ + Creates a path builder that puts the file name first, which is consistent with the subset command + """ + + def path_builder( + case: TestCase, suite: TestSuite, report_file: str + ) -> TestPath: + def find_filename(): + """look for what looks like file names from test reports""" + for e in [case, suite]: + for a in ["file", "filepath"]: + filepath = e._elem.attrib.get(a) + if filepath: + return filepath + return None # failing to find a test name + + filepath = find_filename() + if not filepath: + raise click.ClickException("No file name found in %s" % report_file) + + # default test path in `subset` expects to have this file name + test_path = [client.make_file_path_component(filepath)] + if suite.name: + test_path.append({"type": "testsuite", "name": suite.name}) + if case.name: + test_path.append({"type": "testcase", "name": case.name}) + return test_path + + return path_builder + @classmethod def load_report_files(cls, client, source_roots, file_mask="*.xml"): # client type: RecordTests in def launchable.commands.record.tests.tests diff --git a/tests/data/codeceptjs/record_test_result.json b/tests/data/codeceptjs/record_test_result.json index b875c94be..1bec33ec0 100644 --- a/tests/data/codeceptjs/record_test_result.json +++ b/tests/data/codeceptjs/record_test_result.json @@ -4,8 +4,12 @@ "type": "case", "testPath": [ { - "type": "class", - "name": "Visit homepage" + "type": "file", + "name": "/Users/ono-max/src/github.com/cloudbees-oss/smart-tests-integration-examples/codeceptjs/tests/website_test.js" + }, + { + "type": "testsuite", + "name": "Basic Tests" }, { "type": "testcase", @@ -22,8 +26,12 @@ "type": "case", "testPath": [ { - "type": "class", - "name": "Visit form page" + "type": "file", + "name": "/Users/ono-max/src/github.com/cloudbees-oss/smart-tests-integration-examples/codeceptjs/tests/website_test.js" + }, + { + "type": "testsuite", + "name": "Basic Tests" }, { "type": "testcase", From 2c25e50b293c9610d5726ec8b00d5514045648a3 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Mon, 27 Oct 2025 12:37:22 +0900 Subject: [PATCH 4/6] Add comments to clarify output format in subset handler --- launchable/test_runners/codeceptjs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/launchable/test_runners/codeceptjs.py b/launchable/test_runners/codeceptjs.py index e25ecdff8..f53bf32e2 100644 --- a/launchable/test_runners/codeceptjs.py +++ b/launchable/test_runners/codeceptjs.py @@ -10,6 +10,8 @@ @launchable.subset def subset(client): def handler(output: List[TestPath], rests: List[TestPath]): + # The output would be something like this: + # {"tests": ["test/example_test.js", "test/login_test.js"]} if client.rest: with open(client.rest, "w+", encoding="utf-8") as f: f.write(json.dumps({"tests": [client.formatter(t) for t in rests]})) From 4ffd89e5d3a61050069e284597de3cf21cfbd417 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Mon, 27 Oct 2025 12:47:03 +0900 Subject: [PATCH 5/6] Refactor --- launchable/test_runners/codeceptjs.py | 12 +---- launchable/test_runners/file.py | 13 +----- launchable/test_runners/launchable.py | 65 +++++++++++++++------------ 3 files changed, 39 insertions(+), 51 deletions(-) diff --git a/launchable/test_runners/codeceptjs.py b/launchable/test_runners/codeceptjs.py index f53bf32e2..34f686e26 100644 --- a/launchable/test_runners/codeceptjs.py +++ b/launchable/test_runners/codeceptjs.py @@ -26,16 +26,6 @@ def handler(output: List[TestPath], rests: List[TestPath]): client.run() -@click.argument("reports", required=True, nargs=-1) -@launchable.record.tests -def record_tests(client, reports): - client.path_builder = launchable.CommonRecordTestImpls.create_file_path_builder( - client - ) - - for r in reports: - client.report(r) - client.run() - +record_tests = launchable.CommonRecordTestImpls(__name__).file_profile_report_files() split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() diff --git a/launchable/test_runners/file.py b/launchable/test_runners/file.py index 1d7377c86..a9670d40a 100644 --- a/launchable/test_runners/file.py +++ b/launchable/test_runners/file.py @@ -1,7 +1,6 @@ # # The most bare-bone versions of the test runner support # -import click from . import launchable @@ -15,17 +14,7 @@ def subset(client): client.run() -@click.argument('reports', required=True, nargs=-1) -@launchable.record.tests -def record_tests(client, reports): - client.path_builder = launchable.CommonRecordTestImpls.create_file_path_builder( - client - ) - - for r in reports: - client.report(r) - client.run() - +record_tests = launchable.CommonRecordTestImpls(__name__).file_profile_report_files() split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() diff --git a/launchable/test_runners/launchable.py b/launchable/test_runners/launchable.py index a44da3936..df0907e36 100644 --- a/launchable/test_runners/launchable.py +++ b/launchable/test_runners/launchable.py @@ -116,37 +116,46 @@ def record_tests(client, source_roots): return wrap(record_tests, record_tests_cmd, self.cmdname) - @staticmethod - def create_file_path_builder(client): + def file_profile_report_files(self): """ - Creates a path builder that puts the file name first, which is consistent with the subset command + Suitable for test runners that create a directory full of JUnit report files. + + 'record tests' expect JUnit report/XML file names. """ - def path_builder( - case: TestCase, suite: TestSuite, report_file: str - ) -> TestPath: - def find_filename(): - """look for what looks like file names from test reports""" - for e in [case, suite]: - for a in ["file", "filepath"]: - filepath = e._elem.attrib.get(a) - if filepath: - return filepath - return None # failing to find a test name - - filepath = find_filename() - if not filepath: - raise click.ClickException("No file name found in %s" % report_file) - - # default test path in `subset` expects to have this file name - test_path = [client.make_file_path_component(filepath)] - if suite.name: - test_path.append({"type": "testsuite", "name": suite.name}) - if case.name: - test_path.append({"type": "testcase", "name": case.name}) - return test_path - - return path_builder + @click.argument('source_roots', required=True, nargs=-1) + def record_tests(client, source_roots): + def path_builder( + case: TestCase, suite: TestSuite, report_file: str + ) -> TestPath: + def find_filename(): + """look for what looks like file names from test reports""" + for e in [case, suite]: + for a in ["file", "filepath"]: + filepath = e._elem.attrib.get(a) + if filepath: + return filepath + return None # failing to find a test name + + filepath = find_filename() + if not filepath: + raise click.ClickException("No file name found in %s" % report_file) + + # default test path in `subset` expects to have this file name + test_path = [client.make_file_path_component(filepath)] + if suite.name: + test_path.append({"type": "testsuite", "name": suite.name}) + if case.name: + test_path.append({"type": "testcase", "name": case.name}) + return test_path + + client.path_builder = path_builder + + for r in source_roots: + client.report(r) + client.run() + + return wrap(record_tests, record_tests_cmd, self.cmdname) @classmethod def load_report_files(cls, client, source_roots, file_mask="*.xml"): From 4da3bdb40345cf43fef92009e50ea7a5ff6661e5 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Mon, 27 Oct 2025 13:32:00 +0900 Subject: [PATCH 6/6] Fix: Prevent empty test paths from being added to the test path list --- launchable/test_runners/codeceptjs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launchable/test_runners/codeceptjs.py b/launchable/test_runners/codeceptjs.py index 34f686e26..79b42a93b 100644 --- a/launchable/test_runners/codeceptjs.py +++ b/launchable/test_runners/codeceptjs.py @@ -20,7 +20,8 @@ def handler(output: List[TestPath], rests: List[TestPath]): # read lines as test file names for t in client.stdin(): - client.test_path(t.rstrip("\n")) + if t.rstrip("\n"): + client.test_path(t.rstrip("\n")) client.output_handler = handler client.run()