diff --git a/launchable/test_runners/codeceptjs.py b/launchable/test_runners/codeceptjs.py new file mode 100644 index 000000000..79b42a93b --- /dev/null +++ b/launchable/test_runners/codeceptjs.py @@ -0,0 +1,32 @@ +import json +from typing import List + +import click + +from ..testpath import TestPath +from . import launchable + + +@launchable.subset +def subset(client): + def handler(output: List[TestPath], rests: List[TestPath]): + # The output would be something like this: + # {"tests": ["test/example_test.js", "test/login_test.js"]} + if client.rest: + with open(client.rest, "w+", encoding="utf-8") as f: + f.write(json.dumps({"tests": [client.formatter(t) for t in rests]})) + if output: + click.echo(json.dumps({"tests": [client.formatter(t) for t in output]})) + + # read lines as test file names + for t in client.stdin(): + if t.rstrip("\n"): + client.test_path(t.rstrip("\n")) + client.output_handler = handler + + client.run() + + +record_tests = launchable.CommonRecordTestImpls(__name__).file_profile_report_files() + +split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() diff --git a/launchable/test_runners/file.py b/launchable/test_runners/file.py index 3b9a11549..a9670d40a 100644 --- a/launchable/test_runners/file.py +++ b/launchable/test_runners/file.py @@ -1,10 +1,7 @@ # # The most bare-bone versions of the test runner support # -import click -from junitparser import TestCase, TestSuite # type: ignore -from ..testpath import TestPath from . import launchable @@ -17,39 +14,7 @@ def subset(client): client.run() -@click.argument('reports', required=True, nargs=-1) -@launchable.record.tests -def record_tests(client, reports): - def path_builder(case: TestCase, suite: TestSuite, - report_file: str) -> TestPath: - """path builder that puts the file name first, which is consistent with the subset command""" - def find_filename(): - """look for what looks like file names from test reports""" - for e in [case, suite]: - for a in ["file", "filepath"]: - filepath = e._elem.attrib.get(a) - if filepath: - return filepath - return None # failing to find a test name - - filepath = find_filename() - if not filepath: - raise click.ClickException( - "No file name found in %s" % report_file) - - # default test path in `subset` expects to have this file name - test_path = [client.make_file_path_component(filepath)] - if suite.name: - test_path.append({"type": "testsuite", "name": suite.name}) - if case.name: - test_path.append({"type": "testcase", "name": case.name}) - return test_path - client.path_builder = path_builder - - for r in reports: - client.report(r) - client.run() - +record_tests = launchable.CommonRecordTestImpls(__name__).file_profile_report_files() split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() diff --git a/launchable/test_runners/launchable.py b/launchable/test_runners/launchable.py index eb8cb2331..df0907e36 100644 --- a/launchable/test_runners/launchable.py +++ b/launchable/test_runners/launchable.py @@ -4,12 +4,15 @@ import types import click +from junitparser import TestCase, TestSuite # type: ignore from launchable.commands.detect_flakes import detect_flakes as detect_flakes_cmd from launchable.commands.record.tests import tests as record_tests_cmd from launchable.commands.split_subset import split_subset as split_subset_cmd from launchable.commands.subset import subset as subset_cmd +from ..testpath import TestPath + def cmdname(m): """figure out the sub-command name from a test runner function""" @@ -113,6 +116,47 @@ def record_tests(client, source_roots): return wrap(record_tests, record_tests_cmd, self.cmdname) + def file_profile_report_files(self): + """ + Suitable for test runners that create a directory full of JUnit report files. + + 'record tests' expect JUnit report/XML file names. + """ + + @click.argument('source_roots', required=True, nargs=-1) + def record_tests(client, source_roots): + def path_builder( + case: TestCase, suite: TestSuite, report_file: str + ) -> TestPath: + def find_filename(): + """look for what looks like file names from test reports""" + for e in [case, suite]: + for a in ["file", "filepath"]: + filepath = e._elem.attrib.get(a) + if filepath: + return filepath + return None # failing to find a test name + + filepath = find_filename() + if not filepath: + raise click.ClickException("No file name found in %s" % report_file) + + # default test path in `subset` expects to have this file name + test_path = [client.make_file_path_component(filepath)] + if suite.name: + test_path.append({"type": "testsuite", "name": suite.name}) + if case.name: + test_path.append({"type": "testcase", "name": case.name}) + return test_path + + client.path_builder = path_builder + + for r in source_roots: + client.report(r) + client.run() + + return wrap(record_tests, record_tests_cmd, self.cmdname) + @classmethod def load_report_files(cls, client, source_roots, file_mask="*.xml"): # client type: RecordTests in def launchable.commands.record.tests.tests diff --git a/tests/data/codeceptjs/codeceptjs-result.xml b/tests/data/codeceptjs/codeceptjs-result.xml new file mode 100644 index 000000000..c08455332 --- /dev/null +++ b/tests/data/codeceptjs/codeceptjs-result.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + diff --git a/tests/data/codeceptjs/record_test_result.json b/tests/data/codeceptjs/record_test_result.json new file mode 100644 index 000000000..1bec33ec0 --- /dev/null +++ b/tests/data/codeceptjs/record_test_result.json @@ -0,0 +1,53 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "/Users/ono-max/src/github.com/cloudbees-oss/smart-tests-integration-examples/codeceptjs/tests/website_test.js" + }, + { + "type": "testsuite", + "name": "Basic Tests" + }, + { + "type": "testcase", + "name": "Basic Tests: Visit homepage" + } + ], + "duration": 0.194, + "status": 0, + "stdout": "", + "stderr": "expected web application to include \"Test Pagee\"\n\n + expected - actual\n\n -Test Page Welcome\n +Test Pagee\n ", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "/Users/ono-max/src/github.com/cloudbees-oss/smart-tests-integration-examples/codeceptjs/tests/website_test.js" + }, + { + "type": "testsuite", + "name": "Basic Tests" + }, + { + "type": "testcase", + "name": "Basic Tests: Visit form page" + } + ], + "duration": 0.127, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + } + ], + "testRunner": "codeceptjs", + "group": "", + "noBuild": false, + "flavors": [], + "testSuite": "" +} diff --git a/tests/test_runners/test_codeceptjs.py b/tests/test_runners/test_codeceptjs.py new file mode 100644 index 000000000..e9796e1b9 --- /dev/null +++ b/tests/test_runners/test_codeceptjs.py @@ -0,0 +1,218 @@ +import json +import os +import tempfile +from pathlib import Path +from unittest import mock + +import responses # type: ignore + +from launchable.utils.http_client import get_base_url +from tests.cli_test_case import CliTestCase + + +class CodeceptjsTest(CliTestCase): + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_test_codeceptjs(self): + result = self.cli( + "record", + "tests", + "--session", + self.session, + "codeceptjs", + str(self.test_files_dir.joinpath("codeceptjs-result.xml")), + ) + + self.assert_success(result) + self.assert_record_tests_payload("record_test_result.json") + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset(self): + """Test basic subset functionality with multiple test files""" + pipe = "test/example_test.js\ntest/login_test.js\n" + + # Mock the subset API response to return test paths + mock_response = { + "testPaths": [ + [{"type": "file", "name": "test/example_test.js"}], + [{"type": "file", "name": "test/login_test.js"}], + ], + "rest": [], + "subsettingId": 456, + } + + responses.replace( + responses.POST, + "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace + ), + json=mock_response, + status=200, + ) + + result = self.cli( + "subset", + "--target", + "10%", + "--session", + self.session, + "codeceptjs", + input=pipe, + ) + self.assert_success(result) + + # Verify the output is valid JSON + output = result.output.strip() + output_json = json.loads(output) + self.assertEqual( + output_json["tests"], ["test/example_test.js", "test/login_test.js"] + ) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_with_rest(self): + """Test subset functionality with --rest option to save remaining tests""" + pipe = "test/example_test.js\ntest/other_test.js\n" + + # Mock the subset API response with both testPaths and rest + mock_response = { + "testPaths": [ + [{"type": "file", "name": "test/example_test.js"}], + ], + "rest": [ + [{"type": "file", "name": "test/other_test.js"}], + ], + "subsettingId": 456, + } + + responses.replace( + responses.POST, + "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace + ), + json=mock_response, + status=200, + ) + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" + ) as rest_file: + rest_file_path = rest_file.name + + try: + result = self.cli( + "subset", + "--target", + "50%", + "--session", + self.session, + "--rest", + rest_file_path, + "codeceptjs", + input=pipe, + ) + self.assert_success(result) + + # Verify the output is valid JSON with "tests" key + output = result.output.strip() + output_json = json.loads(output) + self.assertEqual(output_json["tests"], ["test/example_test.js"]) + + # Verify rest file was created and contains valid JSON + self.assertTrue( + Path(rest_file_path).exists(), "Rest file should be created" + ) + with open(rest_file_path, "r") as f: + rest_json = json.load(f) + self.assertEqual(rest_json["tests"], ["test/other_test.js"]) + finally: + # Cleanup + if Path(rest_file_path).exists(): + Path(rest_file_path).unlink() + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_with_single_test(self): + """Test subset functionality with a single test file""" + pipe = "test/single_test.js\n" + + # Mock the subset API response + mock_response = { + "testPaths": [ + [{"type": "file", "name": "test/single_test.js"}], + ], + "rest": [], + "subsettingId": 456, + } + + responses.replace( + responses.POST, + "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace + ), + json=mock_response, + status=200, + ) + + result = self.cli( + "subset", + "--target", + "10%", + "--session", + self.session, + "codeceptjs", + input=pipe, + ) + self.assert_success(result) + + # Verify the output is valid JSON + output = result.output.strip() + output_json = json.loads(output) + self.assertEqual(output_json["tests"], ["test/single_test.js"]) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_strips_newlines(self): + """Test that subset properly strips newlines from test paths""" + # Test with various newline formats + pipe = "test/example_test.js\n\ntest/login_test.js\r\ntest/signup_test.js\n" + + # Mock the subset API response + mock_response = { + "testPaths": [ + [{"type": "file", "name": "test/example_test.js"}], + [{"type": "file", "name": "test/login_test.js"}], + [{"type": "file", "name": "test/signup_test.js"}], + ], + "rest": [], + "subsettingId": 456, + } + + responses.replace( + responses.POST, + "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace + ), + json=mock_response, + status=200, + ) + + result = self.cli( + "subset", + "--target", + "10%", + "--session", + self.session, + "codeceptjs", + input=pipe, + ) + self.assert_success(result) + + # Verify the output is valid JSON + output = result.output.strip() + output_json = json.loads(output) + self.assertEqual( + output_json["tests"], + ["test/example_test.js", "test/login_test.js", "test/signup_test.js"], + )