From a3e6b126baabdd5da87debc253481ed2c94ae6bb Mon Sep 17 00:00:00 2001 From: Gayan Weerakutti Date: Thu, 14 Aug 2025 15:06:53 +0900 Subject: [PATCH 1/4] Add new command 'launchable compare subsets' --- launchable/__main__.py | 2 + launchable/commands/compare/__init__.py | 13 ++ launchable/commands/compare/subsets.py | 43 +++++ .../commands/{inspect => compare}/__init__.py | 0 tests/commands/compare/test_subsets.py | 154 ++++++++++++++++++ 5 files changed, 212 insertions(+) create mode 100644 launchable/commands/compare/__init__.py create mode 100644 launchable/commands/compare/subsets.py rename tests/commands/{inspect => compare}/__init__.py (100%) create mode 100644 tests/commands/compare/test_subsets.py diff --git a/launchable/__main__.py b/launchable/__main__.py index e4307d63b..ef81dfd9a 100644 --- a/launchable/__main__.py +++ b/launchable/__main__.py @@ -9,6 +9,7 @@ from launchable.app import Application +from .commands.compare import compare from .commands.inspect import inspect from .commands.record import record from .commands.split_subset import split_subset @@ -89,6 +90,7 @@ def main(ctx, log_level, plugin_dir, dry_run, skip_cert_verification): main.add_command(verify) main.add_command(inspect) main.add_command(stats) +main.add_command(compare) if __name__ == '__main__': main() diff --git a/launchable/commands/compare/__init__.py b/launchable/commands/compare/__init__.py new file mode 100644 index 000000000..3dc6280ba --- /dev/null +++ b/launchable/commands/compare/__init__.py @@ -0,0 +1,13 @@ +import click + +from launchable.utils.click import GroupWithAlias + +from .subsets import subsets + + +@click.group(cls=GroupWithAlias) +def compare(): + pass + + +compare.add_command(subsets) diff --git a/launchable/commands/compare/subsets.py b/launchable/commands/compare/subsets.py new file mode 100644 index 000000000..1db7b642d --- /dev/null +++ b/launchable/commands/compare/subsets.py @@ -0,0 +1,43 @@ +import click +from tabulate import tabulate + + +def compare_subsets(file_before, file_after): + # Read file_before and map test names to their indices + with open(file_before, 'r') as f: + before_tests = f.read().splitlines() + before_index_map = {test: idx for idx, test in enumerate(before_tests)} + + with open(file_after, 'r') as f: + after_tests = f.read().splitlines() + + # Calculate and store the order difference for each test + changes = [] + for after_idx, test in enumerate(after_tests): + if test in before_index_map: + before_idx = before_index_map[test] + order_diff = after_idx - before_idx + changes.append((test, before_idx + 1, after_idx + 1, order_diff)) + else: + changes.append((test, '-', after_idx + 1, 'NEW')) + + # Sort changes by the absolute value of order change + changes.sort(key=lambda x: (abs(x[3]) if isinstance(x[3], int) else float('inf')), reverse=True) + + # Display results in a tabular format + headers = ["Before", "After", "Order Change", "Test Path"] + rows = [ + (order_before, order_after, f"{order_change:+}" if isinstance(order_change, int) else order_change, test) + for test, order_before, order_after, order_change in changes + ] + click.echo(tabulate(rows, headers=headers, tablefmt="github")) + + +@click.command() +@click.argument('file_before', type=click.Path(exists=True)) +@click.argument('file_after', type=click.Path(exists=True)) +def subsets(file_before, file_after): + """ + Compare two subset files and display changes in test order positions. + """ + compare_subsets(file_before, file_after) diff --git a/tests/commands/inspect/__init__.py b/tests/commands/compare/__init__.py similarity index 100% rename from tests/commands/inspect/__init__.py rename to tests/commands/compare/__init__.py diff --git a/tests/commands/compare/test_subsets.py b/tests/commands/compare/test_subsets.py new file mode 100644 index 000000000..35c71f04c --- /dev/null +++ b/tests/commands/compare/test_subsets.py @@ -0,0 +1,154 @@ +import os +from unittest import mock + +from tests.cli_test_case import CliTestCase + + +class SubsetsTest(CliTestCase): + + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subsets(self): + # Create subset-before.txt + with open("subset-before.txt", "w") as f: + f.write("\n".join([ + "src/test/java/example/DivTest.java", + "src/test/java/example/DB1Test.java", + "src/test/java/example/MulTest.java", + "src/test/java/example/Add2Test.java", + "src/test/java/example/File1Test.java", + "src/test/java/example/File0Test.java", + "src/test/java/example/SubTest.java", + "src/test/java/example/DB0Test.java", + "src/test/java/example/AddTest.java", + ])) + + # Create subset-after.txt + with open("subset-after.txt", "w") as f: + f.write("\n".join([ + "src/test/java/example/Add2Test.java", + "src/test/java/example/MulTest.java", + "src/test/java/example/AddTest.java", + "src/test/java/example/File1Test.java", + "src/test/java/example/DivTest.java", + "src/test/java/example/File0Test.java", + "src/test/java/example/DB1Test.java", + "src/test/java/example/DB0Test.java", + "src/test/java/example/SubTest.java", + ])) + + result = self.cli('compare', 'subsets', "subset-before.txt", "subset-after.txt", mix_stderr=False) + expect = """| Before | After | Order Change | Test Path | +|----------|---------|----------------|--------------------------------------| +| 9 | 3 | -6 | src/test/java/example/AddTest.java | +| 2 | 7 | +5 | src/test/java/example/DB1Test.java | +| 1 | 5 | +4 | src/test/java/example/DivTest.java | +| 4 | 1 | -3 | src/test/java/example/Add2Test.java | +| 7 | 9 | +2 | src/test/java/example/SubTest.java | +| 3 | 2 | -1 | src/test/java/example/MulTest.java | +| 5 | 4 | -1 | src/test/java/example/File1Test.java | +| 6 | 6 | +0 | src/test/java/example/File0Test.java | +| 8 | 8 | +0 | src/test/java/example/DB0Test.java | +""" + + self.assertEqual(result.stdout, expect) + + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subsets_when_new_tests(self): + # Create subset-before.txt + with open("subset-before.txt", "w") as f: + f.write("\n".join([ + "src/test/java/example/SubTest.java", + "src/test/java/example/DivTest.java", + "src/test/java/example/Add2Test.java", + "src/test/java/example/File0Test.java", + "src/test/java/example/AddTest.java", + "src/test/java/example/File1Test.java", + "src/test/java/example/MulTest.java", + "src/test/java/example/DB0Test.java", + "src/test/java/example/DB1Test.java" + ])) + + # Create subset-after.txt (which includes additional test path NewTest.java) + with open("subset-after.txt", "w") as f: + f.write("\n".join([ + "src/test/java/example/NewTest.java", + "src/test/java/example/SubTest.java", + "src/test/java/example/File0Test.java", + "src/test/java/example/DB1Test.java", + "src/test/java/example/DivTest.java", + "src/test/java/example/MulTest.java", + "src/test/java/example/File1Test.java", + "src/test/java/example/DB0Test.java", + "src/test/java/example/Add2Test.java", + "src/test/java/example/AddTest.java" + ])) + + result = self.cli('compare', 'subsets', "subset-before.txt", "subset-after.txt", mix_stderr=False) + expect = """| Before | After | Order Change | Test Path | +|----------|---------|----------------|--------------------------------------| +| - | 1 | NEW | src/test/java/example/NewTest.java | +| 3 | 9 | +6 | src/test/java/example/Add2Test.java | +| 9 | 4 | -5 | src/test/java/example/DB1Test.java | +| 5 | 10 | +5 | src/test/java/example/AddTest.java | +| 2 | 5 | +3 | src/test/java/example/DivTest.java | +| 1 | 2 | +1 | src/test/java/example/SubTest.java | +| 4 | 3 | -1 | src/test/java/example/File0Test.java | +| 7 | 6 | -1 | src/test/java/example/MulTest.java | +| 6 | 7 | +1 | src/test/java/example/File1Test.java | +| 8 | 8 | +0 | src/test/java/example/DB0Test.java | +""" + + self.assertEqual(result.stdout, expect) + + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subsets_when_deleted_tests(self): + # Create subset-before.txt + with open("subset-before.txt", "w") as f: + f.write("\n".join([ + "src/test/java/example/NewTest.java", + "src/test/java/example/SubTest.java", + "src/test/java/example/File0Test.java", + "src/test/java/example/DB1Test.java", + "src/test/java/example/DivTest.java", + "src/test/java/example/MulTest.java", + "src/test/java/example/File1Test.java", + "src/test/java/example/DB0Test.java", + "src/test/java/example/Add2Test.java", + "src/test/java/example/AddTest.java" + ])) + + # Create subset-after.txt (which doesn't include NewTest.java) + with open("subset-after.txt", "w") as f: + f.write("\n".join([ + "src/test/java/example/DB1Test.java", + "src/test/java/example/DB0Test.java", + "src/test/java/example/File1Test.java", + "src/test/java/example/SubTest.java", + "src/test/java/example/AddTest.java", + "src/test/java/example/MulTest.java", + "src/test/java/example/File0Test.java", + "src/test/java/example/Add2Test.java", + "src/test/java/example/DivTest.java" + ])) + + result = self.cli('compare', 'subsets', "subset-before.txt", "subset-after.txt", mix_stderr=False) + expect = """| Before | After | Order Change | Test Path | +|----------|---------|----------------|--------------------------------------| +| 8 | 2 | -6 | src/test/java/example/DB0Test.java | +| 10 | 5 | -5 | src/test/java/example/AddTest.java | +| 7 | 3 | -4 | src/test/java/example/File1Test.java | +| 3 | 7 | +4 | src/test/java/example/File0Test.java | +| 5 | 9 | +4 | src/test/java/example/DivTest.java | +| 4 | 1 | -3 | src/test/java/example/DB1Test.java | +| 2 | 4 | +2 | src/test/java/example/SubTest.java | +| 9 | 8 | -1 | src/test/java/example/Add2Test.java | +| 6 | 6 | +0 | src/test/java/example/MulTest.java | +""" + + self.assertEqual(result.stdout, expect) + + def tearDown(self): + if os.path.exists("subset-before.txt"): + os.remove("subset-before.txt") + if os.path.exists("subset-after.txt"): + os.remove("subset-after.txt") From 3d1fabe1a87addbc90e50dcb1c34c79a648fa738 Mon Sep 17 00:00:00 2001 From: Gayan Weerakutti Date: Thu, 14 Aug 2025 15:54:54 +0900 Subject: [PATCH 2/4] Refactor 'compare subsets' command to display deleted tests --- launchable/commands/compare/subsets.py | 10 ++++++++-- tests/commands/compare/test_subsets.py | 21 +++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/launchable/commands/compare/subsets.py b/launchable/commands/compare/subsets.py index 1db7b642d..90be61215 100644 --- a/launchable/commands/compare/subsets.py +++ b/launchable/commands/compare/subsets.py @@ -3,16 +3,17 @@ def compare_subsets(file_before, file_after): - # Read file_before and map test names to their indices + # Read files and map test paths to their indices with open(file_before, 'r') as f: before_tests = f.read().splitlines() before_index_map = {test: idx for idx, test in enumerate(before_tests)} with open(file_after, 'r') as f: after_tests = f.read().splitlines() + after_index_map = {test: idx for idx, test in enumerate(after_tests)} - # Calculate and store the order difference for each test changes = [] + # Calculate order difference and add each test in file_after to changes for after_idx, test in enumerate(after_tests): if test in before_index_map: before_idx = before_index_map[test] @@ -21,6 +22,11 @@ def compare_subsets(file_before, file_after): else: changes.append((test, '-', after_idx + 1, 'NEW')) + # Add all deleted tests to changes + for before_idx, test in enumerate(before_tests): + if test not in after_index_map: + changes.append((test, before_idx + 1, '-', 'DELETED')) + # Sort changes by the absolute value of order change changes.sort(key=lambda x: (abs(x[3]) if isinstance(x[3], int) else float('inf')), reverse=True) diff --git a/tests/commands/compare/test_subsets.py b/tests/commands/compare/test_subsets.py index 35c71f04c..9a6ffe85b 100644 --- a/tests/commands/compare/test_subsets.py +++ b/tests/commands/compare/test_subsets.py @@ -132,17 +132,18 @@ def test_subsets_when_deleted_tests(self): ])) result = self.cli('compare', 'subsets', "subset-before.txt", "subset-after.txt", mix_stderr=False) - expect = """| Before | After | Order Change | Test Path | + expect = """| Before | After | Order Change | Test Path | |----------|---------|----------------|--------------------------------------| -| 8 | 2 | -6 | src/test/java/example/DB0Test.java | -| 10 | 5 | -5 | src/test/java/example/AddTest.java | -| 7 | 3 | -4 | src/test/java/example/File1Test.java | -| 3 | 7 | +4 | src/test/java/example/File0Test.java | -| 5 | 9 | +4 | src/test/java/example/DivTest.java | -| 4 | 1 | -3 | src/test/java/example/DB1Test.java | -| 2 | 4 | +2 | src/test/java/example/SubTest.java | -| 9 | 8 | -1 | src/test/java/example/Add2Test.java | -| 6 | 6 | +0 | src/test/java/example/MulTest.java | +| 1 | - | DELETED | src/test/java/example/NewTest.java | +| 8 | 2 | -6 | src/test/java/example/DB0Test.java | +| 10 | 5 | -5 | src/test/java/example/AddTest.java | +| 7 | 3 | -4 | src/test/java/example/File1Test.java | +| 3 | 7 | +4 | src/test/java/example/File0Test.java | +| 5 | 9 | +4 | src/test/java/example/DivTest.java | +| 4 | 1 | -3 | src/test/java/example/DB1Test.java | +| 2 | 4 | +2 | src/test/java/example/SubTest.java | +| 9 | 8 | -1 | src/test/java/example/Add2Test.java | +| 6 | 6 | +0 | src/test/java/example/MulTest.java | """ self.assertEqual(result.stdout, expect) From 65699d7b24b30ae04d6092947f5bb774aa739d63 Mon Sep 17 00:00:00 2001 From: Gayan Weerakutti Date: Thu, 14 Aug 2025 16:06:59 +0900 Subject: [PATCH 3/4] Merge methods in commands/compare/subsets.py for readability --- launchable/commands/compare/subsets.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/launchable/commands/compare/subsets.py b/launchable/commands/compare/subsets.py index 90be61215..8117d2930 100644 --- a/launchable/commands/compare/subsets.py +++ b/launchable/commands/compare/subsets.py @@ -2,7 +2,14 @@ from tabulate import tabulate -def compare_subsets(file_before, file_after): +@click.command() +@click.argument('file_before', type=click.Path(exists=True)) +@click.argument('file_after', type=click.Path(exists=True)) +def subsets(file_before, file_after): + """ + Compare two subset files and display changes in test order positions. + """ + # Read files and map test paths to their indices with open(file_before, 'r') as f: before_tests = f.read().splitlines() @@ -37,13 +44,3 @@ def compare_subsets(file_before, file_after): for test, order_before, order_after, order_change in changes ] click.echo(tabulate(rows, headers=headers, tablefmt="github")) - - -@click.command() -@click.argument('file_before', type=click.Path(exists=True)) -@click.argument('file_after', type=click.Path(exists=True)) -def subsets(file_before, file_after): - """ - Compare two subset files and display changes in test order positions. - """ - compare_subsets(file_before, file_after) From 6e8270307b535d1767c918055ea35a78b5367e6f Mon Sep 17 00:00:00 2001 From: Gayan Weerakutti Date: Fri, 15 Aug 2025 12:21:15 +0900 Subject: [PATCH 4/4] Refactor 'compare subsets' to sort results in ASC order --- launchable/commands/compare/subsets.py | 30 ++++++----- tests/commands/compare/test_subsets.py | 70 +++++++++++++------------- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/launchable/commands/compare/subsets.py b/launchable/commands/compare/subsets.py index 8117d2930..d55047256 100644 --- a/launchable/commands/compare/subsets.py +++ b/launchable/commands/compare/subsets.py @@ -1,3 +1,5 @@ +from typing import List, Tuple, Union + import click from tabulate import tabulate @@ -7,7 +9,7 @@ @click.argument('file_after', type=click.Path(exists=True)) def subsets(file_before, file_after): """ - Compare two subset files and display changes in test order positions. + Compare two subset files and display changes in test order positions """ # Read files and map test paths to their indices @@ -19,28 +21,30 @@ def subsets(file_before, file_after): after_tests = f.read().splitlines() after_index_map = {test: idx for idx, test in enumerate(after_tests)} - changes = [] + # List of tuples representing test order changes (before, after, diff, test) + rows: List[Tuple[Union[int, str], Union[int, str], Union[int, str], str]] = [] + # Calculate order difference and add each test in file_after to changes for after_idx, test in enumerate(after_tests): if test in before_index_map: before_idx = before_index_map[test] - order_diff = after_idx - before_idx - changes.append((test, before_idx + 1, after_idx + 1, order_diff)) + diff = after_idx - before_idx + rows.append((before_idx + 1, after_idx + 1, diff, test)) else: - changes.append((test, '-', after_idx + 1, 'NEW')) + rows.append(('-', after_idx + 1, 'NEW', test)) # Add all deleted tests to changes for before_idx, test in enumerate(before_tests): if test not in after_index_map: - changes.append((test, before_idx + 1, '-', 'DELETED')) + rows.append((before_idx + 1, '-', 'DELETED', test)) - # Sort changes by the absolute value of order change - changes.sort(key=lambda x: (abs(x[3]) if isinstance(x[3], int) else float('inf')), reverse=True) + # Sort changes by the order diff + rows.sort(key=lambda x: (0 if isinstance(x[2], str) else 1, x[2])) # Display results in a tabular format - headers = ["Before", "After", "Order Change", "Test Path"] - rows = [ - (order_before, order_after, f"{order_change:+}" if isinstance(order_change, int) else order_change, test) - for test, order_before, order_after, order_change in changes + headers = ["Before", "After", "After - Before", "Test"] + tabular_data = [ + (before, after, f"{diff:+}" if isinstance(diff, int) else diff, test) + for before, after, diff, test in rows ] - click.echo(tabulate(rows, headers=headers, tablefmt="github")) + click.echo(tabulate(tabular_data, headers=headers, tablefmt="github")) diff --git a/tests/commands/compare/test_subsets.py b/tests/commands/compare/test_subsets.py index 9a6ffe85b..df0676bbc 100644 --- a/tests/commands/compare/test_subsets.py +++ b/tests/commands/compare/test_subsets.py @@ -37,17 +37,17 @@ def test_subsets(self): ])) result = self.cli('compare', 'subsets', "subset-before.txt", "subset-after.txt", mix_stderr=False) - expect = """| Before | After | Order Change | Test Path | -|----------|---------|----------------|--------------------------------------| -| 9 | 3 | -6 | src/test/java/example/AddTest.java | -| 2 | 7 | +5 | src/test/java/example/DB1Test.java | -| 1 | 5 | +4 | src/test/java/example/DivTest.java | -| 4 | 1 | -3 | src/test/java/example/Add2Test.java | -| 7 | 9 | +2 | src/test/java/example/SubTest.java | -| 3 | 2 | -1 | src/test/java/example/MulTest.java | -| 5 | 4 | -1 | src/test/java/example/File1Test.java | -| 6 | 6 | +0 | src/test/java/example/File0Test.java | -| 8 | 8 | +0 | src/test/java/example/DB0Test.java | + expect = """| Before | After | After - Before | Test | +|----------|---------|------------------|--------------------------------------| +| 9 | 3 | -6 | src/test/java/example/AddTest.java | +| 4 | 1 | -3 | src/test/java/example/Add2Test.java | +| 3 | 2 | -1 | src/test/java/example/MulTest.java | +| 5 | 4 | -1 | src/test/java/example/File1Test.java | +| 6 | 6 | +0 | src/test/java/example/File0Test.java | +| 8 | 8 | +0 | src/test/java/example/DB0Test.java | +| 7 | 9 | +2 | src/test/java/example/SubTest.java | +| 1 | 5 | +4 | src/test/java/example/DivTest.java | +| 2 | 7 | +5 | src/test/java/example/DB1Test.java | """ self.assertEqual(result.stdout, expect) @@ -84,18 +84,18 @@ def test_subsets_when_new_tests(self): ])) result = self.cli('compare', 'subsets', "subset-before.txt", "subset-after.txt", mix_stderr=False) - expect = """| Before | After | Order Change | Test Path | -|----------|---------|----------------|--------------------------------------| -| - | 1 | NEW | src/test/java/example/NewTest.java | -| 3 | 9 | +6 | src/test/java/example/Add2Test.java | -| 9 | 4 | -5 | src/test/java/example/DB1Test.java | -| 5 | 10 | +5 | src/test/java/example/AddTest.java | -| 2 | 5 | +3 | src/test/java/example/DivTest.java | -| 1 | 2 | +1 | src/test/java/example/SubTest.java | -| 4 | 3 | -1 | src/test/java/example/File0Test.java | -| 7 | 6 | -1 | src/test/java/example/MulTest.java | -| 6 | 7 | +1 | src/test/java/example/File1Test.java | -| 8 | 8 | +0 | src/test/java/example/DB0Test.java | + expect = """| Before | After | After - Before | Test | +|----------|---------|------------------|--------------------------------------| +| - | 1 | NEW | src/test/java/example/NewTest.java | +| 9 | 4 | -5 | src/test/java/example/DB1Test.java | +| 4 | 3 | -1 | src/test/java/example/File0Test.java | +| 7 | 6 | -1 | src/test/java/example/MulTest.java | +| 8 | 8 | +0 | src/test/java/example/DB0Test.java | +| 1 | 2 | +1 | src/test/java/example/SubTest.java | +| 6 | 7 | +1 | src/test/java/example/File1Test.java | +| 2 | 5 | +3 | src/test/java/example/DivTest.java | +| 5 | 10 | +5 | src/test/java/example/AddTest.java | +| 3 | 9 | +6 | src/test/java/example/Add2Test.java | """ self.assertEqual(result.stdout, expect) @@ -132,18 +132,18 @@ def test_subsets_when_deleted_tests(self): ])) result = self.cli('compare', 'subsets', "subset-before.txt", "subset-after.txt", mix_stderr=False) - expect = """| Before | After | Order Change | Test Path | -|----------|---------|----------------|--------------------------------------| -| 1 | - | DELETED | src/test/java/example/NewTest.java | -| 8 | 2 | -6 | src/test/java/example/DB0Test.java | -| 10 | 5 | -5 | src/test/java/example/AddTest.java | -| 7 | 3 | -4 | src/test/java/example/File1Test.java | -| 3 | 7 | +4 | src/test/java/example/File0Test.java | -| 5 | 9 | +4 | src/test/java/example/DivTest.java | -| 4 | 1 | -3 | src/test/java/example/DB1Test.java | -| 2 | 4 | +2 | src/test/java/example/SubTest.java | -| 9 | 8 | -1 | src/test/java/example/Add2Test.java | -| 6 | 6 | +0 | src/test/java/example/MulTest.java | + expect = """| Before | After | After - Before | Test | +|----------|---------|------------------|--------------------------------------| +| 1 | - | DELETED | src/test/java/example/NewTest.java | +| 8 | 2 | -6 | src/test/java/example/DB0Test.java | +| 10 | 5 | -5 | src/test/java/example/AddTest.java | +| 7 | 3 | -4 | src/test/java/example/File1Test.java | +| 4 | 1 | -3 | src/test/java/example/DB1Test.java | +| 9 | 8 | -1 | src/test/java/example/Add2Test.java | +| 6 | 6 | +0 | src/test/java/example/MulTest.java | +| 2 | 4 | +2 | src/test/java/example/SubTest.java | +| 3 | 7 | +4 | src/test/java/example/File0Test.java | +| 5 | 9 | +4 | src/test/java/example/DivTest.java | """ self.assertEqual(result.stdout, expect)