From 5a1c2876c667c919b3f401f41934ca196ea9444d Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Tue, 16 Sep 2025 13:12:23 +0900 Subject: [PATCH 1/2] Rename `retry flake-detection` to the `detect-flake` command --- launchable/__main__.py | 4 +- .../flake_detection.py => detect_flakes.py} | 16 +++--- launchable/commands/retry/__init__.py | 13 ----- launchable/test_runners/bazel.py | 2 +- launchable/test_runners/file.py | 2 +- launchable/test_runners/launchable.py | 10 ++-- launchable/test_runners/raw.py | 2 +- launchable/test_runners/rspec.py | 1 + launchable/utils/commands.py | 2 +- ...ake_detection.py => test_detect_flakes.py} | 49 ++++++++++++++----- 10 files changed, 57 insertions(+), 44 deletions(-) rename launchable/commands/{retry/flake_detection.py => detect_flakes.py} (87%) delete mode 100644 launchable/commands/retry/__init__.py rename tests/commands/{test_flake_detection.py => test_detect_flakes.py} (63%) diff --git a/launchable/__main__.py b/launchable/__main__.py index 3128cde5c..34d8b5fa8 100644 --- a/launchable/__main__.py +++ b/launchable/__main__.py @@ -10,9 +10,9 @@ from launchable.app import Application from .commands.compare import compare +from .commands.detect_flakes import detect_flakes from .commands.inspect import inspect from .commands.record import record -from .commands.retry import retry from .commands.split_subset import split_subset from .commands.stats import stats from .commands.subset import subset @@ -92,7 +92,7 @@ def main(ctx, log_level, plugin_dir, dry_run, skip_cert_verification): main.add_command(inspect) main.add_command(stats) main.add_command(compare) -main.add_command(retry) +main.add_command(detect_flakes, "detect-flakes") if __name__ == '__main__': main() diff --git a/launchable/commands/retry/flake_detection.py b/launchable/commands/detect_flakes.py similarity index 87% rename from launchable/commands/retry/flake_detection.py rename to launchable/commands/detect_flakes.py index 8b225913c..3fcad9321 100644 --- a/launchable/commands/retry/flake_detection.py +++ b/launchable/commands/detect_flakes.py @@ -11,7 +11,7 @@ from launchable.utils.launchable_client import LaunchableClient from launchable.utils.tracking import Tracking, TrackingClient -from ...utils.commands import Command +from ..utils.commands import Command @click.group(help="Early flake detection") @@ -23,14 +23,16 @@ required=True ) @click.option( - '--confidence', - help='Confidence level for flake detection', + '--retry-threshold', + 'retry_threshold', + help='Throughness of how "flake" is detected', type=click.Choice(['low', 'medium', 'high'], case_sensitive=False), + default='medium', required=True, ) @click.pass_context -def flake_detection(ctx, confidence, session): - tracking_client = TrackingClient(Command.FLAKE_DETECTION, app=ctx.obj) +def detect_flakes(ctx, retry_threshold, session): + tracking_client = TrackingClient(Command.DETECT_FLAKE, app=ctx.obj) client = LaunchableClient(app=ctx.obj, tracking_client=tracking_client, test_runner=ctx.invoked_subcommand) session_id = None try: @@ -64,9 +66,9 @@ def run(self): try: res = client.request( "get", - "retry/flake-detection", + "detect-flake", params={ - "confidence": confidence.upper(), + "confidence": retry_threshold.upper(), "session-id": os.path.basename(session_id), "test-runner": ctx.invoked_subcommand}) res.raise_for_status() diff --git a/launchable/commands/retry/__init__.py b/launchable/commands/retry/__init__.py deleted file mode 100644 index 7b11e6e7f..000000000 --- a/launchable/commands/retry/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -import click - -from launchable.utils.click import GroupWithAlias - -from .flake_detection import flake_detection - - -@click.group(cls=GroupWithAlias) -def retry(): - pass - - -retry.add_command(flake_detection, 'flake-detection') diff --git a/launchable/test_runners/bazel.py b/launchable/test_runners/bazel.py index a76257d75..372716dd1 100644 --- a/launchable/test_runners/bazel.py +++ b/launchable/test_runners/bazel.py @@ -33,7 +33,7 @@ def subset(client): split_subset = launchable.CommonSplitSubsetImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).split_subset() -launchable.CommonFlakeDetectionImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).flake_detection() +launchable.CommonFlakeDetectionImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).detect_flakes() @click.argument('workspace', required=True) diff --git a/launchable/test_runners/file.py b/launchable/test_runners/file.py index dcfb4e2c5..3b9a11549 100644 --- a/launchable/test_runners/file.py +++ b/launchable/test_runners/file.py @@ -53,4 +53,4 @@ def find_filename(): split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() -launchable.CommonFlakeDetectionImpls(__name__).flake_detection() +launchable.CommonFlakeDetectionImpls(__name__).detect_flakes() diff --git a/launchable/test_runners/launchable.py b/launchable/test_runners/launchable.py index 8635c2936..33d60b71e 100644 --- a/launchable/test_runners/launchable.py +++ b/launchable/test_runners/launchable.py @@ -5,8 +5,8 @@ import click +from launchable.commands.detect_flakes import detect_flakes as detect_flakes_cmd from launchable.commands.record.tests import tests as record_tests_cmd -from launchable.commands.retry.flake_detection import flake_detection as flake_detection_cmd from launchable.commands.split_subset import split_subset as split_subset_cmd from launchable.commands.subset import subset as subset_cmd from launchable.testpath import unparse_test_path @@ -46,7 +46,7 @@ def subset(f): def flake_detection(f): - return wrap(f, flake_detection_cmd) + return wrap(f, detect_flakes_cmd) def split_subset(f): @@ -184,8 +184,8 @@ def __init__( self._formatter = formatter self._separator = seperator - def flake_detection(self): - def flake_detection(client): + def detect_flakes(self): + def detect_flakes(client): if self._formatter: client.formatter = self._formatter @@ -194,4 +194,4 @@ def flake_detection(client): client.run() - return wrap(flake_detection, flake_detection_cmd, self.cmdname) + return wrap(detect_flakes, detect_flakes_cmd, self.cmdname) diff --git a/launchable/test_runners/raw.py b/launchable/test_runners/raw.py index 4ed980769..565946b17 100644 --- a/launchable/test_runners/raw.py +++ b/launchable/test_runners/raw.py @@ -47,7 +47,7 @@ def subset(client, test_path_file): split_subset = launchable.CommonSplitSubsetImpls(__name__, formatter=unparse_test_path, seperator='\n').split_subset() -launchable.CommonFlakeDetectionImpls(__name__).flake_detection() +launchable.CommonFlakeDetectionImpls(__name__).detect_flakes() @click.argument('test_result_files', required=True, type=click.Path(exists=True), nargs=-1) diff --git a/launchable/test_runners/rspec.py b/launchable/test_runners/rspec.py index 15ffcc155..312137c80 100644 --- a/launchable/test_runners/rspec.py +++ b/launchable/test_runners/rspec.py @@ -3,3 +3,4 @@ subset = launchable.CommonSubsetImpls(__name__).scan_files('*_spec.rb') split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset() record_tests = launchable.CommonRecordTestImpls(__name__).report_files() +launchable.CommonFlakeDetectionImpls(__name__).detect_flakes() diff --git a/launchable/utils/commands.py b/launchable/utils/commands.py index bd65a52ba..dde29c98d 100644 --- a/launchable/utils/commands.py +++ b/launchable/utils/commands.py @@ -8,7 +8,7 @@ class Command(Enum): RECORD_SESSION = 'RECORD_SESSION' SUBSET = 'SUBSET' COMMIT = 'COMMIT' - FLAKE_DETECTION = 'FLAKE_DETECTION' + DETECT_FLAKE = 'DETECT_FLAKE' def display_name(self): return self.value.lower().replace('_', ' ') diff --git a/tests/commands/test_flake_detection.py b/tests/commands/test_detect_flakes.py similarity index 63% rename from tests/commands/test_flake_detection.py rename to tests/commands/test_detect_flakes.py index 4589b8c1f..5266c837b 100644 --- a/tests/commands/test_flake_detection.py +++ b/tests/commands/test_detect_flakes.py @@ -7,7 +7,7 @@ from tests.cli_test_case import CliTestCase -class FlakeDetectionTest(CliTestCase): +class DetectFlakeTest(CliTestCase): @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_flake_detection_success(self): @@ -19,16 +19,15 @@ def test_flake_detection_success(self): } responses.add( responses.GET, - f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", json=mock_json_response, status=200, ) result = self.cli( - "retry", - "flake-detection", + "detect-flakes", "--session", self.session, - "--confidence", + "--retry-threshold", "high", "file", mix_stderr=False, @@ -37,22 +36,47 @@ def test_flake_detection_success(self): self.assertIn("test_flaky_1.py", result.stdout) self.assertIn("test_flaky_2.py", result.stdout) + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_flake_detection_without_retry_threshold_success(self): + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_flaky_1.py"}], + [{"type": "file", "name": "test_flaky_2.py"}], + ] + } + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", + json=mock_json_response, + status=200, + ) + result = self.cli( + "detect-flakes", + "--session", + self.session, + "file", + mix_stderr=False, + ) + self.assert_success(result) + self.assertIn("test_flaky_1.py", result.stdout) + self.assertIn("test_flaky_2.py", result.stdout) + @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_flake_detection_no_flakes(self): mock_json_response = {"testPaths": []} responses.add( responses.GET, - f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", json=mock_json_response, status=200, ) result = self.cli( - "retry", - "flake-detection", + "detect-flakes", "--session", self.session, - "--confidence", + "--retry-threshold", "low", "file", mix_stderr=False, @@ -65,15 +89,14 @@ def test_flake_detection_no_flakes(self): def test_flake_detection_api_error(self): responses.add( responses.GET, - f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/retry/flake-detection", + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/detect-flake", status=500, ) result = self.cli( - "retry", - "flake-detection", + "detect-flakes", "--session", self.session, - "--confidence", + "--retry-threshold", "medium", "file", mix_stderr=False, From 55c99e959aef6ecf57ef9041c58c9c6bef04a2b2 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Tue, 16 Sep 2025 17:12:27 +0900 Subject: [PATCH 2/2] Add detailed output for retrying tests in flake detection --- launchable/commands/detect_flakes.py | 4 ++++ launchable/test_runners/launchable.py | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/launchable/commands/detect_flakes.py b/launchable/commands/detect_flakes.py index 3fcad9321..8928975b0 100644 --- a/launchable/commands/detect_flakes.py +++ b/launchable/commands/detect_flakes.py @@ -6,6 +6,7 @@ from launchable.app import Application from launchable.commands.helper import find_or_create_session from launchable.commands.test_path_writer import TestPathWriter +from launchable.testpath import unparse_test_path from launchable.utils.click import ignorable_error from launchable.utils.env_keys import REPORT_ERROR_KEY from launchable.utils.launchable_client import LaunchableClient @@ -75,6 +76,9 @@ def run(self): test_paths = res.json().get("testPaths", []) if test_paths: self.print(test_paths) + click.echo("Trying to retry the following tests:", err=True) + for detail in res.json().get("testDetails", []): + click.echo(f"{detail.get('reason')}: {unparse_test_path(detail.get('fullTestPath'))}", err=True) except Exception as e: tracking_client.send_error_event( event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, diff --git a/launchable/test_runners/launchable.py b/launchable/test_runners/launchable.py index 33d60b71e..eb8cb2331 100644 --- a/launchable/test_runners/launchable.py +++ b/launchable/test_runners/launchable.py @@ -9,7 +9,6 @@ from launchable.commands.record.tests import tests as record_tests_cmd from launchable.commands.split_subset import split_subset as split_subset_cmd from launchable.commands.subset import subset as subset_cmd -from launchable.testpath import unparse_test_path def cmdname(m): @@ -177,7 +176,7 @@ class CommonFlakeDetectionImpls: def __init__( self, module_name, - formatter=unparse_test_path, + formatter=None, seperator="\n", ): self.cmdname = cmdname(module_name)