From d3e915fcc96aa1b36eff59609dd492ccbb4e4f88 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 9 Jun 2025 14:52:15 -0400 Subject: [PATCH 1/2] Fix the bisect command to not cache results --- bench_runner/scripts/bisect.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bench_runner/scripts/bisect.py b/bench_runner/scripts/bisect.py index 4a8fc19..d12e1cc 100644 --- a/bench_runner/scripts/bisect.py +++ b/bench_runner/scripts/bisect.py @@ -45,6 +45,7 @@ def parse_result(benchmark_json: PathLike) -> float: # The name of the benchmark in the JSON file may be different from the one # used to select the benchmark. Therefore, just take the mean of all the # benchmarks in the JSON file. + result._load_contents.cache_clear() r = result.Result.from_arbitrary_filename(benchmark_json) timing_data = r.get_timing_data() return float(np.mean([x.mean() for x in timing_data.values()])) @@ -123,6 +124,12 @@ def _main( bad_timing = get_result(benchmark, pgo, flags, cpython=cpython) log(f"KNOWN BAD ({bad[:7]}): {format_seconds(bad_timing)}") + if good_timing >= bad_timing: + show_log() + raise ValueError( + f"Good timing ({good_timing}) must be less than bad timing ({bad_timing})." + ) + try: with contextlib.chdir(cpython): subprocess.run(["git", "bisect", "start"]) From 8f0f1a000e8489af46a43a6cc29c3ce294034ed2 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 9 Jun 2025 15:11:01 -0400 Subject: [PATCH 2/2] Better encapsulation --- bench_runner/result.py | 4 ++++ bench_runner/scripts/bisect.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bench_runner/result.py b/bench_runner/result.py index 3720e49..f561126 100644 --- a/bench_runner/result.py +++ b/bench_runner/result.py @@ -40,6 +40,10 @@ def _load_contents(filename: Path) -> dict[str, Any]: return json.load(fd) +def clear_contents_cache() -> None: + _load_contents.cache_clear() + + def _clean(string: str) -> str: """ Clean an arbitrary string to be suitable for a filename. diff --git a/bench_runner/scripts/bisect.py b/bench_runner/scripts/bisect.py index d12e1cc..45220db 100644 --- a/bench_runner/scripts/bisect.py +++ b/bench_runner/scripts/bisect.py @@ -45,7 +45,7 @@ def parse_result(benchmark_json: PathLike) -> float: # The name of the benchmark in the JSON file may be different from the one # used to select the benchmark. Therefore, just take the mean of all the # benchmarks in the JSON file. - result._load_contents.cache_clear() + result.clear_contents_cache() r = result.Result.from_arbitrary_filename(benchmark_json) timing_data = r.get_timing_data() return float(np.mean([x.mean() for x in timing_data.values()]))