From a24bf9e725eacbfd2ed5268372c9d79866a36078 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 24 Jan 2026 06:30:18 +0100 Subject: [PATCH 1/4] add search history class --- src/hyperactive/base/_history.py | 163 +++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 src/hyperactive/base/_history.py diff --git a/src/hyperactive/base/_history.py b/src/hyperactive/base/_history.py new file mode 100644 index 00000000..516b5c2c --- /dev/null +++ b/src/hyperactive/base/_history.py @@ -0,0 +1,163 @@ +"""Search history for tracking optimization trials.""" + +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from __future__ import annotations + + +class SearchHistory: + """Container for tracking optimization trial history. + + This class collects data from each evaluation during optimization runs. + History accumulates across multiple optimization runs on the same experiment. + + Attributes + ---------- + trials : list[dict] + List of all recorded trials. Each trial is a dict containing: + - iteration: int, global iteration index + - run_id: int, which optimization run (0-indexed) + - params: dict, the evaluated parameters + - score: float, the evaluation score (raw, not sign-corrected) + - metadata: dict, additional metadata from the experiment + - eval_time: float, evaluation time in seconds + """ + + def __init__(self): + self._trials: list[dict] = [] + self._current_run_id: int = 0 + + def record( + self, + params: dict, + score: float, + metadata: dict | None, + eval_time: float, + ) -> None: + """Record a single trial. + + Parameters + ---------- + params : dict + The evaluated parameters. + score : float + The evaluation score (raw, not sign-corrected). + metadata : dict or None + Additional metadata from the experiment. + eval_time : float + Evaluation time in seconds. + """ + self._trials.append({ + "iteration": len(self._trials), + "run_id": self._current_run_id, + "params": dict(params), + "score": float(score), + "metadata": dict(metadata) if metadata else {}, + "eval_time": float(eval_time), + }) + + def new_run(self) -> None: + """Signal the start of a new optimization run. + + Increments the run_id counter. Subsequent trials will be tagged + with the new run_id. + """ + self._current_run_id += 1 + + def clear(self) -> None: + """Clear all history data and reset run counter.""" + self._trials = [] + self._current_run_id = 0 + + @property + def history(self) -> list[dict]: + """Return all recorded evaluations as a list. + + Returns + ------- + list[dict] + List of all evaluations. Each entry contains iteration, run_id, + params, score, metadata, and eval_time. + """ + return self._trials + + @property + def n_trials(self) -> int: + """Return the total number of recorded trials. + + Returns + ------- + int + Number of trials across all runs. + """ + return len(self._trials) + + @property + def n_runs(self) -> int: + """Return the number of optimization runs. + + Returns + ------- + int + Number of runs (0 if no trials recorded yet). + """ + return self._current_run_id + 1 + + @property + def best_trial(self) -> dict | None: + """Return the trial with the highest score. + + Returns + ------- + dict or None + The trial dict with the highest score, or None if no trials. + """ + if not self._trials: + return None + return max(self._trials, key=lambda t: t["score"]) + + @property + def best_score(self) -> float | None: + """Return the highest score across all trials. + + Returns + ------- + float or None + The highest score, or None if no trials. + """ + best = self.best_trial + return best["score"] if best else None + + @property + def best_params(self) -> dict | None: + """Return the parameters of the best trial. + + Returns + ------- + dict or None + Parameters of the trial with highest score, or None if no trials. + """ + best = self.best_trial + return best["params"] if best else None + + def get_run(self, run_id: int) -> list[dict]: + """Return all trials from a specific run. + + Parameters + ---------- + run_id : int + The run identifier (0-indexed). + + Returns + ------- + list[dict] + List of trials from the specified run. + """ + return [t for t in self._trials if t["run_id"] == run_id] + + def __len__(self) -> int: + """Return the number of trials.""" + return len(self._trials) + + def __repr__(self) -> str: + return f"SearchHistory(n_trials={self.n_trials}, n_runs={self.n_runs})" From dddf407b40aa7d6ade29f824115abac669bf68d0 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 24 Jan 2026 06:38:09 +0100 Subject: [PATCH 2/4] add data namespace to experiment --- src/hyperactive/base/_experiment.py | 58 +++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/hyperactive/base/_experiment.py b/src/hyperactive/base/_experiment.py index 22023bed..dffae99f 100644 --- a/src/hyperactive/base/_experiment.py +++ b/src/hyperactive/base/_experiment.py @@ -2,9 +2,15 @@ # copyright: hyperactive developers, MIT License (see LICENSE file) +from __future__ import annotations + +import time + import numpy as np from skbase.base import BaseObject +from hyperactive.base._history import SearchHistory + class BaseExperiment(BaseObject): """Base class for experiment.""" @@ -21,6 +27,7 @@ class BaseExperiment(BaseObject): def __init__(self): super().__init__() + self._history = SearchHistory() def __call__(self, params): """Score parameters. Same as score call, returns only a first element.""" @@ -77,8 +84,20 @@ def evaluate(self, params): f"Parameters passed to {type(self)}.evaluate do not match: " f"expected {paramnames}, got {list(params.keys())}." ) + + start_time = time.perf_counter() res, metadata = self._evaluate(params) + eval_time = time.perf_counter() - start_time + res = np.float64(res) + + self._history.record( + params=params, + score=res, + metadata=metadata, + eval_time=eval_time, + ) + return res, metadata def _evaluate(self, params): @@ -141,3 +160,42 @@ def score(self, params): metadata = eval_res[1] return sign * value, metadata + + @property + def data(self) -> SearchHistory: + """Access the collected data from optimization runs. + + Tracks all evaluations during optimization. Data accumulates across + multiple optimization runs on the same experiment instance. + + Returns + ------- + SearchHistory + The data object with the following attributes and methods: + + Attributes: + - ``history``: list[dict] - all recorded evaluations + - ``n_trials``: int - total number of trials + - ``n_runs``: int - number of optimization runs + - ``best_trial``: dict | None - trial with highest score + - ``best_score``: float | None - highest score + - ``best_params``: dict | None - parameters of best trial + + Methods: + - ``get_run(run_id)``: get trials from specific run + - ``clear()``: reset all data + - ``new_run()``: signal start of new run (call before each run) + + Examples + -------- + >>> experiment.data.history # all evaluations as list of dicts + >>> experiment.data.best_score # highest score + >>> experiment.data.get_run(0) # evaluations from first run + >>> experiment.data.clear() # reset data + + To convert to a pandas DataFrame:: + + import pandas as pd + df = pd.DataFrame(experiment.data.history) + """ + return self._history From d6983b54ff42fbe9e426160b27039c95cd1142e4 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 24 Jan 2026 06:38:37 +0100 Subject: [PATCH 3/4] add 'SearchHistory' to init --- src/hyperactive/base/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hyperactive/base/__init__.py b/src/hyperactive/base/__init__.py index 35711c54..e21b446a 100644 --- a/src/hyperactive/base/__init__.py +++ b/src/hyperactive/base/__init__.py @@ -2,6 +2,7 @@ # copyright: hyperactive developers, MIT License (see LICENSE file) from hyperactive.base._experiment import BaseExperiment +from hyperactive.base._history import SearchHistory from hyperactive.base._optimizer import BaseOptimizer -__all__ = ["BaseExperiment", "BaseOptimizer"] +__all__ = ["BaseExperiment", "BaseOptimizer", "SearchHistory"] From ad0ebe1b477c7623e5a2d2608c4d38098886a9ee Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 24 Jan 2026 06:50:25 +0100 Subject: [PATCH 4/4] add tests for exp. data history --- src/hyperactive/base/tests/test_history.py | 348 +++++++++++++++++++++ 1 file changed, 348 insertions(+) create mode 100644 src/hyperactive/base/tests/test_history.py diff --git a/src/hyperactive/base/tests/test_history.py b/src/hyperactive/base/tests/test_history.py new file mode 100644 index 00000000..065e0c42 --- /dev/null +++ b/src/hyperactive/base/tests/test_history.py @@ -0,0 +1,348 @@ +"""Tests for SearchHistory and history tracking in experiments.""" + +# copyright: hyperactive developers, MIT License (see LICENSE file) + +import pytest + +from hyperactive.base import SearchHistory + + +class TestSearchHistory: + """Tests for the SearchHistory class.""" + + def test_init_empty(self): + """Test that a new SearchHistory is empty.""" + history = SearchHistory() + assert history.n_trials == 0 + assert history.n_runs == 1 + assert history.history == [] + assert history.best_trial is None + assert history.best_score is None + assert history.best_params is None + + def test_record_single_trial(self): + """Test recording a single trial.""" + history = SearchHistory() + history.record( + params={"x": 1, "y": 2}, + score=0.5, + metadata={"time": 1.0}, + eval_time=0.1, + ) + + assert history.n_trials == 1 + assert history.n_runs == 1 + + trial = history.history[0] + assert trial["iteration"] == 0 + assert trial["run_id"] == 0 + assert trial["params"] == {"x": 1, "y": 2} + assert trial["score"] == 0.5 + assert trial["metadata"] == {"time": 1.0} + assert trial["eval_time"] == 0.1 + + def test_record_multiple_trials(self): + """Test recording multiple trials in one run.""" + history = SearchHistory() + + for i in range(5): + history.record( + params={"x": i}, + score=float(i), + metadata={}, + eval_time=0.1, + ) + + assert history.n_trials == 5 + assert history.n_runs == 1 + + # Check iteration is global + for i, trial in enumerate(history.history): + assert trial["iteration"] == i + assert trial["run_id"] == 0 + + def test_multiple_runs(self): + """Test that run_id increments across multiple runs.""" + history = SearchHistory() + + history.record(params={"x": 1}, score=0.1, metadata={}, eval_time=0.1) + history.record(params={"x": 2}, score=0.2, metadata={}, eval_time=0.1) + + history.new_run() + history.record(params={"x": 3}, score=0.3, metadata={}, eval_time=0.1) + + assert history.n_trials == 3 + assert history.n_runs == 2 + + # Check run_ids + assert history.history[0]["run_id"] == 0 + assert history.history[1]["run_id"] == 0 + assert history.history[2]["run_id"] == 1 + + # Iteration is global + assert history.history[0]["iteration"] == 0 + assert history.history[1]["iteration"] == 1 + assert history.history[2]["iteration"] == 2 + + def test_best_trial(self): + """Test that best_trial returns the trial with highest score.""" + history = SearchHistory() + history.record(params={"x": 1}, score=0.5, metadata={}, eval_time=0.1) + history.record(params={"x": 2}, score=0.9, metadata={}, eval_time=0.1) + history.record(params={"x": 3}, score=0.3, metadata={}, eval_time=0.1) + + best = history.best_trial + assert best["score"] == 0.9 + assert best["params"] == {"x": 2} + assert history.best_score == 0.9 + assert history.best_params == {"x": 2} + + def test_get_run(self): + """Test filtering trials by run_id.""" + history = SearchHistory() + + history.record(params={"x": 1}, score=0.1, metadata={}, eval_time=0.1) + history.record(params={"x": 2}, score=0.2, metadata={}, eval_time=0.1) + + history.new_run() + history.record(params={"x": 3}, score=0.3, metadata={}, eval_time=0.1) + + run0 = history.get_run(0) + run1 = history.get_run(1) + + assert len(run0) == 2 + assert len(run1) == 1 + assert run0[0]["params"] == {"x": 1} + assert run0[1]["params"] == {"x": 2} + assert run1[0]["params"] == {"x": 3} + + def test_clear(self): + """Test that clear resets all history.""" + history = SearchHistory() + history.record(params={"x": 1}, score=0.5, metadata={}, eval_time=0.1) + + history.clear() + + assert history.n_trials == 0 + assert history.n_runs == 1 + assert history.history == [] + + def test_params_are_copied(self): + """Test that recorded params are copied, not referenced.""" + history = SearchHistory() + params = {"x": 1} + history.record(params=params, score=0.5, metadata={}, eval_time=0.1) + + # Modify original + params["x"] = 999 + + # Recorded params should be unchanged + assert history.history[0]["params"]["x"] == 1 + + def test_metadata_none_becomes_empty_dict(self): + """Test that None metadata becomes an empty dict.""" + history = SearchHistory() + history.record(params={"x": 1}, score=0.5, metadata=None, eval_time=0.1) + + assert history.history[0]["metadata"] == {} + + def test_len(self): + """Test __len__ returns number of trials.""" + history = SearchHistory() + assert len(history) == 0 + + history.record(params={"x": 1}, score=0.5, metadata={}, eval_time=0.1) + assert len(history) == 1 + + def test_repr(self): + """Test __repr__ is informative.""" + history = SearchHistory() + history.record(params={"x": 1}, score=0.5, metadata={}, eval_time=0.1) + + repr_str = repr(history) + assert "n_trials=1" in repr_str + assert "n_runs=1" in repr_str + + +class TestExperimentDataIntegration: + """Tests for data tracking in BaseExperiment via accessor pattern.""" + + def test_experiment_has_data_accessor(self): + """Test that BaseExperiment has data accessor.""" + from hyperactive.base import SearchHistory + from hyperactive.experiment.func import FunctionExperiment + + def objective(params): + return params["x"] ** 2 + + exp = FunctionExperiment(objective) + + assert hasattr(exp, "data") + assert isinstance(exp.data, SearchHistory) + assert exp.data.history == [] + assert exp.data.n_trials == 0 + + def test_evaluate_records_data(self): + """Test that evaluate() records trials to data.""" + from hyperactive.experiment.func import FunctionExperiment + + def objective(params): + return params["x"] ** 2 + + exp = FunctionExperiment(objective) + + exp.evaluate({"x": 2}) + exp.evaluate({"x": 3}) + + assert exp.data.n_trials == 2 + assert len(exp.data.history) == 2 + + trial0 = exp.data.history[0] + assert trial0["params"] == {"x": 2} + assert trial0["score"] == 4.0 + assert trial0["iteration"] == 0 + assert trial0["run_id"] == 0 + assert "eval_time" in trial0 + + def test_score_records_via_evaluate(self): + """Test that score() also records data (via evaluate).""" + from hyperactive.experiment.func import FunctionExperiment + + def objective(params): + return params["x"] ** 2 + + exp = FunctionExperiment(objective) + + exp.score({"x": 5}) + + assert exp.data.n_trials == 1 + assert exp.data.history[0]["score"] == 25.0 + + def test_best_trial_property(self): + """Test best_trial property via accessor.""" + from hyperactive.experiment.func import FunctionExperiment + + def objective(params): + return params["x"] + + exp = FunctionExperiment(objective) + + exp.evaluate({"x": 1}) + exp.evaluate({"x": 5}) + exp.evaluate({"x": 3}) + + assert exp.data.best_trial["score"] == 5.0 + assert exp.data.best_score == 5.0 + + def test_clear_data(self): + """Test data.clear() resets experiment data.""" + from hyperactive.experiment.func import FunctionExperiment + + def objective(params): + return params["x"] + + exp = FunctionExperiment(objective) + exp.evaluate({"x": 1}) + + exp.data.clear() + + assert exp.data.n_trials == 0 + assert exp.data.history == [] + + def test_get_run(self): + """Test data.get_run() filters by run.""" + from hyperactive.experiment.func import FunctionExperiment + + def objective(params): + return params["x"] + + exp = FunctionExperiment(objective) + + exp.evaluate({"x": 1}) + + exp.data.new_run() + exp.evaluate({"x": 2}) + + run0 = exp.data.get_run(0) + run1 = exp.data.get_run(1) + + assert len(run0) == 1 + assert len(run1) == 1 + assert run0[0]["params"] == {"x": 1} + assert run1[0]["params"] == {"x": 2} + + +class TestOptimizerDataIntegration: + """Tests for data tracking with optimizers.""" + + def test_optimizer_records_trials(self): + """Test that optimizer.solve() records trials to experiment data.""" + from hyperactive.experiment.func import FunctionExperiment + from hyperactive.opt import RandomSearch + + def objective(params): + return -((params["x"] - 2) ** 2) + + exp = FunctionExperiment(objective) + opt = RandomSearch( + experiment=exp, + search_space={"x": [0, 1, 2, 3, 4]}, + n_iter=5, + ) + + opt.solve() + + assert exp.data.n_trials > 0 + assert all(t["run_id"] == 0 for t in exp.data.history) + + def test_multiple_solves_accumulate(self): + """Test that multiple solve() calls accumulate trials.""" + from hyperactive.experiment.func import FunctionExperiment + from hyperactive.opt import RandomSearch + + def objective(params): + return -((params["x"] - 2) ** 2) + + exp = FunctionExperiment(objective) + opt = RandomSearch( + experiment=exp, + search_space={"x": [0, 1, 2, 3, 4]}, + n_iter=3, + ) + + opt.solve() + n_trials_first = exp.data.n_trials + + opt.solve() + + assert exp.data.n_trials > n_trials_first + iterations = [t["iteration"] for t in exp.data.history] + assert iterations == list(range(len(iterations))) + + def test_data_accumulates_different_optimizers(self): + """Test data accumulates when using different optimizers.""" + from hyperactive.experiment.func import FunctionExperiment + from hyperactive.opt import GridSearch, RandomSearch + + def objective(params): + return -((params["x"] - 2) ** 2) + + exp = FunctionExperiment(objective) + + opt1 = RandomSearch( + experiment=exp, + search_space={"x": [0, 1, 2, 3, 4]}, + n_iter=3, + ) + opt1.solve() + n_trials_after_opt1 = exp.data.n_trials + + opt2 = GridSearch( + experiment=exp, + search_space={"x": [0, 1, 2, 3, 4]}, + ) + opt2.solve() + + assert exp.data.n_trials > n_trials_after_opt1 + iterations = [t["iteration"] for t in exp.data.history] + assert iterations == list(range(len(iterations)))