From d12fd0440e3306a70a6061be19fc550c7bee978a Mon Sep 17 00:00:00 2001 From: udicaprio Date: Sat, 8 Mar 2025 16:24:22 +0000 Subject: [PATCH 01/11] Update acquisition.py --- bayes_opt/acquisition.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index 167bd5dc0..f5b72c895 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -27,7 +27,7 @@ import numpy as np from numpy.random import RandomState -from scipy.optimize import minimize +from scipy.optimize import minimize, differential_evolution from scipy.special import softmax from scipy.stats import norm from sklearn.gaussian_process import GaussianProcessRegressor @@ -295,8 +295,10 @@ def _l_bfgs_b_minimize( min_acq : float Acquisition function value at `x_min` """ + bounds = space.bounds continuous_dimensions = space.continuous_dimensions continuous_bounds = space.bounds[continuous_dimensions] + discrete_dimensions = ~continuous_dimensions if not continuous_dimensions.any(): min_acq = np.inf @@ -308,21 +310,23 @@ def _l_bfgs_b_minimize( x_min: NDArray[Float] for x_try in x_seeds: - def continuous_acq(x: NDArray[Float], x_try=x_try) -> NDArray[Float]: - x_try[continuous_dimensions] = x - return acq(x_try) - + # def continuous_acq(x: NDArray[Float], x_try=x_try) -> NDArray[Float]: + # x_try[continuous_dimensions] = x + # return acq(x_try) + if all(continuous_dimensions): # Find the minimum of minus the acquisition function - res: OptimizeResult = minimize( - continuous_acq, x_try[continuous_dimensions], bounds=continuous_bounds, method="L-BFGS-B" - ) + res: OptimizeResult = minimize(acq, x_try, bounds=continuous_bounds, method="L-BFGS-B") + else: + res: OptimizeResult = differential_evolution(acq, bounds=bounds, + integrality=discrete_dimensions, + seed=self.random_state) # See if success if not res.success: continue # Store it if better than previous minimum(maximum). if min_acq is None or np.squeeze(res.fun) >= min_acq: - x_try[continuous_dimensions] = res.x + x_try = res.x x_min = x_try min_acq = np.squeeze(res.fun) From 57e1a4d1bef0b3f6e00755f812e0033f1be1a68e Mon Sep 17 00:00:00 2001 From: udicaprio Date: Sat, 8 Mar 2025 21:59:27 +0000 Subject: [PATCH 02/11] Update acquisition.py Code linter optimisation with Ruff --- bayes_opt/acquisition.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index f5b72c895..1d75cfbc6 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -27,7 +27,7 @@ import numpy as np from numpy.random import RandomState -from scipy.optimize import minimize, differential_evolution +from scipy.optimize import differential_evolution, minimize from scipy.special import softmax from scipy.stats import norm from sklearn.gaussian_process import GaussianProcessRegressor @@ -317,8 +317,8 @@ def _l_bfgs_b_minimize( # Find the minimum of minus the acquisition function res: OptimizeResult = minimize(acq, x_try, bounds=continuous_bounds, method="L-BFGS-B") else: - res: OptimizeResult = differential_evolution(acq, bounds=bounds, - integrality=discrete_dimensions, + res: OptimizeResult = differential_evolution(acq, bounds=bounds, + integrality=discrete_dimensions, seed=self.random_state) # See if success if not res.success: From 0cd91763c5d15327c608e48cdcde4a17a68e277f Mon Sep 17 00:00:00 2001 From: udicaprio Date: Sun, 9 Mar 2025 09:46:40 +0000 Subject: [PATCH 03/11] Update acquisition.py Computational-efficiency of the MIP identification increased --- bayes_opt/acquisition.py | 46 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index 1d75cfbc6..0f8df6b26 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -308,28 +308,38 @@ def _l_bfgs_b_minimize( min_acq: float | None = None x_try: NDArray[Float] x_min: NDArray[Float] - for x_try in x_seeds: - # def continuous_acq(x: NDArray[Float], x_try=x_try) -> NDArray[Float]: - # x_try[continuous_dimensions] = x - # return acq(x_try) - if all(continuous_dimensions): - # Find the minimum of minus the acquisition function + #Case of continous optimization + if all(continuous_dimensions): + for x_try in x_seeds: res: OptimizeResult = minimize(acq, x_try, bounds=continuous_bounds, method="L-BFGS-B") - else: - res: OptimizeResult = differential_evolution(acq, bounds=bounds, + if not res.success: + continue + + # Store it if better than previous minimum(maximum). + if min_acq is None or np.squeeze(res.fun) >= min_acq: + x_try = res.x + x_min = x_try + min_acq = np.squeeze(res.fun) + + #Case of mixed-integer optimization + else: + ntrials = max(1, len(x_seeds)//100) + for i in range(ntrials): + xinit = space.random_sample(15*len(space.bounds), random_state=i) + res: OptimizeResult = differential_evolution(acq, bounds=bounds, init=xinit, integrality=discrete_dimensions, seed=self.random_state) - # See if success - if not res.success: - continue - - # Store it if better than previous minimum(maximum). - if min_acq is None or np.squeeze(res.fun) >= min_acq: - x_try = res.x - x_min = x_try - min_acq = np.squeeze(res.fun) - + # See if success + if not res.success: + continue + + # Store it if better than previous minimum(maximum). + if min_acq is None or np.squeeze(res.fun) >= min_acq: + x_try = res.x + x_min = x_try + min_acq = np.squeeze(res.fun) + if min_acq is None: min_acq = np.inf x_min = np.array([np.nan] * space.bounds.shape[0]) From 380fe6472a404ee5e8cb041120bdcfac2c46a794 Mon Sep 17 00:00:00 2001 From: udicaprio Date: Sun, 9 Mar 2025 09:57:42 +0000 Subject: [PATCH 04/11] Update acquisition.py Linter improved --- bayes_opt/acquisition.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index 0f8df6b26..39b4e9d0c 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -321,7 +321,7 @@ def _l_bfgs_b_minimize( x_try = res.x x_min = x_try min_acq = np.squeeze(res.fun) - + #Case of mixed-integer optimization else: ntrials = max(1, len(x_seeds)//100) @@ -339,7 +339,7 @@ def _l_bfgs_b_minimize( x_try = res.x x_min = x_try min_acq = np.squeeze(res.fun) - + if min_acq is None: min_acq = np.inf x_min = np.array([np.nan] * space.bounds.shape[0]) From 95f12517f1ffbd1a976dc8dd79d201145a34debb Mon Sep 17 00:00:00 2001 From: udicaprio Date: Sun, 9 Mar 2025 20:54:31 +0000 Subject: [PATCH 05/11] Update acquisition.py --- bayes_opt/acquisition.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index 39b4e9d0c..bb8157da5 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -309,7 +309,7 @@ def _l_bfgs_b_minimize( x_try: NDArray[Float] x_min: NDArray[Float] - #Case of continous optimization + # Case of continous optimization if all(continuous_dimensions): for x_try in x_seeds: res: OptimizeResult = minimize(acq, x_try, bounds=continuous_bounds, method="L-BFGS-B") @@ -322,14 +322,14 @@ def _l_bfgs_b_minimize( x_min = x_try min_acq = np.squeeze(res.fun) - #Case of mixed-integer optimization + # Case of mixed-integer optimization else: - ntrials = max(1, len(x_seeds)//100) + ntrials = max(1, len(x_seeds) // 100) for i in range(ntrials): - xinit = space.random_sample(15*len(space.bounds), random_state=i) - res: OptimizeResult = differential_evolution(acq, bounds=bounds, init=xinit, - integrality=discrete_dimensions, - seed=self.random_state) + xinit = space.random_sample(15 * len(space.bounds), random_state=i) + res: OptimizeResult = differential_evolution( + acq, bounds=bounds, init=xinit, integrality=discrete_dimensions, seed=self.random_state + ) # See if success if not res.success: continue From fb101a6655623838db8b7aec85029af71827cf0f Mon Sep 17 00:00:00 2001 From: udicaprio Date: Sat, 5 Apr 2025 09:57:52 +0100 Subject: [PATCH 06/11] Rename minimization methods for clarity and update tests accordingly --- bayes_opt/acquisition.py | 23 +++++++++++++---------- tests/test_acquisition.py | 4 ++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index bb8157da5..11c247988 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -219,7 +219,7 @@ def _acq_min( acq, space, n_random=max(n_random, n_l_bfgs_b), n_x_seeds=n_l_bfgs_b ) if n_l_bfgs_b: - x_min_l, min_acq_l = self._l_bfgs_b_minimize(acq, space, x_seeds=x_seeds) + x_min_l, min_acq_l = self._smart_minimize(acq, space, x_seeds=x_seeds) # Either n_random or n_l_bfgs_b is not 0 => at least one of x_min_r and x_min_l is not None if min_acq_r > min_acq_l: return x_min_l @@ -268,7 +268,7 @@ def _random_sample_minimize( x_seeds = [] return x_min, min_acq, x_seeds - def _l_bfgs_b_minimize( + def _smart_minimize( self, acq: Callable[[NDArray[Float]], NDArray[Float]], space: TargetSpace, @@ -295,15 +295,14 @@ def _l_bfgs_b_minimize( min_acq : float Acquisition function value at `x_min` """ - bounds = space.bounds continuous_dimensions = space.continuous_dimensions continuous_bounds = space.bounds[continuous_dimensions] discrete_dimensions = ~continuous_dimensions - if not continuous_dimensions.any(): - min_acq = np.inf - x_min = np.array([np.nan] * space.bounds.shape[0]) - return x_min, min_acq + # if not continuous_dimensions.any(): + # min_acq = np.inf + # x_min = np.array([np.nan] * space.bounds.shape[0]) + # return x_min, min_acq min_acq: float | None = None x_try: NDArray[Float] @@ -325,10 +324,14 @@ def _l_bfgs_b_minimize( # Case of mixed-integer optimization else: ntrials = max(1, len(x_seeds) // 100) - for i in range(ntrials): - xinit = space.random_sample(15 * len(space.bounds), random_state=i) + for _ in range(ntrials): + xinit = space.random_sample(15 * len(space.bounds), random_state=self.random_state) res: OptimizeResult = differential_evolution( - acq, bounds=bounds, init=xinit, integrality=discrete_dimensions, seed=self.random_state + acq, + bounds=space.bounds, + init=xinit, + integrality=discrete_dimensions, + rng=self.random_state, ) # See if success if not res.success: diff --git a/tests/test_acquisition.py b/tests/test_acquisition.py index 1191976df..38e2d4fed 100644 --- a/tests/test_acquisition.py +++ b/tests/test_acquisition.py @@ -105,7 +105,7 @@ def test_upper_confidence_bound(gp, target_space, random_state): assert acq.kappa == 0.5 -def test_l_bfgs_fails(target_space, random_state): +def test_smart_minimize_fails(target_space, random_state): acq = acquisition.UpperConfidenceBound(random_state=random_state) def fun(x): @@ -114,7 +114,7 @@ def fun(x): except IndexError: return np.nan - _, min_acq_l = acq._l_bfgs_b_minimize(fun, space=target_space, x_seeds=np.array([[2.5, 0.5]])) + _, min_acq_l = acq._smart_minimize(fun, space=target_space, x_seeds=np.array([[2.5, 0.5]])) assert min_acq_l == np.inf From 0e859dffd315911599a9409c18ff46cee3e2efe9 Mon Sep 17 00:00:00 2001 From: udicaprio Date: Sat, 5 Apr 2025 11:26:24 +0100 Subject: [PATCH 07/11] Refactor differential evolution import and optimize acquisition function --- bayes_opt/acquisition.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index 11c247988..e85df5dc0 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -27,7 +27,7 @@ import numpy as np from numpy.random import RandomState -from scipy.optimize import differential_evolution, minimize +from scipy.optimize._differentialevolution import DifferentialEvolutionSolver, minimize from scipy.special import softmax from scipy.stats import norm from sklearn.gaussian_process import GaussianProcessRegressor @@ -297,12 +297,6 @@ def _smart_minimize( """ continuous_dimensions = space.continuous_dimensions continuous_bounds = space.bounds[continuous_dimensions] - discrete_dimensions = ~continuous_dimensions - - # if not continuous_dimensions.any(): - # min_acq = np.inf - # x_min = np.array([np.nan] * space.bounds.shape[0]) - # return x_min, min_acq min_acq: float | None = None x_try: NDArray[Float] @@ -326,20 +320,17 @@ def _smart_minimize( ntrials = max(1, len(x_seeds) // 100) for _ in range(ntrials): xinit = space.random_sample(15 * len(space.bounds), random_state=self.random_state) - res: OptimizeResult = differential_evolution( - acq, - bounds=space.bounds, - init=xinit, - integrality=discrete_dimensions, - rng=self.random_state, - ) + de = DifferentialEvolutionSolver(acq, bounds=space.bounds, init=xinit, rng=self.random_state) + res: OptimizeResult = de.solve() + # See if success if not res.success: continue # Store it if better than previous minimum(maximum). if min_acq is None or np.squeeze(res.fun) >= min_acq: - x_try = res.x + x_try_sc = de._unscale_parameters(res.x) + x_try = space.kernel_transform(x_try_sc).flatten() x_min = x_try min_acq = np.squeeze(res.fun) From fb9d6a2d125c305adb7cedeeb98efa6c002a62ba Mon Sep 17 00:00:00 2001 From: udicaprio Date: Sun, 27 Apr 2025 14:30:58 +0100 Subject: [PATCH 08/11] Modification to differential evolution solver integration and refine continuous parameter search --- bayes_opt/acquisition.py | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index 38716e2ba..12ceb2d11 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -375,19 +375,35 @@ def _smart_minimize( ntrials = max(1, len(x_seeds) // 100) for _ in range(ntrials): xinit = space.random_sample(15 * len(space.bounds), random_state=self.random_state) - de = DifferentialEvolutionSolver(acq, bounds=space.bounds, init=xinit, rng=self.random_state) - res: OptimizeResult = de.solve() - - # See if success - if not res.success: + de = DifferentialEvolutionSolver( + acq, bounds=space.bounds, init=xinit, rng=self.random_state, polish=False + ) + res_de: OptimizeResult = de.solve() + # Check if success + if not res_de.success: continue - # Store it if better than previous minimum(maximum). - if min_acq is None or np.squeeze(res.fun) >= min_acq: - x_try_sc = de._unscale_parameters(res.x) - x_try = space.kernel_transform(x_try_sc).flatten() - x_min = x_try - min_acq = np.squeeze(res.fun) + x_min = res_de.x + min_acq = np.squeeze(res_de.fun) + + # Refine the identification of continous parameters with deterministic search + if any(continuous_dimensions): + x_try = x_min.copy() + + def continuous_acq(x: NDArray[Float], x_try=x_try) -> NDArray[Float]: + x_try[continuous_dimensions] = x + return acq(x_try) + + res: OptimizeResult = minimize( + continuous_acq, + x_min[continuous_dimensions], + bounds=continuous_bounds, + method="L-BFGS-B", + ) + if np.squeeze(res.fun) >= min_acq and res.success: + x_try[continuous_dimensions] = res.x + x_min = x_try + min_acq = np.squeeze(res.fun) if min_acq is None: min_acq = np.inf From 07f75b216237819dc508ef4d68f5ebfb2a283ae5 Mon Sep 17 00:00:00 2001 From: udicaprio Date: Sun, 27 Apr 2025 15:27:18 +0100 Subject: [PATCH 09/11] Refactor DifferentialEvolutionSolver initialization to support scipy version compatibility --- bayes_opt/acquisition.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index 12ceb2d11..50a566463 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -27,6 +27,8 @@ import numpy as np from numpy.random import RandomState +from packaging import version +from scipy import __version__ as scipy_version from scipy.optimize._differentialevolution import DifferentialEvolutionSolver, minimize from scipy.special import softmax from scipy.stats import norm @@ -373,11 +375,17 @@ def _smart_minimize( # Case of mixed-integer optimization else: ntrials = max(1, len(x_seeds) // 100) + for _ in range(ntrials): xinit = space.random_sample(15 * len(space.bounds), random_state=self.random_state) - de = DifferentialEvolutionSolver( - acq, bounds=space.bounds, init=xinit, rng=self.random_state, polish=False - ) + + de_parameters = {"func": acq, "bounds": space.bounds, "polish": False, "init": xinit} + if version.parse(scipy_version) < version.parse("1.15.0"): + de_parameters["seed"] = self.random_state + else: + de_parameters["rng"] = self.random_state + + de = DifferentialEvolutionSolver(**de_parameters) res_de: OptimizeResult = de.solve() # Check if success if not res_de.success: From a241f626bed1faea81f627e122c3176684ead059 Mon Sep 17 00:00:00 2001 From: udicaprio Date: Thu, 1 May 2025 08:42:05 +0200 Subject: [PATCH 10/11] Removes fixed algorithm for acquisition function minimization and applies kernel transformation to x_min --- bayes_opt/acquisition.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index 50a566463..f31862d6c 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -405,9 +405,7 @@ def continuous_acq(x: NDArray[Float], x_try=x_try) -> NDArray[Float]: res: OptimizeResult = minimize( continuous_acq, x_min[continuous_dimensions], - bounds=continuous_bounds, - method="L-BFGS-B", - ) + bounds=continuous_bounds) if np.squeeze(res.fun) >= min_acq and res.success: x_try[continuous_dimensions] = res.x x_min = x_try @@ -417,6 +415,7 @@ def continuous_acq(x: NDArray[Float], x_try=x_try) -> NDArray[Float]: min_acq = np.inf x_min = np.array([np.nan] * space.bounds.shape[0]) + x_min=space.kernel_transform(x_min).reshape(x_min.shape) # Clip output to make sure it lies within the bounds. Due to floating # point technicalities this is not always the case. return np.clip(x_min, space.bounds[:, 0], space.bounds[:, 1]), min_acq From 79b8e0356c8de7addb07f6326ad45e7194d3d97f Mon Sep 17 00:00:00 2001 From: udicaprio Date: Thu, 1 May 2025 09:34:58 +0200 Subject: [PATCH 11/11] Fixing formatting issue --- bayes_opt/acquisition.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bayes_opt/acquisition.py b/bayes_opt/acquisition.py index f31862d6c..37487ca0c 100644 --- a/bayes_opt/acquisition.py +++ b/bayes_opt/acquisition.py @@ -403,9 +403,8 @@ def continuous_acq(x: NDArray[Float], x_try=x_try) -> NDArray[Float]: return acq(x_try) res: OptimizeResult = minimize( - continuous_acq, - x_min[continuous_dimensions], - bounds=continuous_bounds) + continuous_acq, x_min[continuous_dimensions], bounds=continuous_bounds + ) if np.squeeze(res.fun) >= min_acq and res.success: x_try[continuous_dimensions] = res.x x_min = x_try @@ -415,7 +414,7 @@ def continuous_acq(x: NDArray[Float], x_try=x_try) -> NDArray[Float]: min_acq = np.inf x_min = np.array([np.nan] * space.bounds.shape[0]) - x_min=space.kernel_transform(x_min).reshape(x_min.shape) + x_min = space.kernel_transform(x_min).reshape(x_min.shape) # Clip output to make sure it lies within the bounds. Due to floating # point technicalities this is not always the case. return np.clip(x_min, space.bounds[:, 0], space.bounds[:, 1]), min_acq