From 78c8a9de372525805da0d856e2bc4edcfdee993e Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Wed, 30 Apr 2025 15:20:05 +0200 Subject: [PATCH 1/4] re-add support for user-specified starting point --- doc/source/optimization.rst | 6 ++++++ kernel_tuner/strategies/common.py | 9 +++++++-- kernel_tuner/strategies/diff_evo.py | 5 ++--- kernel_tuner/strategies/firefly_algorithm.py | 5 ++++- kernel_tuner/strategies/genetic_algorithm.py | 2 ++ kernel_tuner/strategies/greedy_ils.py | 2 +- kernel_tuner/strategies/greedy_mls.py | 8 +++++++- kernel_tuner/strategies/pso.py | 6 ++++-- kernel_tuner/strategies/simulated_annealing.py | 2 +- test/strategies/test_strategies.py | 14 ++++++++++++++ 10 files changed, 48 insertions(+), 11 deletions(-) diff --git a/doc/source/optimization.rst b/doc/source/optimization.rst index 59219ad51..2a188d442 100644 --- a/doc/source/optimization.rst +++ b/doc/source/optimization.rst @@ -46,6 +46,12 @@ cache files, serving a value from the cache for the first time in the run also c Only unique function evaluations are counted, so the second time a parameter configuration is selected by the strategy it is served from the cache, but not counted as a unique function evaluation. +All optimization algorithms, except for brute_force, random_sample, and bayes_opt, allow the user to specify an initial guess or +starting point for the optimization, called ``x0``. This can be passed to the strategy using the ``strategy_options=`` dictionary with ``"x0"`` as key and +a list of values for each parameter in tune_params to note the starting point. For example, for a kernel that has parameters ``block_size_x`` (64, 128, 256) +and ``tile_size_x`` (1,2,3), one could pass ``strategy_options=dict(x0=[128,2])`` to ``tune_kernel()`` to make sure the strategy starts from +the configuration with ``block_size_x=128, tile_size_x=2``. The order in the ``x0`` list should match the order in the tunable parameters dictionary. + Below all the strategies are listed with their strategy-specific options that can be passed in a dictionary to the ``strategy_options=`` argument of ``tune_kernel()``. diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index d01eae937..0bca59c96 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -44,7 +44,7 @@ def make_strategy_options_doc(strategy_options): def get_options(strategy_options, options): """Get the strategy-specific options or their defaults from user-supplied strategy_options.""" - accepted = list(options.keys()) + ["max_fevals", "time_limit"] + accepted = list(options.keys()) + ["max_fevals", "time_limit", "x0"] for key in strategy_options: if key not in accepted: raise ValueError(f"Unrecognized option {key} in strategy_options") @@ -114,6 +114,11 @@ def __call__(self, x, check_restrictions=True): return return_value + def get_start_pos(self): + """Get starting position for optimization.""" + _, x0, _ = self.get_bounds_x0_eps() + return x0 + def get_bounds_x0_eps(self): """Compute bounds, x0 (the initial guess), and eps.""" values = list(self.searchspace.tune_params.values()) @@ -130,7 +135,7 @@ def get_bounds_x0_eps(self): bounds = [(0, eps * len(v)) for v in values] if x0: # x0 has been supplied by the user, map x0 into [0, eps*len(v)] - x0 = scale_from_params(x0, self.tuning_options, eps) + x0 = scale_from_params(x0, self.searchspace.tune_params, eps) else: # get a valid x0 pos = list(self.searchspace.get_random_sample(1)[0]) diff --git a/kernel_tuner/strategies/diff_evo.py b/kernel_tuner/strategies/diff_evo.py index 5ad2b9474..a2c9d9d00 100644 --- a/kernel_tuner/strategies/diff_evo.py +++ b/kernel_tuner/strategies/diff_evo.py @@ -15,12 +15,11 @@ def tune(searchspace: Searchspace, runner, tuning_options): - method, popsize, maxiter = common.get_options(tuning_options.strategy_options, _options) # build a bounds array as needed for the optimizer cost_func = CostFunc(searchspace, tuning_options, runner) - bounds = cost_func.get_bounds() + bounds, x0, _ = cost_func.get_bounds_x0_eps() # ensure particles start from legal points population = list(list(p) for p in searchspace.get_random_sample(popsize)) @@ -29,7 +28,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): opt_result = None try: opt_result = differential_evolution(cost_func, bounds, maxiter=maxiter, popsize=popsize, init=population, - polish=False, strategy=method, disp=tuning_options.verbose) + polish=False, strategy=method, disp=tuning_options.verbose, x0=x0) except util.StopCriterionReached as e: if tuning_options.verbose: print(e) diff --git a/kernel_tuner/strategies/firefly_algorithm.py b/kernel_tuner/strategies/firefly_algorithm.py index dc43aae6f..92f03dcd4 100644 --- a/kernel_tuner/strategies/firefly_algorithm.py +++ b/kernel_tuner/strategies/firefly_algorithm.py @@ -21,7 +21,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): cost_func = CostFunc(searchspace, tuning_options, runner, scaling=True) # using this instead of get_bounds because scaling is used - bounds, _, eps = cost_func.get_bounds_x0_eps() + bounds, x0, eps = cost_func.get_bounds_x0_eps() num_particles, maxiter, B0, gamma, alpha = common.get_options(tuning_options.strategy_options, _options) @@ -38,6 +38,9 @@ def tune(searchspace: Searchspace, runner, tuning_options): for i, particle in enumerate(swarm): particle.position = scale_from_params(population[i], searchspace.tune_params, eps) + # include user provided starting point + swarm[0].position = x0 + # compute initial intensities for j in range(num_particles): try: diff --git a/kernel_tuner/strategies/genetic_algorithm.py b/kernel_tuner/strategies/genetic_algorithm.py index c29c150b5..9ab5d5ad6 100644 --- a/kernel_tuner/strategies/genetic_algorithm.py +++ b/kernel_tuner/strategies/genetic_algorithm.py @@ -27,6 +27,8 @@ def tune(searchspace: Searchspace, runner, tuning_options): population = list(list(p) for p in searchspace.get_random_sample(pop_size)) + population[0] = cost_func.get_start_pos() + for generation in range(generations): # determine fitness of population members diff --git a/kernel_tuner/strategies/greedy_ils.py b/kernel_tuner/strategies/greedy_ils.py index a4c521746..1906f730c 100644 --- a/kernel_tuner/strategies/greedy_ils.py +++ b/kernel_tuner/strategies/greedy_ils.py @@ -31,7 +31,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): cost_func = CostFunc(searchspace, tuning_options, runner) #while searching - candidate = searchspace.get_random_sample(1)[0] + candidate = cost_func.get_start_pos() best_score = cost_func(candidate, check_restrictions=False) last_improvement = 0 diff --git a/kernel_tuner/strategies/greedy_mls.py b/kernel_tuner/strategies/greedy_mls.py index 1b34da501..e32968dc0 100644 --- a/kernel_tuner/strategies/greedy_mls.py +++ b/kernel_tuner/strategies/greedy_mls.py @@ -24,9 +24,15 @@ def tune(searchspace: Searchspace, runner, tuning_options): fevals = 0 + first_candidate = cost_func.get_start_pos() + #while searching while fevals < max_fevals: - candidate = searchspace.get_random_sample(1)[0] + if first_candidate: + candidate = first_candidate + first_candidate = None + else: + candidate = searchspace.get_random_sample(1)[0] try: base_hillclimb(candidate, neighbor, max_fevals, searchspace, tuning_options, cost_func, restart=restart, randomize=randomize, order=order) diff --git a/kernel_tuner/strategies/pso.py b/kernel_tuner/strategies/pso.py index 5b0df1429..0834f52c0 100644 --- a/kernel_tuner/strategies/pso.py +++ b/kernel_tuner/strategies/pso.py @@ -21,8 +21,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): cost_func = CostFunc(searchspace, tuning_options, runner, scaling=True) #using this instead of get_bounds because scaling is used - bounds, _, eps = cost_func.get_bounds_x0_eps() - + bounds, x0, eps = cost_func.get_bounds_x0_eps() num_particles, maxiter, w, c1, c2 = common.get_options(tuning_options.strategy_options, _options) @@ -39,6 +38,9 @@ def tune(searchspace: Searchspace, runner, tuning_options): for i, particle in enumerate(swarm): particle.position = scale_from_params(population[i], searchspace.tune_params, eps) + # include user provided starting point + swarm[0].position = x0 + # start optimization for i in range(maxiter): if tuning_options.verbose: diff --git a/kernel_tuner/strategies/simulated_annealing.py b/kernel_tuner/strategies/simulated_annealing.py index dce929b7b..7d60a04db 100644 --- a/kernel_tuner/strategies/simulated_annealing.py +++ b/kernel_tuner/strategies/simulated_annealing.py @@ -30,7 +30,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): max_feval = tuning_options.strategy_options.get("max_fevals", max_iter) # get random starting point and evaluate cost - pos = list(searchspace.get_random_sample(1)[0]) + pos = cost_func.get_start_pos() old_cost = cost_func(pos, check_restrictions=False) # main optimization loop diff --git a/test/strategies/test_strategies.py b/test/strategies/test_strategies.py index 096be38b0..d72027cda 100644 --- a/test/strategies/test_strategies.py +++ b/test/strategies/test_strategies.py @@ -78,3 +78,17 @@ def test_strategies(vector_add, strategy): for expected_key, expected_type in expected_items.items(): assert expected_key in res assert isinstance(res[expected_key], expected_type) + + # check if strategy respects user-specified starting point (x0) + if not strategy in ["brute_force", "random_sample", "bayes_opt"]: + x0 = [256] + filter_options["x0"] = x0 + + results, _ = kernel_tuner.tune_kernel(*vector_add, strategy=strategy, strategy_options=filter_options, + verbose=False, cache=cache_filename, simulation_mode=True) + + assert results[0]["block_size_x"] == x0[0] + + + + From 768d189a13f389fe56125e2bef524f378574a9cb Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Fri, 4 Jul 2025 09:02:11 +0200 Subject: [PATCH 2/4] bugfix in start position generation and greedy mls --- kernel_tuner/strategies/common.py | 8 ++------ kernel_tuner/strategies/greedy_mls.py | 10 +++------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index 07bc28c70..c364fdaa8 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -153,12 +153,8 @@ def get_bounds_x0_eps(self): else: bounds = self.get_bounds() if not x0: - x0 = [(min_v + max_v) / 2.0 for (min_v, max_v) in bounds] - eps = 1e9 - for v_list in values: - if len(v_list) > 1: - vals = np.sort(v_list) - eps = min(eps, np.amin(np.gradient(vals))) + x0 = list(self.searchspace.get_random_sample(1)[0]) + eps = 1 self.tuning_options["eps"] = eps logging.debug('get_bounds_x0_eps called') diff --git a/kernel_tuner/strategies/greedy_mls.py b/kernel_tuner/strategies/greedy_mls.py index e32968dc0..a651e11d7 100644 --- a/kernel_tuner/strategies/greedy_mls.py +++ b/kernel_tuner/strategies/greedy_mls.py @@ -24,16 +24,10 @@ def tune(searchspace: Searchspace, runner, tuning_options): fevals = 0 - first_candidate = cost_func.get_start_pos() + candidate = cost_func.get_start_pos() #while searching while fevals < max_fevals: - if first_candidate: - candidate = first_candidate - first_candidate = None - else: - candidate = searchspace.get_random_sample(1)[0] - try: base_hillclimb(candidate, neighbor, max_fevals, searchspace, tuning_options, cost_func, restart=restart, randomize=randomize, order=order) except util.StopCriterionReached as e: @@ -41,6 +35,8 @@ def tune(searchspace: Searchspace, runner, tuning_options): print(e) return cost_func.results + candidate = searchspace.get_random_sample(1)[0] + fevals = len(tuning_options.unique_results) return cost_func.results From f0afa0bfe365400a36efe72cd956cdcf6efc8415 Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Fri, 4 Jul 2025 10:00:03 +0200 Subject: [PATCH 3/4] force error on unsupported options --- kernel_tuner/strategies/brute_force.py | 3 +++ kernel_tuner/strategies/common.py | 5 ++++- kernel_tuner/strategies/random_sample.py | 2 +- test/strategies/test_strategies.py | 4 +++- test/test_time_budgets.py | 4 +++- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/kernel_tuner/strategies/brute_force.py b/kernel_tuner/strategies/brute_force.py index a0e3f8ebe..e1e5fdb60 100644 --- a/kernel_tuner/strategies/brute_force.py +++ b/kernel_tuner/strategies/brute_force.py @@ -6,6 +6,9 @@ def tune(searchspace: Searchspace, runner, tuning_options): + # Force error on unsupported options + common.get_options(tuning_options.strategy_options or [], _options, unsupported=["max_fevals", "time_limit", "x0", "searchspace_construction_options"]) + # call the runner return runner.run(searchspace.sorted_list(), tuning_options) diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index c364fdaa8..58a6f044f 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -43,9 +43,12 @@ def make_strategy_options_doc(strategy_options): return doc -def get_options(strategy_options, options): +def get_options(strategy_options, options, unsupported=None): """Get the strategy-specific options or their defaults from user-supplied strategy_options.""" accepted = list(options.keys()) + ["max_fevals", "time_limit", "x0", "searchspace_construction_options"] + if unsupported: + for key in unsupported: + accepted.remove(key) for key in strategy_options: if key not in accepted: raise ValueError(f"Unrecognized option {key} in strategy_options (allowed: {accepted})") diff --git a/kernel_tuner/strategies/random_sample.py b/kernel_tuner/strategies/random_sample.py index 06ab4b9f6..86caccfa3 100644 --- a/kernel_tuner/strategies/random_sample.py +++ b/kernel_tuner/strategies/random_sample.py @@ -11,7 +11,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): # get the samples - fraction = common.get_options(tuning_options.strategy_options, _options)[0] + fraction = common.get_options(tuning_options.strategy_options, _options, unsupported=["x0"])[0] assert 0 <= fraction <= 1.0 num_samples = int(np.ceil(searchspace.size * fraction)) diff --git a/test/strategies/test_strategies.py b/test/strategies/test_strategies.py index d72027cda..771e60c08 100644 --- a/test/strategies/test_strategies.py +++ b/test/strategies/test_strategies.py @@ -44,7 +44,9 @@ def test_strategies(vector_add, strategy): filter_options = {opt:val for opt, val in options.items() if opt in kernel_tuner.interface.strategy_map[strategy]._options} else: filter_options = options - filter_options["max_fevals"] = 10 + + if strategy != "brute_force": + filter_options["max_fevals"] = 10 results, _ = kernel_tuner.tune_kernel(*vector_add, strategy=strategy, strategy_options=filter_options, verbose=False, cache=cache_filename, simulation_mode=True) diff --git a/test/test_time_budgets.py b/test/test_time_budgets.py index acf10a0e9..8773801c8 100644 --- a/test/test_time_budgets.py +++ b/test/test_time_budgets.py @@ -69,9 +69,11 @@ def test_some_time_budget(env): @skip_if_no_gcc def test_full_time_budget(env): """Ensure that given ample time budget, the entire space is explored.""" - res, _ = tune_kernel(*env, strategy="brute_force", strategy_options={"time_limit": 10.0}) # Ensure that the entire space is explored. tune_params = env[-1] size_all = len(list(product(*tune_params.values()))) + + res, _ = tune_kernel(*env, strategy="random_sample", strategy_options={"fraction": 1.0, "time_limit": 10.0}) + assert len(res) == size_all From ad88894c0dd73fc029b75fa759d2c36f267fb76a Mon Sep 17 00:00:00 2001 From: fjwillemsen Date: Fri, 4 Jul 2025 14:19:13 +0200 Subject: [PATCH 4/4] Extra check to make sure non-x0 supporting strategies raise an exception, implemented unsupported options check in Bayesian Optimization --- kernel_tuner/strategies/bayes_opt.py | 42 +++++++++++++++------------- test/strategies/test_strategies.py | 13 +++++---- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/kernel_tuner/strategies/bayes_opt.py b/kernel_tuner/strategies/bayes_opt.py index 663cb12c8..99d5e865e 100644 --- a/kernel_tuner/strategies/bayes_opt.py +++ b/kernel_tuner/strategies/bayes_opt.py @@ -13,7 +13,7 @@ # BO imports from kernel_tuner.searchspace import Searchspace -from kernel_tuner.strategies.common import CostFunc +from kernel_tuner.strategies.common import CostFunc, get_options from kernel_tuner.util import StopCriterionReached try: @@ -26,6 +26,24 @@ supported_methods = ["poi", "ei", "lcb", "lcb-srinivas", "multi", "multi-advanced", "multi-fast", "multi-ultrafast"] +# _options dict is used for generating documentation, but is not used to check for unsupported strategy_options in bayes_opt +_options = dict( + covariancekernel=( + 'The Covariance kernel to use, choose any from "constantrbf", "rbf", "matern32", "matern52"', + "matern32", + ), + covariancelengthscale=("The covariance length scale", 1.5), + method=( + "The Bayesian Optimization method to use, choose any from " + ", ".join(supported_methods), + "multi-ultrafast", + ), + samplingmethod=( + "Method used for initial sampling the parameter space, either random or Latin Hypercube Sampling (LHS)", + "lhs", + ), + popsize=("Number of initial samples", 20), +) + def generate_normalized_param_dicts(tune_params: dict, eps: float) -> Tuple[dict, dict]: """Generates normalization and denormalization dictionaries.""" @@ -92,6 +110,9 @@ def tune(searchspace: Searchspace, runner, tuning_options): :rtype: list(dict()), dict() """ + # we don't actually use this for Bayesian Optimization, but it is used to check for unsupported options + get_options(tuning_options.strategy_options, _options, unsupported=["x0"]) + max_fevals = tuning_options.strategy_options.get("max_fevals", 100) prune_parameterspace = tuning_options.strategy_options.get("pruneparameterspace", True) if not bayes_opt_present: @@ -143,25 +164,6 @@ def tune(searchspace: Searchspace, runner, tuning_options): return cost_func.results -# _options dict is used for generating documentation, but is not used to check for unsupported strategy_options in bayes_opt -_options = dict( - covariancekernel=( - 'The Covariance kernel to use, choose any from "constantrbf", "rbf", "matern32", "matern52"', - "matern32", - ), - covariancelengthscale=("The covariance length scale", 1.5), - method=( - "The Bayesian Optimization method to use, choose any from " + ", ".join(supported_methods), - "multi-ultrafast", - ), - samplingmethod=( - "Method used for initial sampling the parameter space, either random or Latin Hypercube Sampling (LHS)", - "lhs", - ), - popsize=("Number of initial samples", 20), -) - - class BayesianOptimization: def __init__( self, diff --git a/test/strategies/test_strategies.py b/test/strategies/test_strategies.py index 771e60c08..0d8ec0458 100644 --- a/test/strategies/test_strategies.py +++ b/test/strategies/test_strategies.py @@ -82,14 +82,17 @@ def test_strategies(vector_add, strategy): assert isinstance(res[expected_key], expected_type) # check if strategy respects user-specified starting point (x0) + x0 = [256] + filter_options["x0"] = x0 if not strategy in ["brute_force", "random_sample", "bayes_opt"]: - x0 = [256] - filter_options["x0"] = x0 - results, _ = kernel_tuner.tune_kernel(*vector_add, strategy=strategy, strategy_options=filter_options, - verbose=False, cache=cache_filename, simulation_mode=True) - + verbose=False, cache=cache_filename, simulation_mode=True) assert results[0]["block_size_x"] == x0[0] + else: + with pytest.raises(ValueError): + results, _ = kernel_tuner.tune_kernel(*vector_add, strategy=strategy, strategy_options=filter_options, + verbose=False, cache=cache_filename, simulation_mode=True) +