re-add support for user-specified starting point

doc/source/optimization.rst

@@ -46,6 +46,12 @@ cache files, serving a value from the cache for the first time in the run also c
 Only unique function evaluations are counted, so the second time a parameter configuration is selected by the strategy it is served from the 
 cache, but not counted as a unique function evaluation.
 
+All optimization algorithms, except for brute_force, random_sample, and bayes_opt, allow the user to specify an initial guess or 
+starting point for the optimization, called ``x0``. This can be passed to the strategy using the ``strategy_options=`` dictionary with ``"x0"`` as key and
+a list of values for each parameter in tune_params to note the starting point. For example, for a kernel that has parameters ``block_size_x`` (64, 128, 256)
+and ``tile_size_x`` (1,2,3), one could pass ``strategy_options=dict(x0=[128,2])`` to ``tune_kernel()`` to make sure the strategy starts from
+the configuration with ``block_size_x=128, tile_size_x=2``. The order in the ``x0`` list should match the order in the tunable parameters dictionary.
+
 Below all the strategies are listed with their strategy-specific options that can be passed in a dictionary to the ``strategy_options=`` argument
 of ``tune_kernel()``.
 

kernel_tuner/strategies/bayes_opt.py

@@ -13,7 +13,7 @@
 
 # BO imports
 from kernel_tuner.searchspace import Searchspace
-from kernel_tuner.strategies.common import CostFunc
+from kernel_tuner.strategies.common import CostFunc, get_options
 from kernel_tuner.util import StopCriterionReached
 
 try:
@@ -26,6 +26,24 @@
 
 supported_methods = ["poi", "ei", "lcb", "lcb-srinivas", "multi", "multi-advanced", "multi-fast", "multi-ultrafast"]
 
+# _options dict is used for generating documentation, but is not used to check for unsupported strategy_options in bayes_opt
+_options = dict(
+    covariancekernel=(
+        'The Covariance kernel to use, choose any from "constantrbf", "rbf", "matern32", "matern52"',
+        "matern32",
+    ),
+    covariancelengthscale=("The covariance length scale", 1.5),
+    method=(
+        "The Bayesian Optimization method to use, choose any from " + ", ".join(supported_methods),
+        "multi-ultrafast",
+    ),
+    samplingmethod=(
+        "Method used for initial sampling the parameter space, either random or Latin Hypercube Sampling (LHS)",
+        "lhs",
+    ),
+    popsize=("Number of initial samples", 20),
+)
+
 
 def generate_normalized_param_dicts(tune_params: dict, eps: float) -> Tuple[dict, dict]:
     """Generates normalization and denormalization dictionaries."""
@@ -92,6 +110,9 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     :rtype: list(dict()), dict()
 
     """
+    # we don't actually use this for Bayesian Optimization, but it is used to check for unsupported options  
+    get_options(tuning_options.strategy_options, _options, unsupported=["x0"])
+
     max_fevals = tuning_options.strategy_options.get("max_fevals", 100)
     prune_parameterspace = tuning_options.strategy_options.get("pruneparameterspace", True)
     if not bayes_opt_present:
@@ -143,25 +164,6 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     return cost_func.results
 
 
-# _options dict is used for generating documentation, but is not used to check for unsupported strategy_options in bayes_opt
-_options = dict(
-    covariancekernel=(
-        'The Covariance kernel to use, choose any from "constantrbf", "rbf", "matern32", "matern52"',
-        "matern32",
-    ),
-    covariancelengthscale=("The covariance length scale", 1.5),
-    method=(
-        "The Bayesian Optimization method to use, choose any from " + ", ".join(supported_methods),
-        "multi-ultrafast",
-    ),
-    samplingmethod=(
-        "Method used for initial sampling the parameter space, either random or Latin Hypercube Sampling (LHS)",
-        "lhs",
-    ),
-    popsize=("Number of initial samples", 20),
-)
-
-
 class BayesianOptimization:
     def __init__(
         self,

kernel_tuner/strategies/brute_force.py

@@ -6,6 +6,9 @@
 
 def tune(searchspace: Searchspace, runner, tuning_options):
 
+    # Force error on unsupported options
+    common.get_options(tuning_options.strategy_options or [], _options, unsupported=["max_fevals", "time_limit", "x0", "searchspace_construction_options"])
+
     # call the runner
     return runner.run(searchspace.sorted_list(), tuning_options)
 

kernel_tuner/strategies/common.py

@@ -43,9 +43,12 @@ def make_strategy_options_doc(strategy_options):
     return doc
 
 
-def get_options(strategy_options, options):
+def get_options(strategy_options, options, unsupported=None):
     """Get the strategy-specific options or their defaults from user-supplied strategy_options."""
-    accepted = list(options.keys()) + ["max_fevals", "time_limit", "searchspace_construction_options"]
+    accepted = list(options.keys()) + ["max_fevals", "time_limit", "x0", "searchspace_construction_options"]
+    if unsupported:
+        for key in unsupported:
+            accepted.remove(key)
     for key in strategy_options:
         if key not in accepted:
             raise ValueError(f"Unrecognized option {key} in strategy_options (allowed: {accepted})")
@@ -124,6 +127,11 @@ def __call__(self, x, check_restrictions=True):
 
         return return_value
 
+    def get_start_pos(self):
+        """Get starting position for optimization."""
+        _, x0, _ = self.get_bounds_x0_eps()
+        return x0
+
     def get_bounds_x0_eps(self):
         """Compute bounds, x0 (the initial guess), and eps."""
         values = list(self.searchspace.tune_params.values())
@@ -140,20 +148,16 @@ def get_bounds_x0_eps(self):
             bounds = [(0, eps * len(v)) for v in values]
             if x0:
                 # x0 has been supplied by the user, map x0 into [0, eps*len(v)]
-                x0 = scale_from_params(x0, self.tuning_options, eps)
+                x0 = scale_from_params(x0, self.searchspace.tune_params, eps)
             else:
                 # get a valid x0
                 pos = list(self.searchspace.get_random_sample(1)[0])
                 x0 = scale_from_params(pos, self.searchspace.tune_params, eps)
         else:
             bounds = self.get_bounds()
             if not x0:
-                x0 = [(min_v + max_v) / 2.0 for (min_v, max_v) in bounds]
-            eps = 1e9
-            for v_list in values:
-                if len(v_list) > 1:
-                    vals = np.sort(v_list)
-                    eps = min(eps, np.amin(np.gradient(vals)))
+                x0 = list(self.searchspace.get_random_sample(1)[0])
+            eps = 1
 
         self.tuning_options["eps"] = eps
         logging.debug('get_bounds_x0_eps called')

kernel_tuner/strategies/diff_evo.py

@@ -15,12 +15,11 @@
 
 def tune(searchspace: Searchspace, runner, tuning_options):
 
-
     method, popsize, maxiter = common.get_options(tuning_options.strategy_options, _options)
 
     # build a bounds array as needed for the optimizer
     cost_func = CostFunc(searchspace, tuning_options, runner)
-    bounds = cost_func.get_bounds()
+    bounds, x0, _ = cost_func.get_bounds_x0_eps()
 
     # ensure particles start from legal points
     population = list(list(p) for p in searchspace.get_random_sample(popsize))
@@ -29,7 +28,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     opt_result = None
     try:
         opt_result = differential_evolution(cost_func, bounds, maxiter=maxiter, popsize=popsize, init=population,
-                                        polish=False, strategy=method, disp=tuning_options.verbose)
+                                        polish=False, strategy=method, disp=tuning_options.verbose, x0=x0)
     except util.StopCriterionReached as e:
         if tuning_options.verbose:
             print(e)

kernel_tuner/strategies/firefly_algorithm.py

@@ -21,7 +21,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     cost_func = CostFunc(searchspace, tuning_options, runner, scaling=True)
 
     # using this instead of get_bounds because scaling is used
-    bounds, _, eps = cost_func.get_bounds_x0_eps()
+    bounds, x0, eps = cost_func.get_bounds_x0_eps()
 
     num_particles, maxiter, B0, gamma, alpha = common.get_options(tuning_options.strategy_options, _options)
 
@@ -38,6 +38,9 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     for i, particle in enumerate(swarm):
         particle.position = scale_from_params(population[i], searchspace.tune_params, eps)
 
+    # include user provided starting point
+    swarm[0].position = x0
+
     # compute initial intensities
     for j in range(num_particles):
         try:

kernel_tuner/strategies/genetic_algorithm.py

@@ -27,6 +27,8 @@ def tune(searchspace: Searchspace, runner, tuning_options):
 
     population = list(list(p) for p in searchspace.get_random_sample(pop_size))
 
+    population[0] = cost_func.get_start_pos()
+
     for generation in range(generations):
 
         # determine fitness of population members

kernel_tuner/strategies/greedy_ils.py

@@ -31,7 +31,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     cost_func = CostFunc(searchspace, tuning_options, runner)
 
     #while searching
-    candidate = searchspace.get_random_sample(1)[0]
+    candidate = cost_func.get_start_pos()
     best_score = cost_func(candidate, check_restrictions=False)
 
     last_improvement = 0

kernel_tuner/strategies/greedy_mls.py

@@ -24,17 +24,19 @@ def tune(searchspace: Searchspace, runner, tuning_options):
 
     fevals = 0
 
+    candidate = cost_func.get_start_pos()
+
     #while searching
     while fevals < max_fevals:
-        candidate = searchspace.get_random_sample(1)[0]
-
         try:
             base_hillclimb(candidate, neighbor, max_fevals, searchspace, tuning_options, cost_func, restart=restart, randomize=randomize, order=order)
         except util.StopCriterionReached as e:
             if tuning_options.verbose:
                 print(e)
             return cost_func.results
 
+        candidate = searchspace.get_random_sample(1)[0]
+
         fevals = len(tuning_options.unique_results)
 
     return cost_func.results

kernel_tuner/strategies/pso.py

@@ -21,8 +21,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     cost_func = CostFunc(searchspace, tuning_options, runner, scaling=True)
 
     #using this instead of get_bounds because scaling is used
-    bounds, _, eps = cost_func.get_bounds_x0_eps()
-
+    bounds, x0, eps = cost_func.get_bounds_x0_eps()
 
     num_particles, maxiter, w, c1, c2 = common.get_options(tuning_options.strategy_options, _options)
 
@@ -39,6 +38,9 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     for i, particle in enumerate(swarm):
         particle.position = scale_from_params(population[i], searchspace.tune_params, eps)
 
+    # include user provided starting point
+    swarm[0].position = x0
+
     # start optimization
     for i in range(maxiter):
         if tuning_options.verbose:

kernel_tuner/strategies/random_sample.py

@@ -11,7 +11,7 @@
 
 def tune(searchspace: Searchspace, runner, tuning_options):
     # get the samples
-    fraction = common.get_options(tuning_options.strategy_options, _options)[0]
+    fraction = common.get_options(tuning_options.strategy_options, _options, unsupported=["x0"])[0]
     assert 0 <= fraction <= 1.0
     num_samples = int(np.ceil(searchspace.size * fraction))
 

kernel_tuner/strategies/simulated_annealing.py

@@ -33,7 +33,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     max_fevals = min(searchspace.size, max_fevals)
 
     # get random starting point and evaluate cost
-    pos = list(searchspace.get_random_sample(1)[0])
+    pos = cost_func.get_start_pos()
     old_cost = cost_func(pos, check_restrictions=False)
 
     # main optimization loop

test/strategies/test_strategies.py

@@ -44,7 +44,9 @@ def test_strategies(vector_add, strategy):
         filter_options = {opt:val for opt, val in options.items() if opt in kernel_tuner.interface.strategy_map[strategy]._options}
     else:
         filter_options = options
-    filter_options["max_fevals"] = 10
+
+    if strategy != "brute_force":
+        filter_options["max_fevals"] = 10
 
     results, _ = kernel_tuner.tune_kernel(*vector_add, strategy=strategy, strategy_options=filter_options,
                                          verbose=False, cache=cache_filename, simulation_mode=True)
@@ -78,3 +80,20 @@ def test_strategies(vector_add, strategy):
         for expected_key, expected_type in expected_items.items():
             assert expected_key in res
             assert isinstance(res[expected_key], expected_type)
+
+    # check if strategy respects user-specified starting point (x0)
+    x0 = [256]
+    filter_options["x0"] = x0
+    if not strategy in ["brute_force", "random_sample", "bayes_opt"]:
+        results, _ = kernel_tuner.tune_kernel(*vector_add, strategy=strategy, strategy_options=filter_options,
+                                            verbose=False, cache=cache_filename, simulation_mode=True)
+        assert results[0]["block_size_x"] == x0[0]
+    else:
+        with pytest.raises(ValueError):
+            results, _ = kernel_tuner.tune_kernel(*vector_add, strategy=strategy, strategy_options=filter_options,
+                                            verbose=False, cache=cache_filename, simulation_mode=True)
+
+
+
+
+

test/test_time_budgets.py

@@ -69,9 +69,11 @@ def test_some_time_budget(env):
 @skip_if_no_gcc
 def test_full_time_budget(env):
     """Ensure that given ample time budget, the entire space is explored."""
-    res, _ = tune_kernel(*env, strategy="brute_force", strategy_options={"time_limit": 10.0})
 
     # Ensure that the entire space is explored.
     tune_params = env[-1]
     size_all = len(list(product(*tune_params.values())))
+
+    res, _ = tune_kernel(*env, strategy="random_sample", strategy_options={"fraction": 1.0, "time_limit": 10.0})
+
     assert len(res) == size_all