update cspace generation

Author: jacobdenobel

README.md

@@ -147,7 +147,7 @@ such as custom selection, recombination, or adaptation routines.
 
 #### Tuning <a name="tuning"></a>
 
-To facilitate **automated hyperparameter tuning**, ModularCMAES now provides functionality to **create and manipulate configuration spaces** directly compatible with popular optimization and AutoML tools, such as **SMAC**, **BOHB**, or **Optuna**. This functionality allows users to systematically explore both **algorithmic module combinations** and **numerical hyperparameters** (e.g., population size, learning rates, damping coefficients).
+To facilitate **automated hyperparameter tuning**, ModularCMAES now provides functionality to **create and manipulate configuration spaces** directly compatible with popular optimization and AutoML tools, such as **SMAC**, **BOHB**. This functionality allows users to systematically explore both **algorithmic module combinations** and **numerical hyperparameters** (e.g., population size, learning rates, damping coefficients).
 
 #### Configuration Space Generation
 
@@ -177,7 +177,7 @@ Example:
 
 ```python
 # Get only the module configuration space (no numeric parameters)
-cs_modules = get_configspace(only_modules=True)
+cs_modules = get_configspace(add_popsize=False, add_sigma=False, add_learning_rates=False)
 ```
 
 #### Creating Settings from a Configuration
@@ -191,8 +191,15 @@ from ConfigSpace import Configuration
 # Sample or load a configuration
 config = cs.sample_configuration()
 
+# Or for defaults
+default = cs.default_configuration()
+
+# The config can be edited
+config['sampler'] = 'HALTON'
+
 # Convert the configuration to a Settings object
-settings = settings_from_config(dim=10, config=config)
+# Note that keyword arguments like lb in the next example, can be passed to settings like so
+settings = settings_from_config(dim=10, config=config, lb=np.ones(10))
 ```
 
 The resulting `Settings` object can then be passed directly to the C++ backend:

modcma/c_maes/__init__.py

@@ -11,7 +11,7 @@
     ForbiddenGreaterThanRelation,
     UniformIntegerHyperparameter,
     NormalFloatHyperparameter,
-    EqualsCondition
+    EqualsCondition,
 )
 
 from .cmaescpp import (
@@ -38,9 +38,7 @@
 )  # pyright: ignore[reportMissingModuleSource]
 
 
-
-
-def get_module_options(name: str) -> tuple:
+def _get_module_options(name: str) -> tuple:
     if not hasattr(Modules, name):
         raise NameError(f"Modules has no member {name}")
 
@@ -51,15 +49,16 @@ def get_module_options(name: str) -> tuple:
     module_class = default_value.__class__
     if issubclass(module_class, Enum):
         other_values = [
-            x for x in module_class.__members__.values() if x is not default_value
+            x.name for x in module_class.__members__.values() 
+            if x is not default_value
         ]
-        return tuple([default_value] + other_values)
+        return tuple([default_value.name] + other_values)
     raise TypeError(f"{name} has a unparsable type {type(default_value)}")
 
 
 def get_all_module_options() -> dict:
     return {
-        name: get_module_options(name)
+        name: _get_module_options(name)
         for name in dir(Modules)
         if not name.startswith("_")
     }
@@ -76,7 +75,7 @@ def _make_numeric_parameter(
 
     if isinstance(default, int):
         return UniformIntegerHyperparameter(name, lb, ub, default, log=True)
-    
+
     elif isinstance(default, float):
         db = min(default - lb, ub - default)
         return NormalFloatHyperparameter(name, default, 0.3 * db, lb, ub)
@@ -87,61 +86,82 @@ def _make_numeric_parameter(
     )
 
 
-def _get_numeric_config(
-    dim: int,
+def get_configspace(
+    dim: int = None, 
+    add_popsize: bool = True, 
+    add_sigma: bool = True, 
+    add_learning_rates: bool = True
 ) -> ConfigurationSpace:
-    cs = ConfigurationSpace(
-        {
-            "lambda0": _make_numeric_parameter("lambda0", dim, 1, 50 * dim),
-            "mu0": _make_numeric_parameter("mu0", dim, 1, 50 * dim),
-            
-            "sigma0": _make_numeric_parameter("sigma0", dim, 1e-15, 1e15), # TODO: should be based on lb-ub
-            "cs": _make_numeric_parameter("cs", dim, 0, 1.0),
-            "cc": _make_numeric_parameter("cc", dim, 0, 1.0),
-            "cmu": _make_numeric_parameter("cmu", dim, 0, 1.0),
-            "c1": _make_numeric_parameter("c1", dim, 0, 1.0),
-            "damps": _make_numeric_parameter("damps", dim, 0, 10.0),
-        }
-    )
-    cs.add(ForbiddenGreaterThanRelation(cs["mu0"], cs["lambda0"]))
-
-    return cs
-
-
-def get_configspace(dim: int = None, only_modules: bool = False) -> ConfigurationSpace:
     cspace = ConfigurationSpace()
     for name, options in get_all_module_options().items():
         cspace.add(Categorical(name, options, default=options[0]))
 
-    if only_modules:
-        return cspace
-
-    if dim is None:
+    if dim is None and (add_popsize or add_sigma or add_learning_rates):
         warnings.warn(
             "Filling configspace with default numeric values for dim=2, "
             "since no dim was provided and only_modules was set to False"
         )
         dim = 2
 
-    cspace.add_configuration_space("", _get_numeric_config(dim), delimiter="")
+    if add_popsize:
+        cspace.add(_make_numeric_parameter("lambda0", dim, 1, 50 * dim))
+        cspace.add(_make_numeric_parameter("mu0", dim, 1, 50 * dim))
+        cspace.add(ForbiddenGreaterThanRelation(cspace["mu0"], cspace["lambda0"]))
+    
+    if add_sigma:
+        cspace.add(_make_numeric_parameter("sigma0", dim, 1e-15, 1e15))
+    
+    if add_learning_rates:
+        cspace.add(_make_numeric_parameter("cs", dim, 0, 1.0))
+        cspace.add(_make_numeric_parameter("cc", dim, 0, 1.0))
+        cspace.add(_make_numeric_parameter("cmu", dim, 0, 1.0))
+        cspace.add(_make_numeric_parameter("c1", dim, 0, 1.0))
+        cspace.add(_make_numeric_parameter("damps", dim, 0, 10.0))
+        
     return cspace
 
 
 def settings_from_config(
     dim: int, 
     config: Configuration, 
-    lb: np.ndarray = None, 
-    ub: np.ndarray = None
+    **kwargs
 ) -> Settings:
     modules = Modules()
-    via_setings = {}
+    via_setings = kwargs
     default_config = get_configspace(dim).get_default_configuration()
     for name, value in dict(config).items():
         if hasattr(modules, name):
+            attr_class = type(getattr(modules, name))
+            if issubclass(attr_class, Enum):
+                value = getattr(attr_class, value)
             setattr(modules, name, value)
             continue
         if default_config[name] != value:
             via_setings[name] = value
-    
-    settings = Settings(dim, modules, lb=lb, ub=ub, **via_setings) 
+
+    settings = Settings(dim, modules, **via_setings)
     return settings
+
+
+__all__ = (
+    "settings_from_config",
+    "get_configspace",
+    "get_all_module_options",
+    "Settings",
+    "Modules",
+    "constants",
+    "utils",
+    "sampling",
+    "mutation",
+    "selection",
+    "parameters",
+    "bounds",
+    "restart",
+    "options",
+    "repelling",
+    "Population",
+    "Parameters",
+    "ModularCMAES",
+    "center",
+    "es",
+)

scripts/tuning/run_smac.py

@@ -0,0 +1,121 @@
+import os
+import time
+import argparse
+from itertools import product
+from functools import partial
+
+import ioh
+import numpy as np
+from smac import Scenario
+from smac.acquisition.function import PriorAcquisitionFunction
+from smac import AlgorithmConfigurationFacade, HyperparameterOptimizationFacade
+from ConfigSpace import Configuration
+
+from modcma import c_maes
+
+
+DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "data"))
+
+
+def calc_aoc(logger, budget, fid, iid, dim):
+    data = logger.data()
+    data1 = data["None"][fid][dim][iid][0]
+    fvals = [x["raw_y_best"] for x in data1.values()]
+    fvals = np.array(fvals)
+    if np.isnan(fvals).any():
+        np.nan_to_num(fvals, copy=False, nan=1e8)
+    if len(fvals) < budget:
+        fvals = np.concatenate([fvals, (budget - len(fvals)) * [np.min(fvals)]])
+    parts = np.log10(np.clip(fvals[:budget], 1e-8, 1e2)) + 8
+    return np.mean(parts) / 10
+
+
+def get_bbob_performance(
+    config: Configuration, instance: str, seed: int = 0, dim: int = 5
+):
+    fid, iid = instance.split(",")
+    fid = int(fid[1:])
+    iid = int(iid[:-1])
+    np.random.seed(seed + iid)
+    c_maes.utils.set_seed(seed + iid)
+    BUDGET = dim * 10_000
+    l3 = ioh.logger.Store(
+        triggers=[ioh.logger.trigger.ALWAYS], properties=[ioh.logger.property.RAWYBEST]
+    )
+    problem = ioh.get_problem(fid, iid, dim)
+    problem.attach_logger(l3)
+
+    settings = c_maes.settings_from_config(
+        dim, 
+        config, 
+        budget=BUDGET, 
+        target= problem.optimum.y + 1e-9,
+        ub=problem.bounds.ub,
+        lb=problem.bounds.lb
+    )
+    par = c_maes.Parameters(settings)
+
+    try:
+        cma = c_maes.ModularCMAES(par)
+        cma.run(problem)
+    except Exception as e:
+        print(
+            f"Found target {problem.state.current_best.y} target, but exception ({e}), so run failed"
+        )
+        print(config)
+        return [np.inf]
+    
+    auc = calc_aoc(l3, BUDGET, fid, iid, dim)
+    return [auc] 
+
+
+def run_smac(fid, dim, use_learning_rates):
+    print(f"Running SMAC with fid={fid}, lr={use_learning_rates} and d={dim}")
+    cma_cs = c_maes.get_configspace(dim, add_learning_rates=use_learning_rates)
+
+    iids = range(1, 51)
+    if fid == "all":
+        fids = range(1, 25)
+        max_budget = 240
+    else:
+        fids = [fid]
+        max_budget = 50
+
+    args = list(product(fids, iids))
+    np.random.shuffle(args)
+    inst_feats = {str(arg): [arg[0]] for idx, arg in enumerate(args)}
+    scenario = Scenario(
+        cma_cs,
+        name=str(int(time.time())) + "-" + "CMA",
+        deterministic=False,
+        n_trials=100_000,
+        instances=args,
+        instance_features=inst_feats,
+        output_directory=os.path.join(
+            DATA_DIR, f"BBOB_F{fid}_{dim}D_LR{use_learning_rates}"
+        ),
+        n_workers=16,
+    )
+    pf = PriorAcquisitionFunction(
+        acquisition_function=AlgorithmConfigurationFacade.get_acquisition_function(
+            scenario
+        ),
+        decay_beta=scenario.n_trials / 10,
+    )
+    eval_func = partial(get_bbob_performance, dim=dim)
+    intensifier = HyperparameterOptimizationFacade.get_intensifier(
+        scenario, max_config_calls=max_budget
+    )
+    smac = HyperparameterOptimizationFacade(
+        scenario, eval_func, acquisition_function=pf, intensifier=intensifier
+    )
+    smac.optimize()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--fid", type=int, default=1)
+    parser.add_argument("--dim", type=int, default=5)
+    parser.add_argument("--use_learning_rates", action="store_true")
+    args = parser.parse_args()
+    run_smac(args.fid, args.dim, args.use_learning_rates)

tests/test_c_tuning.py

@@ -15,9 +15,6 @@ def test_module_options(self):
         self.assertEqual(options, options | {"sample_sigma": (False, True)})
         self.assertEqual(options, options | {"repelling_restart": (False, True)})
 
-    def test_numeric_options(self):
-        options = c_maes._get_numeric_config(2)
-        self.assertIsInstance(options, ConfigurationSpace)
 
     def test_configspace(self):
         cspace = c_maes.get_configspace(2)
@@ -30,7 +27,7 @@ def test_configspace(self):
 
         changed = deepcopy(default)
         changed['cc'] = 0.1
-        changed['sampler'] = c_maes.options.BaseSampler.HALTON
+        changed['sampler'] = "HALTON"
         settings_changed = c_maes.settings_from_config(2, changed)
         self.assertEqual(settings_changed.cc, 0.1)
         self.assertEqual(settings_changed.modules.sampler, c_maes.options.BaseSampler.HALTON)