added scripts

Author: jacobdenobel

scripts/matrix/get_data.py

@@ -1,17 +1,21 @@
 from time import perf_counter
-
+import warnings
 import numpy as np
 import modcma.c_maes as modcma
 import ioh
 import pandas as pd 
 
 from multiprocessing import Pool
-
+import cma as pycma
 
 from pprint import pprint
 
 np.random.seed(12)
 
+DIMS = 2, 3, 5, 10, 20, 40, 100
+FUNCTIONS = [1, 2, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+N_REPEATS = 100
+
 def run_modma(problem: ioh.ProblemType, 
               x0: np.ndarray, 
               logger_obj,
@@ -68,16 +72,10 @@ def reset(self):
         for item in self.names:
             setattr(self, item, 0)
 
-
-if __name__ == "__main__":
-    dims = 2, 3, 5, 10, 20, 40, 100
-    functions = [1, 2, 6, 7, 8, 9, 10, 11, 12, 13, 14]
-    
-    n_repeats = 100
+def collect_modcma():
     options = modcma.options.MatrixAdaptationType.__members__
     del options['COVARIANCE_NO_EIGV']
 
-
     def collect(name, option):
         logger = ioh.logger.Analyzer(
             folder_name=name, 
@@ -86,11 +84,11 @@ def collect(name, option):
         )
         collector = RestartCollector()
         logger.add_run_attributes(collector, collector.names)
-        for fid in functions:
-            for d in dims:
+        for fid in FUNCTIONS:
+            for d in DIMS:
                 problem = ioh.get_problem(fid, 1, d)
                 problem.attach_logger(logger)
-                for i in range(n_repeats):
+                for i in range(N_REPEATS):
                     modcma.utils.set_seed(21 + fid * d * i)
                     collector.reset()
                     run_modma(problem, np.zeros(d), collector, option)
@@ -100,6 +98,55 @@ def collect(name, option):
     with Pool(len(options)) as p:
         p.starmap(collect, options.items())
 
+
+def run_pycma(problem: ioh.ProblemType, x0: np.ndarray):
+    options = pycma.CMAOptions()
+    options['CMA_active'] = False
+    options["verbose"] = -1
+    options["CMA_diagonal"] = False
+
+    cma = pycma.CMAEvolutionStrategy(x0, 2.0, options=options)
+    settings = modcma.Settings(problem.meta_data.n_variables)
+    assert settings.lambda0 == cma.sp.popsize
+    start = perf_counter()
+
+    target = problem.optimum.y + 1e-8
+    budget = problem.meta_data.n_variables * 100_000
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        while problem.state.evaluations < budget:
+            X, y = cma.ask_and_eval(problem)
+            cma.tell(X, y)
+            if problem.current_best.y <= target:
+                break
+
+    stop = perf_counter()
+    elapsed = stop - start
+
+    return elapsed, cma.countiter, problem.state.evaluations, cma.sm.count_eigen
+
+def collect_pycma():
+    logger = ioh.logger.Analyzer(
+        folder_name="pycma", 
+        algorithm_name="pycma",
+        root="data"
+    )
+    for fid in FUNCTIONS:
+        for d in DIMS:
+            problem = ioh.get_problem(fid, 1, d)
+            problem.attach_logger(logger)
+            for i in range(N_REPEATS):
+                np.seed(21 + fid * d * i)
+                modcma.utils.set_seed()
+                run_pycma(problem, np.zeros(d))
+                print("pycma", fid, d, problem.state.current_best_internal.y, problem.state.evaluations)
+                problem.reset()
+
+
+
+if __name__ == "__main__":
+    collect_pycma()
     # problem = ioh.get_problem(fid, 1, d)
     # run_modma(problem, np.zeros(d), modcma.options.CMSA)
     # print(problem.state.evaluations, problem.state.current_best_internal.y)

scripts/matrix/test_timing_modules.py

@@ -1,10 +1,12 @@
 from time import perf_counter
+import warnings
 
 import numpy as np
 import modcma.c_maes as modcma
 import ioh
 import pandas as pd 
 import matplotlib.pyplot as plt
+import cma as pycma
 
 from pprint import pprint
 
@@ -33,6 +35,28 @@ def run_modma(problem: ioh.ProblemType, x0: np.ndarray, matrix_adaptation = modc
     return elapsed, cma.p.stats.t, problem.state.evaluations, cma.p.stats.n_updates
 
 
+def run_pycma(problem: ioh.ProblemType, x0: np.ndarray, max_generations=1000):
+    options = pycma.CMAOptions()
+    options['CMA_active'] = False
+    # options['maxfevals'] = n_evaluations
+    options["verbose"] = -1
+    options["CMA_diagonal"] = False
+    # pprint(options)
+
+    cma = pycma.CMAEvolutionStrategy(x0, 2.0, options=options)
+    settings = modcma.Settings(problem.meta_data.n_variables)
+    assert settings.lambda0 == cma.sp.popsize
+    start = perf_counter()
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        for i in range(max_generations):
+            X, y = cma.ask_and_eval(problem)
+            cma.tell(X, y)
+    stop = perf_counter()
+    elapsed = stop - start
+
+    return elapsed, cma.countiter, problem.state.evaluations, cma.sm.count_eigen
+
 def collect():
     fid = 2
     dims = 2, 3, 5, 10, 20, 40, 100, 200, 500, 1000
@@ -58,5 +82,17 @@ def collect():
 
 
 if __name__ == "__main__":
-    stats = pd.read_csv("time_stats.csv")
+    fid = 2
+    dims = 2, 3, 5, 10, 20, 40, 100, 200, 500, 1000
+    n_repeats = 15
+
+    stats = []
+    for d in dims:
+        for _ in range(n_repeats):
+            problem = ioh.get_problem(fid, 1, d)
+            time, n_gen, n_evals, n_updates = run_pycma(problem, np.zeros(d))
+            stats.append(("pycma", d, time, n_gen, n_evals, n_updates))
+            print(stats[-1])
+    stats = pd.DataFrame(stats, columns=["method", "dim", "time", "n_gen", "n_evals", "n_updates"])
+    stats.to_csv("time_stats_pycma.csv")
     print(stats)

src/matrix_adaptation.cpp

@@ -37,6 +37,25 @@ namespace matrix_adaptation
 		adapt_evolution_paths_inner(pop, w, stats, settings, mu, lambda);
 	}
 
+	void None::adapt_evolution_paths_inner(const Population& pop, const Weights& w,
+		const Stats& stats, const parameters::Settings& settings, const size_t mu, const size_t lambda)
+	{
+		if (!settings.one_plus_one)
+			ps = (1.0 - w.cs) * ps + (w.sqrt_cs_mueff * dz);
+	}
+
+	Vector None::compute_y(const Vector& zi)
+	{
+		return zi;
+	}
+
+
+	Vector None::invert_y(const Vector& yi)
+	{
+		return yi;
+	}
+
+
 
 	void CovarianceAdaptation::adapt_ps(const Weights& w)
 	{
@@ -274,23 +293,6 @@ namespace matrix_adaptation
 		return M_inv * yi;
 	}
 
-	void None::adapt_evolution_paths_inner(const Population& pop, const Weights& w,
-		const Stats& stats, const parameters::Settings& settings, const size_t mu, const size_t lambda)
-	{
-		if (!settings.one_plus_one)
-			ps = (1.0 - w.cs) * ps + (w.sqrt_cs_mueff * dz);
-	}
-
-	Vector None::compute_y(const Vector& zi)
-	{
-		return zi;
-	}
-
-
-	Vector None::invert_y(const Vector& yi)
-	{
-		return yi;
-	}
 
 	void CholeskyAdaptation::adapt_evolution_paths_inner(const Population& pop,
 		const parameters::Weights& w,