Adjust benchmarking to work with incorporated feedback

benchmarking/countingones_benchmark.py

@@ -1,4 +1,5 @@
 import argparse
+import random
 from pathlib import Path
 
 import numpy as np
@@ -9,12 +10,12 @@
 
 class DEHBOptimizerCountingOnes(DEHBOptimizerBase):
     def __init__(self, dehb_params, fevals, brackets, walltime, use_ask_tell, use_restart,
-                 benchmark_name, verbose, n_continuous, n_categorical) -> None:
+                 benchmark_name, n_continuous, n_categorical) -> None:
         self.n_continuous = n_continuous
         self.n_categorical = n_categorical
         super().__init__(dehb_params=dehb_params, fevals=fevals, brackets=brackets,
                          walltime=walltime, use_ask_tell=use_ask_tell, use_restart=use_restart,
-                         benchmark_name=benchmark_name, verbose=verbose)
+                         benchmark_name=benchmark_name)
 
     def _objective_function(self, config, fidelity):
         res = self.benchmark.objective_function(config,
@@ -82,12 +83,6 @@ def input_arguments():
         default=1,
         help="Number of CPU workers for DEHB to distribute function evaluations to.",
     ),
-    parser.add_argument(
-        "--verbose",
-        action="store_true",
-        default=True,
-        help="Decides verbosity of DEHB optimization.",
-    )
     parser.add_argument(
         "--brackets",
         type=int,
@@ -137,7 +132,9 @@ def main():
     trajectories = []
     for seed in seeds:
         print(f"Running benchmark {benchmark_name} on seed {seed}")
-        dehb_params["seed"] = seed
+        np.random.seed(seed)
+        random.seed(seed)
+        dehb_params["seed"] = int(seed)
         dehb_optimizer = DEHBOptimizerCountingOnes(
             dehb_params=dehb_params,
             fevals=args.fevals,
@@ -146,7 +143,6 @@ def main():
             use_ask_tell=args.ask_tell,
             use_restart=args.restart,
             benchmark_name=benchmark_name,
-            verbose=args.verbose,
             n_continuous=args.n_continuous,
             n_categorical=args.n_categorical,
         )

benchmarking/hpobench_benchmark.py

@@ -110,12 +110,6 @@ def input_arguments():
         default=1,
         help="Number of CPU workers for DEHB to distribute function evaluations to.",
     ),
-    parser.add_argument(
-        "--verbose",
-        action="store_true",
-        default=True,
-        help="Decides verbosity of DEHB optimization.",
-    )
     parser.add_argument(
         "--brackets",
         type=int,
@@ -165,7 +159,7 @@ def main():
         trajectories = []
         for seed in seeds:
             print(f"Running benchmark {benchmark_name} on seed {seed}")
-            dehb_params["seed"] = seed
+            dehb_params["seed"] = int(seed)
             dehb_optimizer = DEHBOptimizerHPOBench(
                 dehb_params=dehb_params,
                 fevals=args.fevals,
@@ -174,7 +168,6 @@ def main():
                 use_ask_tell=args.ask_tell,
                 use_restart=args.restart,
                 benchmark_name=benchmark_name,
-                verbose=args.verbose,
             )
             traj = dehb_optimizer.run()
             trajectories.append(traj)

benchmarking/mfpbench_benchmark.py

@@ -73,12 +73,6 @@ def input_arguments():
         default=1,
         help="Number of CPU workers for DEHB to distribute function evaluations to.",
     ),
-    parser.add_argument(
-        "--verbose",
-        action="store_true",
-        default=False,
-        help="Decides verbosity of DEHB optimization.",
-    )
     parser.add_argument(
         "--brackets",
         type=int,
@@ -128,7 +122,7 @@ def main():
         trajectories = []
         for seed in seeds:
             print(f"Running benchmark {benchmark_name} on seed {seed}")
-            dehb_params["seed"] = seed
+            dehb_params["seed"] = int(seed)
             dehb_optimizer = DEHBOptimizerMFPBench(
                 dehb_params=dehb_params,
                 fevals=args.fevals,
@@ -137,7 +131,6 @@ def main():
                 use_ask_tell=args.ask_tell,
                 use_restart=args.restart,
                 benchmark_name=benchmark_name,
-                verbose=args.verbose,
             )
             traj = dehb_optimizer.run()
             trajectories.append(traj)

benchmarking/utils.py

@@ -48,8 +48,7 @@ def create_table_for_benchmark(results: dict) -> list:
 
 class DEHBOptimizerBase():
     def __init__(self, dehb_params, fevals, brackets, walltime, use_ask_tell, use_restart,
-                 benchmark_name, verbose) -> None:
-        self.verbose = verbose
+                 benchmark_name) -> None:
         self.fevals = fevals
         self.brackets = brackets
         self.walltime = walltime
@@ -104,7 +103,7 @@ def _run_for(self, fevals=None, brackets=None, walltime=None):
             return self.dehb.traj
 
         traj,_ ,_ = self.dehb.run(fevals=fevals, brackets=brackets,
-                                  total_cost=walltime, verbose=self.verbose)
+                                  total_cost=walltime)
         return traj
     def _run_with_restart(self):
         pre_restart_brackets, post_restart_brackets = None, None

pyproject.toml

@@ -11,7 +11,7 @@ dependencies = [
   "dask>=2.27.0",
   "distributed>=2.27.0",
   "ConfigSpace>=0.4.16",
-  "pandas>=1.4.4",
+  "pandas>=1.2.4",
   "pyarrow>=16.1.0"
 ]
 classifiers = [