From faebf5acdd5da759bf07a6449949401ee4826465 Mon Sep 17 00:00:00 2001 From: "Alexander V. Hopp" Date: Fri, 17 Jan 2025 14:10:21 +0100 Subject: [PATCH 1/3] Add Michalewicz benchmark --- benchmarks/domains/__init__.py | 2 + benchmarks/domains/synthetic_michalewicz.py | 126 ++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 benchmarks/domains/synthetic_michalewicz.py diff --git a/benchmarks/domains/__init__.py b/benchmarks/domains/__init__.py index 4a0e956a8..218a88caf 100644 --- a/benchmarks/domains/__init__.py +++ b/benchmarks/domains/__init__.py @@ -2,9 +2,11 @@ from benchmarks.definition.config import Benchmark from benchmarks.domains.synthetic_2C1D_1C import synthetic_2C1D_1C_benchmark +from benchmarks.domains.synthetic_michalewicz import synthetic_michalewicz BENCHMARKS: list[Benchmark] = [ synthetic_2C1D_1C_benchmark, + synthetic_michalewicz, ] __all__ = ["BENCHMARKS"] diff --git a/benchmarks/domains/synthetic_michalewicz.py b/benchmarks/domains/synthetic_michalewicz.py new file mode 100644 index 000000000..ca0cac51a --- /dev/null +++ b/benchmarks/domains/synthetic_michalewicz.py @@ -0,0 +1,126 @@ +"""5-dimensional Michalewicz function in a continuous space.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +from numpy import pi, sin +from pandas import DataFrame + +from baybe.campaign import Campaign +from baybe.parameters import NumericalContinuousParameter +from baybe.recommenders import RandomRecommender +from baybe.searchspace import SearchSpace +from baybe.simulation import simulate_scenarios +from baybe.targets import NumericalTarget +from benchmarks.definition import ( + Benchmark, + ConvergenceExperimentSettings, +) + + +def _lookup(arr: np.ndarray, /) -> np.ndarray: + """Numpy-based lookup callable defining the objective function.""" + try: + assert np.all((arr >= 0) & (arr <= pi)) + except AssertionError: + raise ValueError("Inputs are not in the valid ranges.") + x1, x2, x3, x4, x5 = np.array_split(arr, 5, axis=1) + + return -( + sin(x1) * sin(1 * x1**2 / pi) ** (2 * 10) + + sin(x2) * sin(2 * x2**2 / pi) ** (2 * 10) + + sin(x3) * sin(3 * x3**2 / pi) ** (2 * 10) + + sin(x4) * sin(4 * x4**2 / pi) ** (2 * 10) + + sin(x5) * sin(5 * x5**2 / pi) ** (2 * 10) + ) + + +def lookup(df: pd.DataFrame, /) -> pd.DataFrame: + """Dataframe-based lookup callable used as the loop-closing element.""" + return pd.DataFrame( + _lookup(df[["x1", "x2", "x3", "x4", "x5"]].to_numpy()), + columns=["target"], + index=df.index, + ) + + +def synthetic_michalewicz(settings: ConvergenceExperimentSettings) -> DataFrame: + """5-dimensional Michalewicz function. + + Details of the function can be found at https://www.sfu.ca/~ssurjano/michal.html + + Inputs: + x1,...,x5 continuous [0, pi] + Output: continuous + Objective: Minimization + Optimal Input: + {x1: 2.203, x2: 1.571, x3: 1.285, x4: 1.923, x5: 1.720e} + Optimal Output: -4.687658 + """ + parameters = [ + NumericalContinuousParameter(name=f"x{i}", bounds=(0, pi)) for i in range(1, 6) + ] + + target = NumericalTarget(name="target", mode="MIN") + searchspace = SearchSpace.from_product(parameters=parameters) + objective = target.to_objective() + + scenarios: dict[str, Campaign] = { + "Random Recommender": Campaign( + searchspace=searchspace, + recommender=RandomRecommender(), + objective=objective, + ), + "Default Recommender": Campaign( + searchspace=searchspace, + objective=objective, + ), + } + + return simulate_scenarios( + scenarios, + lookup, + batch_size=settings.batch_size, + n_doe_iterations=settings.n_doe_iterations, + n_mc_iterations=settings.n_mc_iterations, + impute_mode="error", + ) + + +benchmark_config = ConvergenceExperimentSettings( + batch_size=5, + n_doe_iterations=25, + n_mc_iterations=20, +) + +synthetic_michalewicz = Benchmark( + function=synthetic_michalewicz, + best_possible_result=-4.687658, + settings=benchmark_config, +) + +if __name__ == "__main__": + # Visualization of the 2-dimensional variant + + import matplotlib.pyplot as plt + from matplotlib import cm + + X1 = np.linspace(0, pi, 50) + X2 = np.linspace(0, pi, 50) + X1, X2 = np.meshgrid(X1, X2) + + # Michalewicz function + Z = -1 * ( + (np.sin(X1) * np.sin((1 * X1**2) / np.pi) ** 20) + + (np.sin(X2) * np.sin((2 * X2**2) / np.pi) ** 20) + ) + + ax = plt.figure().add_subplot(projection="3d") + surf = ax.plot_surface(X1, X2, Z, rstride=1, cstride=1, cmap=cm.jet, linewidth=0.1) + + ax.set_xlabel("x1", fontsize=10) + ax.set_ylabel("x2", fontsize=10) + ax.tick_params(axis="both", which="major", labelsize=6) + + plt.show() From fa73b0eb4a953b2b7470b914bba96b3fe1dac1ce Mon Sep 17 00:00:00 2001 From: "Alexander V. Hopp" Date: Fri, 17 Jan 2025 14:28:40 +0100 Subject: [PATCH 2/3] Fix definition of benchmark --- benchmarks/domains/__init__.py | 4 ++-- benchmarks/domains/synthetic_michalewicz.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/benchmarks/domains/__init__.py b/benchmarks/domains/__init__.py index 218a88caf..c25fc1ba4 100644 --- a/benchmarks/domains/__init__.py +++ b/benchmarks/domains/__init__.py @@ -2,11 +2,11 @@ from benchmarks.definition.config import Benchmark from benchmarks.domains.synthetic_2C1D_1C import synthetic_2C1D_1C_benchmark -from benchmarks.domains.synthetic_michalewicz import synthetic_michalewicz +from benchmarks.domains.synthetic_michalewicz import synthetic_michalewicz_benchmark BENCHMARKS: list[Benchmark] = [ synthetic_2C1D_1C_benchmark, - synthetic_michalewicz, + synthetic_michalewicz_benchmark, ] __all__ = ["BENCHMARKS"] diff --git a/benchmarks/domains/synthetic_michalewicz.py b/benchmarks/domains/synthetic_michalewicz.py index ca0cac51a..886187c09 100644 --- a/benchmarks/domains/synthetic_michalewicz.py +++ b/benchmarks/domains/synthetic_michalewicz.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import numpy as np import pandas as pd from numpy import pi, sin @@ -18,6 +20,9 @@ ConvergenceExperimentSettings, ) +if TYPE_CHECKING: + from mpl_toolkits.mplot3d import Axes3D + def _lookup(arr: np.ndarray, /) -> np.ndarray: """Numpy-based lookup callable defining the objective function.""" @@ -94,7 +99,7 @@ def synthetic_michalewicz(settings: ConvergenceExperimentSettings) -> DataFrame: n_mc_iterations=20, ) -synthetic_michalewicz = Benchmark( +synthetic_michalewicz_benchmark = Benchmark( function=synthetic_michalewicz, best_possible_result=-4.687658, settings=benchmark_config, @@ -116,7 +121,7 @@ def synthetic_michalewicz(settings: ConvergenceExperimentSettings) -> DataFrame: + (np.sin(X2) * np.sin((2 * X2**2) / np.pi) ** 20) ) - ax = plt.figure().add_subplot(projection="3d") + ax: Axes3D = plt.figure().add_subplot(projection="3d") surf = ax.plot_surface(X1, X2, Z, rstride=1, cstride=1, cmap=cm.jet, linewidth=0.1) ax.set_xlabel("x1", fontsize=10) From a92b32efff16208d22f2ed896dfba979a2693034 Mon Sep 17 00:00:00 2001 From: "Alexander V. Hopp" Date: Fri, 17 Jan 2025 14:29:12 +0100 Subject: [PATCH 3/3] Simplify plotting of 2D example --- benchmarks/domains/synthetic_michalewicz.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/domains/synthetic_michalewicz.py b/benchmarks/domains/synthetic_michalewicz.py index 886187c09..f2becd514 100644 --- a/benchmarks/domains/synthetic_michalewicz.py +++ b/benchmarks/domains/synthetic_michalewicz.py @@ -109,7 +109,6 @@ def synthetic_michalewicz(settings: ConvergenceExperimentSettings) -> DataFrame: # Visualization of the 2-dimensional variant import matplotlib.pyplot as plt - from matplotlib import cm X1 = np.linspace(0, pi, 50) X2 = np.linspace(0, pi, 50) @@ -122,7 +121,7 @@ def synthetic_michalewicz(settings: ConvergenceExperimentSettings) -> DataFrame: ) ax: Axes3D = plt.figure().add_subplot(projection="3d") - surf = ax.plot_surface(X1, X2, Z, rstride=1, cstride=1, cmap=cm.jet, linewidth=0.1) + surf = ax.plot_surface(X1, X2, Z) ax.set_xlabel("x1", fontsize=10) ax.set_ylabel("x2", fontsize=10)