From 1fe2cad15b2c3900b7497ebd0d78d3129deec5e4 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 13 Dec 2024 12:47:44 +0100 Subject: [PATCH 01/26] Add test case for switching hysteresis --- baybe/recommenders/meta/sequential.py | 7 +++++-- tests/test_meta_recommenders.py | 28 +++++++++++++++++++-------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index 431b8a049..463667a54 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -49,8 +49,11 @@ class TwoPhaseMetaRecommender(MetaRecommender): """The recommender used by the meta recommender after the switch.""" switch_after: int = field(default=1) - """The number of experiments after which the recommender is switched for the next - requested batch.""" + """The number of experiments required for the recommender to switch.""" + + remain_switched: bool = False + """Determines if the recommender should remain switched even if the number of + experiments falls below the threshold value in subsequent calls.""" @override def select_recommender( diff --git a/tests/test_meta_recommenders.py b/tests/test_meta_recommenders.py index 26d66a33b..f8c0dc70b 100644 --- a/tests/test_meta_recommenders.py +++ b/tests/test_meta_recommenders.py @@ -16,23 +16,35 @@ RECOMMENDERS = [RandomRecommender(), FPSRecommender(), BotorchRecommender()] -def test_twophase_meta_recommender(): - """The recommender switches the recommender at the requested point.""" +@pytest.mark.parametrize("remain_switched", [False, True]) +def test_twophase_meta_recommender(remain_switched): + """The recommender switches the recommender at the requested point and + remains / reverts the switch depending on the configuration.""" # noqa + # Cross the switching point forwards and backwards + switch_after = 3 + training_sizes = [2, 3, 2] + + # Recommender objects initial_recommender = RandomRecommender() subsequent_recommender = RandomRecommender() - switch_after = 3 recommender = TwoPhaseMetaRecommender( initial_recommender=initial_recommender, recommender=subsequent_recommender, switch_after=switch_after, + remain_switched=remain_switched, ) - for training_size in range(6): - rec = select_recommender(recommender, training_size) + + # Query the meta recommender + switch_point_passed = False + for n_data in training_sizes: + rec = select_recommender(recommender, n_data) target = ( - initial_recommender - if training_size < switch_after - else subsequent_recommender + subsequent_recommender + if (n_data >= switch_after) or (switch_point_passed and remain_switched) + else initial_recommender ) + if not switch_point_passed and n_data >= switch_after: + switch_point_passed = True assert rec is target From 2008ba7d70ba43b129c0076378efa8f299b8ca38 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 13 Dec 2024 12:53:58 +0100 Subject: [PATCH 02/26] Adjust logic to make the hysteresis test pass --- baybe/recommenders/meta/sequential.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index 463667a54..13f3b8d78 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -55,6 +55,9 @@ class TwoPhaseMetaRecommender(MetaRecommender): """Determines if the recommender should remain switched even if the number of experiments falls below the threshold value in subsequent calls.""" + _has_switched: bool = False + """Indicates if the switch has already occurred.""" + @override def select_recommender( self, @@ -64,11 +67,14 @@ def select_recommender( measurements: pd.DataFrame | None = None, pending_experiments: pd.DataFrame | None = None, ) -> PureRecommender: - return ( - self.recommender - if (measurements is not None) and (len(measurements) >= self.switch_after) - else self.initial_recommender - ) + n_data = len(measurements) if measurements is not None else 0 + if (n_data >= self.switch_after) or ( + self._has_switched and self.remain_switched + ): + self._has_switched = True + return self.recommender + + return self.initial_recommender @override def __str__(self) -> str: From 36ef187375c83f0fd07088495543d15bdb84915b Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 13 Dec 2024 16:54:52 +0100 Subject: [PATCH 03/26] Update __str__ method --- baybe/recommenders/meta/sequential.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index 13f3b8d78..94d4824db 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -82,6 +82,8 @@ def __str__(self) -> str: to_string("Initial recommender", self.initial_recommender), to_string("Recommender", self.recommender), to_string("Switch after", self.switch_after, single_line=True), + to_string("Remain switched", self.remain_switched, single_line=True), + to_string("Has switched", self._has_switched, single_line=True), ] return to_string(self.__class__.__name__, *fields) From 87ccd41be37def28872e78d55bf8614d06495b46 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Tue, 17 Dec 2024 09:24:44 +0100 Subject: [PATCH 04/26] Make arguments to MetaRecommender.select_recommender optional --- CHANGELOG.md | 1 + baybe/recommenders/meta/base.py | 4 ++-- baybe/recommenders/meta/sequential.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a04d1fb5..1f1753838 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `CustomDiscreteParameter` does not allow duplicated rows in `data` anymore - De-/activating Polars via `BAYBE_DEACTIVATE_POLARS` now requires passing values compatible with `strtobool` +- All arguments to `MetaRecommender.select_recommender` are now optional ### Fixed - Rare bug arising from degenerate `SubstanceParameter.comp_df` rows that caused diff --git a/baybe/recommenders/meta/base.py b/baybe/recommenders/meta/base.py index 6c39c11db..5d16d5231 100644 --- a/baybe/recommenders/meta/base.py +++ b/baybe/recommenders/meta/base.py @@ -31,8 +31,8 @@ class MetaRecommender(SerialMixin, RecommenderProtocol, ABC): @abstractmethod def select_recommender( self, - batch_size: int, - searchspace: SearchSpace, + batch_size: int | None = None, + searchspace: SearchSpace | None = None, objective: Objective | None = None, measurements: pd.DataFrame | None = None, pending_experiments: pd.DataFrame | None = None, diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index 94d4824db..c5c69501d 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -61,7 +61,7 @@ class TwoPhaseMetaRecommender(MetaRecommender): @override def select_recommender( self, - batch_size: int, + batch_size: int | None = None, searchspace: SearchSpace | None = None, objective: Objective | None = None, measurements: pd.DataFrame | None = None, @@ -146,7 +146,7 @@ class SequentialMetaRecommender(MetaRecommender): @override def select_recommender( self, - batch_size: int, + batch_size: int | None = None, searchspace: SearchSpace | None = None, objective: Objective | None = None, measurements: pd.DataFrame | None = None, @@ -236,7 +236,7 @@ def default_iterator(self): @override def select_recommender( self, - batch_size: int, + batch_size: int | None = None, searchspace: SearchSpace | None = None, objective: Objective | None = None, measurements: pd.DataFrame | None = None, From 8d0a84049f5c35bb0863e5d849f09af56442aad6 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Tue, 17 Dec 2024 20:57:15 +0100 Subject: [PATCH 05/26] Add Partition class --- baybe/utils/interval.py | 28 +++++++++++++++++++ .../utils/test_partition_validation.py | 20 +++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 tests/validation/utils/test_partition_validation.py diff --git a/baybe/utils/interval.py b/baybe/utils/interval.py index 4a382fddf..098990de2 100644 --- a/baybe/utils/interval.py +++ b/baybe/utils/interval.py @@ -3,10 +3,13 @@ import gc from collections.abc import Iterable from functools import singledispatchmethod +from itertools import pairwise from typing import TYPE_CHECKING, Any +import cattrs import numpy as np from attrs import define, field +from attrs.validators import min_len from baybe.serialization import SerialMixin, converter from baybe.utils.numerical import DTypeFloatNumpy @@ -149,6 +152,31 @@ def use_fallback_constructor_hook(value: Any, cls: type[Interval]) -> Interval: return Interval.create(value) +@define +class Partition: + """A partition of the real number line into right-open intervals.""" + + thresholds: tuple[float, ...] = field( + converter=lambda x: cattrs.structure(x, tuple[float, ...]), validator=min_len(1) + ) + """The thresholds separating the real number line into intervals.""" + + @thresholds.validator + def _validate_thresholds(self, _, value): + if not all(x < y for x, y in pairwise(value)): + raise ValueError("Thresholds must be strictly monotonically increasing.") + + def get_interval_index(self, value: float, /) -> int: + """Return the index of the interval containing the given value.""" + return next( + i for i, v in enumerate((*self.thresholds, float("inf"))) if value < v + ) + + def __len__(self) -> int: + """Return the number of intervals defined by the partition.""" + return len(self.thresholds) + 1 + + # Register structure hooks converter.register_structure_hook(Interval, use_fallback_constructor_hook) diff --git a/tests/validation/utils/test_partition_validation.py b/tests/validation/utils/test_partition_validation.py new file mode 100644 index 000000000..2a18f01b4 --- /dev/null +++ b/tests/validation/utils/test_partition_validation.py @@ -0,0 +1,20 @@ +"""Validation tests for partitions.""" + +import pytest +from pytest import param + +from baybe.utils.interval import Partition + + +@pytest.mark.parametrize( + ("thresholds", "error", "match"), + [ + param(1, TypeError, None, id="not_iterable"), + param([], ValueError, None, id="too_short"), + param([1, 0], ValueError, "monotonically increasing", id="decreasing"), + ], +) +def test_invalid_thresholds(thresholds, error, match): + """Providing invalid thresholds raises an error.""" + with pytest.raises(error, match=match): + Partition(thresholds) From 42d038ab40aabe4034ae486160022eab9eec5e46 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Tue, 17 Dec 2024 20:58:02 +0100 Subject: [PATCH 06/26] Add BatchSizeControlledMetaRecommender class --- baybe/recommenders/meta/adaptive.py | 48 +++++++++++++++++++++++++++++ tests/test_meta_recommenders.py | 11 +++++++ 2 files changed, 59 insertions(+) create mode 100644 baybe/recommenders/meta/adaptive.py diff --git a/baybe/recommenders/meta/adaptive.py b/baybe/recommenders/meta/adaptive.py new file mode 100644 index 000000000..bd6fabfce --- /dev/null +++ b/baybe/recommenders/meta/adaptive.py @@ -0,0 +1,48 @@ +"""Meta recommenders that adaptively select recommenders based on the context.""" + +import pandas as pd +from attrs import define, field +from attrs.validators import deep_iterable, instance_of +from typing_extensions import override + +from baybe.objectives.base import Objective +from baybe.recommenders.meta.base import MetaRecommender +from baybe.recommenders.pure.base import PureRecommender +from baybe.searchspace.core import SearchSpace +from baybe.utils.interval import Partition + + +@define +class BatchSizeControlledMetaRecommender(MetaRecommender): + """A meta recommender that selects recommenders according to the batch size.""" + + recommenders: list[PureRecommender] = field( + converter=list, validator=deep_iterable(instance_of(PureRecommender)) + ) + """The recommenders for the individual batch size intervals.""" + + partition: Partition = field( + converter=lambda x: Partition(x) if not isinstance(x, Partition) else x + ) + """The partition mapping batch size intervals to recommenders. """ + + @partition.validator + def _validate_partitioning(self, _, value): + if (lr := len(self.recommenders)) != (lp := len(value)): + raise ValueError( + f"The number of recommenders (given: {lr}) must be equal to the number " + f"of intervals defined by the partition (given: {lp})." + ) + + @override + def select_recommender( + self, + batch_size: int | None = None, + searchspace: SearchSpace | None = None, + objective: Objective | None = None, + measurements: pd.DataFrame | None = None, + pending_experiments: pd.DataFrame | None = None, + ) -> PureRecommender: + if batch_size is None: + raise ValueError("A batch size is required.") + return self.recommenders[self.partition.get_interval_index(batch_size)] diff --git a/tests/test_meta_recommenders.py b/tests/test_meta_recommenders.py index f8c0dc70b..23329db8a 100644 --- a/tests/test_meta_recommenders.py +++ b/tests/test_meta_recommenders.py @@ -11,6 +11,7 @@ StreamingSequentialMetaRecommender, TwoPhaseMetaRecommender, ) +from baybe.recommenders.meta.adaptive import BatchSizeControlledMetaRecommender from tests.conftest import select_recommender RECOMMENDERS = [RandomRecommender(), FPSRecommender(), BotorchRecommender()] @@ -117,3 +118,13 @@ def test_streaming_sequential_meta_recommender(recommenders): # Selection with smaller training size raises an error with pytest.raises(RuntimeError): select_recommender(meta_recommender, training_size - 1) + + +def test_batch_size_controlled_meta_recommender(): + """The recommender retrieves the right recommender for the requested batch size.""" + thresholds = [2, 5] + meta_recommender = BatchSizeControlledMetaRecommender(RECOMMENDERS, thresholds) + for i, threshold in enumerate(thresholds): + assert meta_recommender.select_recommender(threshold - 1) is RECOMMENDERS[i] + assert meta_recommender.select_recommender(threshold) is RECOMMENDERS[i + 1] + assert meta_recommender.select_recommender(threshold + 1) is RECOMMENDERS[i + 1] From b947bc2dc7d118825aea8c253ac191ac7612c0e6 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Wed, 18 Dec 2024 13:05:49 +0100 Subject: [PATCH 07/26] Remove current state logic of meta recommenders --- baybe/recommenders/meta/base.py | 65 ++------------------------------- 1 file changed, 3 insertions(+), 62 deletions(-) diff --git a/baybe/recommenders/meta/base.py b/baybe/recommenders/meta/base.py index 5d16d5231..259ad1b1e 100644 --- a/baybe/recommenders/meta/base.py +++ b/baybe/recommenders/meta/base.py @@ -6,7 +6,7 @@ import cattrs import pandas as pd -from attrs import define, field +from attrs import define from typing_extensions import override from baybe.objectives.base import Objective @@ -22,12 +22,6 @@ class MetaRecommender(SerialMixin, RecommenderProtocol, ABC): """Abstract base class for all meta recommenders.""" - _current_recommender: PureRecommender | None = field(default=None, init=False) - """The current recommender.""" - - _used_recommender_ids: set[int] = field(factory=set, init=False) - """Set of ids from recommenders that were used by this meta recommender.""" - @abstractmethod def select_recommender( self, @@ -43,56 +37,6 @@ def select_recommender( on the method arguments. """ - def get_current_recommender(self) -> PureRecommender: - """Get the current recommender, if available.""" - if self._current_recommender is None: - raise RuntimeError( - f"No recommendation has been requested from the " - f"'{self.__class__.__name__}' yet. Because the recommender is a " - f"'{MetaRecommender.__name__}', this means no actual recommender has " - f"been selected so far. The recommender will be available after the " - f"next '{self.recommend.__name__}' call." - ) - return self._current_recommender - - def get_next_recommender( - self, - batch_size: int, - searchspace: SearchSpace, - objective: Objective | None = None, - measurements: pd.DataFrame | None = None, - pending_experiments: pd.DataFrame | None = None, - ) -> PureRecommender: - """Get the recommender for the next recommendation. - - Returns the next recommender in row that has not yet been used for generating - recommendations. In case of multiple consecutive calls, this means that - the same recommender instance is returned until its :meth:`recommend` method - is called. - - See :meth:`baybe.recommenders.base.RecommenderProtocol.recommend` for details - on the method arguments. - """ - # Check if the stored recommender instance can be returned - if ( - self._current_recommender is not None - and id(self._current_recommender) not in self._used_recommender_ids - ): - recommender = self._current_recommender - - # Otherwise, fetch the next recommender waiting in row - else: - recommender = self.select_recommender( - batch_size=batch_size, - searchspace=searchspace, - objective=objective, - measurements=measurements, - pending_experiments=pending_experiments, - ) - self._current_recommender = recommender - - return recommender - @override def recommend( self, @@ -103,7 +47,7 @@ def recommend( pending_experiments: pd.DataFrame | None = None, ) -> pd.DataFrame: """See :meth:`baybe.recommenders.base.RecommenderProtocol.recommend`.""" - recommender = self.get_next_recommender( + recommender = self.select_recommender( batch_size=batch_size, searchspace=searchspace, objective=objective, @@ -123,15 +67,12 @@ def recommend( } ) - recommendations = recommender.recommend( + return recommender.recommend( batch_size=batch_size, searchspace=searchspace, pending_experiments=pending_experiments, **optional_args, ) - self._used_recommender_ids.add(id(recommender)) - - return recommendations # Register (un-)structure hooks From f70a7aade32ab0c7ce90d4743b8febd72aa3a84d Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 20 Dec 2024 15:39:41 +0100 Subject: [PATCH 08/26] Reimplement SequentialMetaRecommender logic and test --- baybe/recommenders/meta/sequential.py | 70 ++++++++++++++++++--------- tests/test_meta_recommenders.py | 30 +++++++++--- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index c5c69501d..4b083095b 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -8,7 +8,7 @@ from typing import Literal import pandas as pd -from attrs import define, field +from attrs import define, field, fields from attrs.validators import deep_iterable, in_, instance_of from typing_extensions import override @@ -137,11 +137,23 @@ class SequentialMetaRecommender(MetaRecommender): # with `_cattrs_include_init_false=True`. However, the way # `get_base_structure_hook` is currently designed prevents such a hook from # taking action. - _step: int = field(default=-1, alias="_step") - """Counts how often the recommender has already been switched.""" + _step: int = field(default=0, alias="_step") + """An index pointing to the current position in the recommender sequence.""" - _n_last_measurements: int = field(default=-1, alias="_n_last_measurements") - """The number of measurements that were available at the last call.""" + _was_used: bool = field(default=False) + """Boolean flag indicating if the current recommender has been used.""" + + _n_last_measurements: int = field(default=0, alias="_n_last_measurements") + """The number of measurements available at the last successful recommend call.""" + + def _get_recommender_at_current_step(self) -> PureRecommender: + """Get the recommender at the current sequence position.""" + idx = self._step + if self.mode == "reuse_last": + idx = min(idx, len(self.recommenders) - 1) + elif self.mode == "cyclic": + idx %= len(self.recommenders) + return self.recommenders[idx] @override def select_recommender( @@ -152,40 +164,52 @@ def select_recommender( measurements: pd.DataFrame | None = None, pending_experiments: pd.DataFrame | None = None, ) -> PureRecommender: - n_data = len(measurements) if measurements is not None else 0 - - # If the training dataset size has increased, move to the next recommender - if n_data > self._n_last_measurements: - self._step += 1 - # If the training dataset size has decreased, something went wrong - elif n_data < self._n_last_measurements: + if ( + n_data := len(measurements) if measurements is not None else 0 + ) < self._n_last_measurements: raise RuntimeError( f"The training dataset size decreased from {self._n_last_measurements} " f"to {n_data} since the last function call, which indicates that " f"'{self.__class__.__name__}' was not used as intended." ) - # Get the right index for the "next" recommender - idx = self._step - if self.mode == "reuse_last": - idx = min(idx, len(self.recommenders) - 1) - elif self.mode == "cyclic": - idx %= len(self.recommenders) + # Check conditions if the next recommender in sequence is required + more_data = n_data > self._n_last_measurements + if (not self._was_used) or (not more_data): + return self._get_recommender_at_current_step() - # Get the recommender + # Move on to the next recommender + self._step += 1 + self._was_used = False try: - recommender = self.recommenders[idx] + return self._get_recommender_at_current_step() except IndexError as ex: raise NoRecommendersLeftError( f"A total of {self._step+1} recommender(s) was/were requested but the " - f"provided sequence contains only {self._step} element(s)." + f"provided sequence contains only {self._step} element(s). " + f"Add more recommenders or adjust the " + f"'{fields(SequentialMetaRecommender).mode.name}' attribute." ) from ex + @override + def recommend( + self, + batch_size: int, + searchspace: SearchSpace, + objective: Objective | None = None, + measurements: pd.DataFrame | None = None, + pending_experiments: pd.DataFrame | None = None, + ) -> pd.DataFrame: + recommendation = super().recommend( + batch_size, searchspace, objective, measurements, pending_experiments + ) + self._was_used = True + # Remember the training dataset size for the next call - self._n_last_measurements = n_data + self._n_last_measurements = len(measurements) if measurements is not None else 0 - return recommender + return recommendation @override def __str__(self) -> str: diff --git a/tests/test_meta_recommenders.py b/tests/test_meta_recommenders.py index 23329db8a..1f5113d7b 100644 --- a/tests/test_meta_recommenders.py +++ b/tests/test_meta_recommenders.py @@ -50,30 +50,43 @@ def test_twophase_meta_recommender(remain_switched): @pytest.mark.parametrize("mode", ["raise", "reuse_last", "cyclic"]) -@pytest.mark.parametrize("recommenders", [RECOMMENDERS]) -def test_sequential_meta_recommender(recommenders, mode): +def test_sequential_meta_recommender(mode): """The recommender provides its recommenders in the right order.""" - meta_recommender = SequentialMetaRecommender(recommenders=recommenders, mode=mode) + meta_recommender = SequentialMetaRecommender(recommenders=RECOMMENDERS, mode=mode) training_size = 0 # First iteration over provided recommender sequence - for reference in recommenders: + for reference in RECOMMENDERS: training_size += 1 # The returned recommender coincides with what was put in recommender = select_recommender(meta_recommender, training_size) assert recommender is reference + # If the current recommender was not used (by recommending via the meta + # recommender), a subsequent call returns the same recommender + recommender = select_recommender(meta_recommender, training_size + 1) + assert recommender is reference + + # Pretend the previous data size increase did not happen + meta_recommender._n_last_measurements = training_size + + # Pretend the recommender was used + meta_recommender._was_used = True + # Selection with unchanged training size yields again the same recommender recommender = select_recommender(meta_recommender, training_size) assert recommender is reference # Selection with smaller training size raises an error - with pytest.raises(RuntimeError): + with pytest.raises( + RuntimeError, + match=f"decreased from {training_size} to {training_size-1}", + ): select_recommender(meta_recommender, training_size - 1) # Second iteration over provided recommender sequence - for cycled in recommenders: + for cycled in RECOMMENDERS: training_size += 1 if mode == "raise": @@ -84,13 +97,16 @@ def test_sequential_meta_recommender(recommenders, mode): elif mode == "reuse_last": # The last recommender is selected repeatedly recommender = select_recommender(meta_recommender, training_size) - assert recommender == recommenders[-1] + assert recommender == RECOMMENDERS[-1] elif mode == "cyclic": # The selection restarts from the first recommender recommender = select_recommender(meta_recommender, training_size) assert recommender == cycled + # Pretend the recommender was used + meta_recommender._was_used = True + @pytest.mark.parametrize( "recommenders", From a684662c15eb7541909094f72fb112d5891a30fb Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 20 Dec 2024 16:30:41 +0100 Subject: [PATCH 09/26] Reimplement StreamingSequentialMetaRecommender Extract common logic into new base class --- baybe/recommenders/meta/sequential.py | 189 ++++++++++++-------------- tests/test_meta_recommenders.py | 57 ++++---- 2 files changed, 109 insertions(+), 137 deletions(-) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index 4b083095b..54ae015b9 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -4,6 +4,7 @@ # mypy: disable-error-code="arg-type" import gc +from abc import abstractmethod from collections.abc import Iterable, Iterator from typing import Literal @@ -89,71 +90,27 @@ def __str__(self) -> str: @define -class SequentialMetaRecommender(MetaRecommender): - """A meta recommender that uses a pre-defined sequence of recommenders. - - A new recommender is taken from the sequence whenever at least one new measurement - is available, until all recommenders are exhausted. More precisely, a recommender - change is triggered whenever the size of the training dataset increases; the - actual content of the dataset is ignored. - - Note: - The provided sequence of recommenders will be internally pre-collected into a - list. If this is not acceptable, consider using - :class:`baybe.recommenders.meta.sequential.StreamingSequentialMetaRecommender` - instead. - - Raises: - RuntimeError: If the training dataset size decreased compared to the previous - call. - NoRecommendersLeftError: If more recommenders are requested than there are - recommenders available and ``mode="raise"``. - """ - - # Exposed - recommenders: list[PureRecommender] = field( - converter=list, validator=deep_iterable(instance_of(PureRecommender)) - ) - """A finite-length sequence of recommenders to be used. For infinite-length - iterables, see - :class:`baybe.recommenders.meta.sequential.StreamingSequentialMetaRecommender`.""" - - mode: Literal["raise", "reuse_last", "cyclic"] = field( - default="raise", - validator=in_(("raise", "reuse_last", "cyclic")), - ) - """Defines what shall happen when the last recommender in the sequence has been - consumed but additional recommender changes are triggered: - - * ``"raise"``: An error is raised. - * ``"reuse_last"``: The last recommender in the sequence is used indefinitely. - * ``"cycle"``: The selection restarts from the beginning of the sequence. - """ - - # Private +class _BaseSequentialMetaRecommender(MetaRecommender): # TODO: These should **not** be exposed via the constructor but the workaround # is currently needed for correct (de-)serialization. A proper approach would be # to not set them via the constructor but through a custom hook in combination # with `_cattrs_include_init_false=True`. However, the way # `get_base_structure_hook` is currently designed prevents such a hook from # taking action. - _step: int = field(default=0, alias="_step") + _step: int = field(default=0, alias="_step", kw_only=True) """An index pointing to the current position in the recommender sequence.""" - _was_used: bool = field(default=False) + _was_used: bool = field(default=False, alias="_was_used", kw_only=True) """Boolean flag indicating if the current recommender has been used.""" - _n_last_measurements: int = field(default=0, alias="_n_last_measurements") + _n_last_measurements: int = field( + default=0, alias="_n_last_measurements", kw_only=True + ) """The number of measurements available at the last successful recommend call.""" + @abstractmethod def _get_recommender_at_current_step(self) -> PureRecommender: """Get the recommender at the current sequence position.""" - idx = self._step - if self.mode == "reuse_last": - idx = min(idx, len(self.recommenders) - 1) - elif self.mode == "cyclic": - idx %= len(self.recommenders) - return self.recommenders[idx] @override def select_recommender( @@ -182,15 +139,7 @@ def select_recommender( # Move on to the next recommender self._step += 1 self._was_used = False - try: - return self._get_recommender_at_current_step() - except IndexError as ex: - raise NoRecommendersLeftError( - f"A total of {self._step+1} recommender(s) was/were requested but the " - f"provided sequence contains only {self._step} element(s). " - f"Add more recommenders or adjust the " - f"'{fields(SequentialMetaRecommender).mode.name}' attribute." - ) from ex + return self._get_recommender_at_current_step() @override def recommend( @@ -211,6 +160,65 @@ def recommend( return recommendation + +@define +class SequentialMetaRecommender(_BaseSequentialMetaRecommender): + """A meta recommender that uses a pre-defined sequence of recommenders. + + A new recommender is taken from the sequence whenever at least one new measurement + is available, until all recommenders are exhausted. More precisely, a recommender + change is triggered whenever the size of the training dataset increases; the + actual content of the dataset is ignored. + + Note: + The provided sequence of recommenders will be internally pre-collected into a + list. If this is not acceptable, consider using + :class:`baybe.recommenders.meta.sequential.StreamingSequentialMetaRecommender` + instead. + + Raises: + RuntimeError: If the training dataset size decreased compared to the previous + call. + NoRecommendersLeftError: If more recommenders are requested than there are + recommenders available and ``mode="raise"``. + """ + + recommenders: list[PureRecommender] = field( + converter=list, validator=deep_iterable(instance_of(PureRecommender)) + ) + """A finite-length sequence of recommenders to be used. For infinite-length + iterables, see + :class:`baybe.recommenders.meta.sequential.StreamingSequentialMetaRecommender`.""" + + mode: Literal["raise", "reuse_last", "cyclic"] = field( + default="raise", + validator=in_(("raise", "reuse_last", "cyclic")), + ) + """Defines what shall happen when the last recommender in the sequence has been + consumed but additional recommender changes are triggered: + + * ``"raise"``: An error is raised. + * ``"reuse_last"``: The last recommender in the sequence is used indefinitely. + * ``"cycle"``: The selection restarts from the beginning of the sequence. + """ + + @override + def _get_recommender_at_current_step(self) -> PureRecommender: + idx = self._step + if self.mode == "reuse_last": + idx = min(idx, len(self.recommenders) - 1) + elif self.mode == "cyclic": + idx %= len(self.recommenders) + try: + return self.recommenders[idx] + except IndexError as ex: + raise NoRecommendersLeftError( + f"A total of {self._step+1} recommender(s) was/were requested but " + f"the provided sequence contains only {self._step} element(s). " + f"Add more recommenders or adjust the " + f"'{fields(SequentialMetaRecommender).mode.name}' attribute." + ) from ex + @override def __str__(self) -> str: fields = [ @@ -221,7 +229,7 @@ def __str__(self) -> str: @define -class StreamingSequentialMetaRecommender(MetaRecommender): +class StreamingSequentialMetaRecommender(_BaseSequentialMetaRecommender): """A meta recommender that switches between recommenders from an iterable. Similar to :class:`baybe.recommenders.meta.sequential.SequentialMetaRecommender` @@ -234,68 +242,39 @@ class StreamingSequentialMetaRecommender(MetaRecommender): recommenders available. """ - # Exposed recommenders: Iterable[PureRecommender] = field() """An iterable providing the recommenders to be used.""" - # Private - # TODO: See :class:`baybe.recommenders.meta.sequential.SequentialMetaRecommender` - _step: int = field(init=False, default=-1) - """Counts how often the recommender has already been switched.""" - - _n_last_measurements: int = field(init=False, default=-1) - """The number of measurements that were available at the last call.""" - _iterator: Iterator = field(init=False) """The iterator used to traverse the recommenders.""" _last_recommender: PureRecommender | None = field(init=False, default=None) """The recommender returned from the last call.""" + _step_of_last_recommender: int = field(init=False, default=-1) + """The position of the latest recommender fetched from the iterable.""" + @_iterator.default def default_iterator(self): """Initialize the recommender iterator.""" return iter(self.recommenders) @override - def select_recommender( - self, - batch_size: int | None = None, - searchspace: SearchSpace | None = None, - objective: Objective | None = None, - measurements: pd.DataFrame | None = None, - pending_experiments: pd.DataFrame | None = None, - ) -> PureRecommender: - use_last = True - n_data = len(measurements) if measurements is not None else 0 - - # If the training dataset size has increased, move to the next recommender - if n_data > self._n_last_measurements: - self._step += 1 - use_last = False - - # If the training dataset size has decreased, something went wrong - elif n_data < self._n_last_measurements: - raise RuntimeError( - f"The training dataset size decreased from {self._n_last_measurements} " - f"to {n_data} since the last function call, which indicates that " - f"'{self.__class__.__name__}' was not used as intended." - ) - - # Get the recommender - try: - if not use_last: + def _get_recommender_at_current_step(self) -> PureRecommender: + if self._step != self._step_of_last_recommender: + try: self._last_recommender = next(self._iterator) - except StopIteration as ex: - raise NoRecommendersLeftError( - f"A total of {self._step+1} recommender(s) was/were requested but the " - f"provided iterator provided only {self._step} element(s)." - ) from ex + except StopIteration as ex: + raise NoRecommendersLeftError( + f"A total of {self._step+1} recommender(s) was/were requested but " + f"the provided iterator provided only {self._step} element(s). " + ) from ex + self._step_of_last_recommender = self._step - # Remember the training dataset size for the next call - self._n_last_measurements = n_data + # By now, the first recommender has been fetched + assert self._last_recommender is not None - return self._last_recommender # type: ignore[return-value] + return self._last_recommender @override def __str__(self) -> str: diff --git a/tests/test_meta_recommenders.py b/tests/test_meta_recommenders.py index 1f5113d7b..7226506f9 100644 --- a/tests/test_meta_recommenders.py +++ b/tests/test_meta_recommenders.py @@ -8,10 +8,10 @@ FPSRecommender, RandomRecommender, SequentialMetaRecommender, - StreamingSequentialMetaRecommender, TwoPhaseMetaRecommender, ) from baybe.recommenders.meta.adaptive import BatchSizeControlledMetaRecommender +from baybe.recommenders.meta.sequential import StreamingSequentialMetaRecommender from tests.conftest import select_recommender RECOMMENDERS = [RandomRecommender(), FPSRecommender(), BotorchRecommender()] @@ -49,10 +49,27 @@ def test_twophase_meta_recommender(remain_switched): assert rec is target -@pytest.mark.parametrize("mode", ["raise", "reuse_last", "cyclic"]) -def test_sequential_meta_recommender(mode): +@pytest.mark.parametrize( + ("cls", "mode"), + [ + (SequentialMetaRecommender, "raise"), + (SequentialMetaRecommender, "reuse_last"), + (SequentialMetaRecommender, "cyclic"), + (StreamingSequentialMetaRecommender, None), + ], +) +def test_sequential_meta_recommender(cls, mode): """The recommender provides its recommenders in the right order.""" - meta_recommender = SequentialMetaRecommender(recommenders=RECOMMENDERS, mode=mode) + # Create meta recommender + if cls == SequentialMetaRecommender: + meta_recommender = SequentialMetaRecommender( + recommenders=RECOMMENDERS, mode=mode + ) + else: + meta_recommender = StreamingSequentialMetaRecommender( + recommenders=(r for r in RECOMMENDERS) # <-- generator comprehension + ) + training_size = 0 # First iteration over provided recommender sequence @@ -85,6 +102,10 @@ def test_sequential_meta_recommender(mode): ): select_recommender(meta_recommender, training_size - 1) + # For streaming recommenders, no second iteration is possible + if cls == StreamingSequentialMetaRecommender: + return + # Second iteration over provided recommender sequence for cycled in RECOMMENDERS: training_size += 1 @@ -108,34 +129,6 @@ def test_sequential_meta_recommender(mode): meta_recommender._was_used = True -@pytest.mark.parametrize( - "recommenders", - [ - RECOMMENDERS, # list - (rec for rec in RECOMMENDERS), # generator - ], -) -def test_streaming_sequential_meta_recommender(recommenders): - """The recommender provides its recommenders in the right order.""" - meta_recommender = StreamingSequentialMetaRecommender(recommenders=recommenders) - training_size = 0 - - for reference in RECOMMENDERS: - training_size += 1 - - # The returned recommender coincides with what was put in - recommender = select_recommender(meta_recommender, training_size) - assert recommender is reference - - # Selection with unchanged training size yields again the same recommender - recommender = select_recommender(meta_recommender, training_size) - assert recommender is reference - - # Selection with smaller training size raises an error - with pytest.raises(RuntimeError): - select_recommender(meta_recommender, training_size - 1) - - def test_batch_size_controlled_meta_recommender(): """The recommender retrieves the right recommender for the requested batch size.""" thresholds = [2, 5] From 0741be1d45e294ff7bf2abcf66fe7b9ad9292b5d Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 20 Dec 2024 17:24:43 +0100 Subject: [PATCH 10/26] Improve docstrings --- baybe/recommenders/meta/sequential.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index 54ae015b9..03807eda4 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -197,9 +197,9 @@ class SequentialMetaRecommender(_BaseSequentialMetaRecommender): """Defines what shall happen when the last recommender in the sequence has been consumed but additional recommender changes are triggered: - * ``"raise"``: An error is raised. - * ``"reuse_last"``: The last recommender in the sequence is used indefinitely. - * ``"cycle"``: The selection restarts from the beginning of the sequence. + * ``"raise"``: An error is raised. + * ``"reuse_last"``: The last recommender in the sequence is used indefinitely. + * ``"cycle"``: The selection restarts from the beginning of the sequence. """ @override @@ -233,9 +233,15 @@ class StreamingSequentialMetaRecommender(_BaseSequentialMetaRecommender): """A meta recommender that switches between recommenders from an iterable. Similar to :class:`baybe.recommenders.meta.sequential.SequentialMetaRecommender` - but without explicit list conversion. Consequently, it supports arbitrary - iterables, possibly of infinite length. The downside is that serialization is not - supported. + but without explicit list conversion. This enables a number of advanced use cases: + + * It supports arbitrary iterables, allowing to configure recommender sequences + of infinite length. This is useful when the total number of iterations unknown + in advance. + * It can be used to adaptively adjust the recommender sequence based on the + latest context available outside the class, by modifying the iterable on the fly. + + The downside is that serialization is not supported. Raises: NoRecommendersLeftError: If more recommenders are requested than there are From cae08693ed4eabfc8d0d4db1f15a19bcb5ebb2d1 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 20 Dec 2024 17:32:58 +0100 Subject: [PATCH 11/26] Adjust recommender retrieval in Campaign class --- baybe/campaign.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/baybe/campaign.py b/baybe/campaign.py index 6bd36585b..ff69d5176 100644 --- a/baybe/campaign.py +++ b/baybe/campaign.py @@ -403,9 +403,6 @@ def recommend( def posterior(self, candidates: pd.DataFrame) -> Posterior: """Get the posterior predictive distribution for the given candidates. - The predictive distribution is based on the surrogate model of the last used - recommender. - Args: candidates: The candidate points in experimental recommendations. For details, see :meth:`baybe.surrogates.base.Surrogate.posterior`. @@ -430,9 +427,19 @@ def posterior(self, candidates: pd.DataFrame) -> Posterior: with torch.no_grad(): return surrogate.posterior(candidates) - def get_surrogate(self) -> SurrogateProtocol: + def get_surrogate( + self, + batch_size: int | None = None, + pending_experiments: pd.DataFrame | None = None, + ) -> SurrogateProtocol: """Get the current surrogate model. + Args: + batch_size: See :meth:`recommend`. + Only required when using meta recommenders that demand it. + pending_experiments: See :meth:`recommend`. + Only required when using meta recommenders that demand it. + Raises: RuntimeError: If the current recommender does not provide a surrogate model. @@ -453,7 +460,13 @@ def get_surrogate(self) -> SurrogateProtocol: pure_recommender: RecommenderProtocol if isinstance(self.recommender, MetaRecommender): - pure_recommender = self.recommender.get_current_recommender() + pure_recommender = self.recommender.select_recommender( + batch_size, + self.searchspace, + self.objective, + self.measurements, + pending_experiments, + ) else: pure_recommender = self.recommender From 5b32fcb0cde9d53f8a31d2135574933ef3ee516e Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 10 Jan 2025 13:50:31 +0100 Subject: [PATCH 12/26] Add class variable indicating statefulness of the recommender type --- baybe/recommenders/meta/base.py | 5 ++++- baybe/recommenders/meta/sequential.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/baybe/recommenders/meta/base.py b/baybe/recommenders/meta/base.py index 259ad1b1e..e77402d28 100644 --- a/baybe/recommenders/meta/base.py +++ b/baybe/recommenders/meta/base.py @@ -2,7 +2,7 @@ import gc from abc import ABC, abstractmethod -from typing import Any +from typing import Any, ClassVar import cattrs import pandas as pd @@ -22,6 +22,9 @@ class MetaRecommender(SerialMixin, RecommenderProtocol, ABC): """Abstract base class for all meta recommenders.""" + is_stateful: ClassVar[bool] = False + """Boolean flag indicating if the meta recommender is stateful.""" + @abstractmethod def select_recommender( self, diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index 03807eda4..e127810a4 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -6,7 +6,7 @@ import gc from abc import abstractmethod from collections.abc import Iterable, Iterator -from typing import Literal +from typing import ClassVar, Literal import pandas as pd from attrs import define, field, fields @@ -91,6 +91,8 @@ def __str__(self) -> str: @define class _BaseSequentialMetaRecommender(MetaRecommender): + is_stateful: ClassVar[bool] = True + # TODO: These should **not** be exposed via the constructor but the workaround # is currently needed for correct (de-)serialization. A proper approach would be # to not set them via the constructor but through a custom hook in combination From 7aa3b14c1045845ea0e3f8951968992fba6c06f5 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 10 Jan 2025 14:14:38 +0100 Subject: [PATCH 13/26] Replace default factory method with inline factory --- baybe/recommenders/meta/sequential.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index e127810a4..c6faee3aa 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -9,7 +9,7 @@ from typing import ClassVar, Literal import pandas as pd -from attrs import define, field, fields +from attrs import Factory, define, field, fields from attrs.validators import deep_iterable, in_, instance_of from typing_extensions import override @@ -253,7 +253,10 @@ class StreamingSequentialMetaRecommender(_BaseSequentialMetaRecommender): recommenders: Iterable[PureRecommender] = field() """An iterable providing the recommenders to be used.""" - _iterator: Iterator = field(init=False) + _iterator: Iterator = field( + init=False, + default=Factory(lambda self: iter(self.recommenders), takes_self=True), + ) """The iterator used to traverse the recommenders.""" _last_recommender: PureRecommender | None = field(init=False, default=None) @@ -262,11 +265,6 @@ class StreamingSequentialMetaRecommender(_BaseSequentialMetaRecommender): _step_of_last_recommender: int = field(init=False, default=-1) """The position of the latest recommender fetched from the iterable.""" - @_iterator.default - def default_iterator(self): - """Initialize the recommender iterator.""" - return iter(self.recommenders) - @override def _get_recommender_at_current_step(self) -> PureRecommender: if self._step != self._step_of_last_recommender: From a799ae59d39a4533336028220f57813babaf10c8 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 10 Jan 2025 14:16:00 +0100 Subject: [PATCH 14/26] Use None for unitialized step indicator --- baybe/recommenders/meta/sequential.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index c6faee3aa..84cab7b0a 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -262,7 +262,7 @@ class StreamingSequentialMetaRecommender(_BaseSequentialMetaRecommender): _last_recommender: PureRecommender | None = field(init=False, default=None) """The recommender returned from the last call.""" - _step_of_last_recommender: int = field(init=False, default=-1) + _step_of_last_recommender: int | None = field(init=False, default=None) """The position of the latest recommender fetched from the iterable.""" @override From 2124c9dadd2a32b0e967cfa8dade74a557d6b15b Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 10 Jan 2025 14:22:44 +0100 Subject: [PATCH 15/26] Allow meta recommender to be composed of other meta recommenders Requiring that they are composed of pure recommenders is an unnecessary limitation. Allowing other meta recommenders as building blocks can indeed be useful, for example, using a two-phase recommender where the second phase uses an adaptive (e.g. batch-size dependent) meta recommender --- baybe/recommenders/meta/adaptive.py | 8 ++++---- baybe/recommenders/meta/base.py | 3 +-- baybe/recommenders/meta/sequential.py | 24 ++++++++++++------------ 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/baybe/recommenders/meta/adaptive.py b/baybe/recommenders/meta/adaptive.py index bd6fabfce..7d56dd461 100644 --- a/baybe/recommenders/meta/adaptive.py +++ b/baybe/recommenders/meta/adaptive.py @@ -6,8 +6,8 @@ from typing_extensions import override from baybe.objectives.base import Objective +from baybe.recommenders.base import RecommenderProtocol from baybe.recommenders.meta.base import MetaRecommender -from baybe.recommenders.pure.base import PureRecommender from baybe.searchspace.core import SearchSpace from baybe.utils.interval import Partition @@ -16,8 +16,8 @@ class BatchSizeControlledMetaRecommender(MetaRecommender): """A meta recommender that selects recommenders according to the batch size.""" - recommenders: list[PureRecommender] = field( - converter=list, validator=deep_iterable(instance_of(PureRecommender)) + recommenders: list[RecommenderProtocol] = field( + converter=list, validator=deep_iterable(instance_of(RecommenderProtocol)) ) """The recommenders for the individual batch size intervals.""" @@ -42,7 +42,7 @@ def select_recommender( objective: Objective | None = None, measurements: pd.DataFrame | None = None, pending_experiments: pd.DataFrame | None = None, - ) -> PureRecommender: + ) -> RecommenderProtocol: if batch_size is None: raise ValueError("A batch size is required.") return self.recommenders[self.partition.get_interval_index(batch_size)] diff --git a/baybe/recommenders/meta/base.py b/baybe/recommenders/meta/base.py index e77402d28..f112ad548 100644 --- a/baybe/recommenders/meta/base.py +++ b/baybe/recommenders/meta/base.py @@ -11,7 +11,6 @@ from baybe.objectives.base import Objective from baybe.recommenders.base import RecommenderProtocol -from baybe.recommenders.pure.base import PureRecommender from baybe.recommenders.pure.nonpredictive.base import NonPredictiveRecommender from baybe.searchspace import SearchSpace from baybe.serialization import SerialMixin, converter, unstructure_base @@ -33,7 +32,7 @@ def select_recommender( objective: Objective | None = None, measurements: pd.DataFrame | None = None, pending_experiments: pd.DataFrame | None = None, - ) -> PureRecommender: + ) -> RecommenderProtocol: """Select a pure recommender for the given experimentation context. See :meth:`baybe.recommenders.base.RecommenderProtocol.recommend` for details diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index 84cab7b0a..f24534f32 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -15,8 +15,8 @@ from baybe.exceptions import NoRecommendersLeftError from baybe.objectives.base import Objective +from baybe.recommenders.base import RecommenderProtocol from baybe.recommenders.meta.base import MetaRecommender -from baybe.recommenders.pure.base import PureRecommender from baybe.recommenders.pure.bayesian.botorch import BotorchRecommender from baybe.recommenders.pure.nonpredictive.sampling import RandomRecommender from baybe.searchspace import SearchSpace @@ -43,10 +43,10 @@ class TwoPhaseMetaRecommender(MetaRecommender): required when using the meta recommender with stateful recommenders. """ - initial_recommender: PureRecommender = field(factory=RandomRecommender) + initial_recommender: RecommenderProtocol = field(factory=RandomRecommender) """The initial recommender used by the meta recommender.""" - recommender: PureRecommender = field(factory=BotorchRecommender) + recommender: RecommenderProtocol = field(factory=BotorchRecommender) """The recommender used by the meta recommender after the switch.""" switch_after: int = field(default=1) @@ -67,7 +67,7 @@ def select_recommender( objective: Objective | None = None, measurements: pd.DataFrame | None = None, pending_experiments: pd.DataFrame | None = None, - ) -> PureRecommender: + ) -> RecommenderProtocol: n_data = len(measurements) if measurements is not None else 0 if (n_data >= self.switch_after) or ( self._has_switched and self.remain_switched @@ -111,7 +111,7 @@ class _BaseSequentialMetaRecommender(MetaRecommender): """The number of measurements available at the last successful recommend call.""" @abstractmethod - def _get_recommender_at_current_step(self) -> PureRecommender: + def _get_recommender_at_current_step(self) -> RecommenderProtocol: """Get the recommender at the current sequence position.""" @override @@ -122,7 +122,7 @@ def select_recommender( objective: Objective | None = None, measurements: pd.DataFrame | None = None, pending_experiments: pd.DataFrame | None = None, - ) -> PureRecommender: + ) -> RecommenderProtocol: # If the training dataset size has decreased, something went wrong if ( n_data := len(measurements) if measurements is not None else 0 @@ -185,8 +185,8 @@ class SequentialMetaRecommender(_BaseSequentialMetaRecommender): recommenders available and ``mode="raise"``. """ - recommenders: list[PureRecommender] = field( - converter=list, validator=deep_iterable(instance_of(PureRecommender)) + recommenders: list[RecommenderProtocol] = field( + converter=list, validator=deep_iterable(instance_of(RecommenderProtocol)) ) """A finite-length sequence of recommenders to be used. For infinite-length iterables, see @@ -205,7 +205,7 @@ class SequentialMetaRecommender(_BaseSequentialMetaRecommender): """ @override - def _get_recommender_at_current_step(self) -> PureRecommender: + def _get_recommender_at_current_step(self) -> RecommenderProtocol: idx = self._step if self.mode == "reuse_last": idx = min(idx, len(self.recommenders) - 1) @@ -250,7 +250,7 @@ class StreamingSequentialMetaRecommender(_BaseSequentialMetaRecommender): recommenders available. """ - recommenders: Iterable[PureRecommender] = field() + recommenders: Iterable[RecommenderProtocol] = field() """An iterable providing the recommenders to be used.""" _iterator: Iterator = field( @@ -259,14 +259,14 @@ class StreamingSequentialMetaRecommender(_BaseSequentialMetaRecommender): ) """The iterator used to traverse the recommenders.""" - _last_recommender: PureRecommender | None = field(init=False, default=None) + _last_recommender: RecommenderProtocol | None = field(init=False, default=None) """The recommender returned from the last call.""" _step_of_last_recommender: int | None = field(init=False, default=None) """The position of the latest recommender fetched from the iterable.""" @override - def _get_recommender_at_current_step(self) -> PureRecommender: + def _get_recommender_at_current_step(self) -> RecommenderProtocol: if self._step != self._step_of_last_recommender: try: self._last_recommender = next(self._iterator) From fba716796fe0a1a4887b8b772115004a1fb7b834 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 10 Jan 2025 14:44:35 +0100 Subject: [PATCH 16/26] Rename to BatchSizeAdaptiveMetaRecommender --- baybe/recommenders/meta/adaptive.py | 2 +- tests/test_meta_recommenders.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/baybe/recommenders/meta/adaptive.py b/baybe/recommenders/meta/adaptive.py index 7d56dd461..94eb54990 100644 --- a/baybe/recommenders/meta/adaptive.py +++ b/baybe/recommenders/meta/adaptive.py @@ -13,7 +13,7 @@ @define -class BatchSizeControlledMetaRecommender(MetaRecommender): +class BatchSizeAdaptiveMetaRecommender(MetaRecommender): """A meta recommender that selects recommenders according to the batch size.""" recommenders: list[RecommenderProtocol] = field( diff --git a/tests/test_meta_recommenders.py b/tests/test_meta_recommenders.py index 7226506f9..cd92bfca1 100644 --- a/tests/test_meta_recommenders.py +++ b/tests/test_meta_recommenders.py @@ -10,7 +10,7 @@ SequentialMetaRecommender, TwoPhaseMetaRecommender, ) -from baybe.recommenders.meta.adaptive import BatchSizeControlledMetaRecommender +from baybe.recommenders.meta.adaptive import BatchSizeAdaptiveMetaRecommender from baybe.recommenders.meta.sequential import StreamingSequentialMetaRecommender from tests.conftest import select_recommender @@ -129,10 +129,10 @@ def test_sequential_meta_recommender(cls, mode): meta_recommender._was_used = True -def test_batch_size_controlled_meta_recommender(): +def test_batch_size_adpative_meta_recommender(): """The recommender retrieves the right recommender for the requested batch size.""" thresholds = [2, 5] - meta_recommender = BatchSizeControlledMetaRecommender(RECOMMENDERS, thresholds) + meta_recommender = BatchSizeAdaptiveMetaRecommender(RECOMMENDERS, thresholds) for i, threshold in enumerate(thresholds): assert meta_recommender.select_recommender(threshold - 1) is RECOMMENDERS[i] assert meta_recommender.select_recommender(threshold) is RECOMMENDERS[i + 1] From 4466b0f156514c478efd4cfcd286cc5d9f298866 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 10 Jan 2025 14:53:29 +0100 Subject: [PATCH 17/26] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f1753838..922ee1cb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `arrays_to_dataframes` decorator to create lookups from array-based callables - `DiscreteConstraint.get_valid` to conveniently access valid candidates - Functionality for persisting benchmarking results on S3 from a manual pipeline run +- `remain_switched` option to `TwoPhaseMetaRecommender` +- `BatchSizeAdaptiveMetaRecommender` for selecting recommenders based on batch size +- `is_stateful` class variable to `MetaRecommenders` ### Changed - `SubstanceParameter` encodings are now computed exclusively with the @@ -30,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - De-/activating Polars via `BAYBE_DEACTIVATE_POLARS` now requires passing values compatible with `strtobool` - All arguments to `MetaRecommender.select_recommender` are now optional +- `MetaRecommender`s can now be composed of other `MetaRecommender`s ### Fixed - Rare bug arising from degenerate `SubstanceParameter.comp_df` rows that caused From 281ad572958ee7d72034a67adb5440ff28561164 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 10 Jan 2025 15:06:24 +0100 Subject: [PATCH 18/26] Deprecate obsolete methods of MetaRecommender --- CHANGELOG.md | 2 ++ baybe/recommenders/meta/base.py | 18 ++++++++++++++++++ tests/test_deprecations.py | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 922ee1cb5..56e034484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `SubstanceEncoding` value `RDKIT`. As a replacement, `RDKIT2DDESCRIPTORS` can be used. - The `metadata` attribute of `SubspaceDiscrete` no longer exists. Metadata is now exclusively handled by the `Campaign` class. +- `get_current_recommender` and `get_next_recommender` of `MetaRecommender` have become + obsolete and calling them is no longer possible ## [0.11.3] - 2024-11-06 ### Fixed diff --git a/baybe/recommenders/meta/base.py b/baybe/recommenders/meta/base.py index f112ad548..3a1cda89e 100644 --- a/baybe/recommenders/meta/base.py +++ b/baybe/recommenders/meta/base.py @@ -9,8 +9,10 @@ from attrs import define from typing_extensions import override +from baybe.exceptions import DeprecationError from baybe.objectives.base import Objective from baybe.recommenders.base import RecommenderProtocol +from baybe.recommenders.pure.base import PureRecommender from baybe.recommenders.pure.nonpredictive.base import NonPredictiveRecommender from baybe.searchspace import SearchSpace from baybe.serialization import SerialMixin, converter, unstructure_base @@ -39,6 +41,22 @@ def select_recommender( on the method arguments. """ + def get_current_recommender(self) -> PureRecommender: + """Deprecated! Use :meth:`select_recommender` instead.""" # noqa: D401 + raise DeprecationError( + f"'{MetaRecommender.__name__}.get_current_recommender' has become " + f"obsolete. Use " + f"'{MetaRecommender.__name__}.{self.select_recommender.__name__}' instead." + ) + + def get_next_recommender(self) -> PureRecommender: + """Deprecated! Use :meth:`select_recommender` instead.""" # noqa: D401 + raise DeprecationError( + f"'{MetaRecommender.__name__}.get_next_recommender' has become " + f"obsolete. Use " + f"'{MetaRecommender.__name__}.{self.select_recommender.__name__}' instead." + ) + @override def recommend( self, diff --git a/tests/test_deprecations.py b/tests/test_deprecations.py index 9ce45a704..39f38e712 100644 --- a/tests/test_deprecations.py +++ b/tests/test_deprecations.py @@ -25,6 +25,7 @@ NumericalContinuousParameter, NumericalDiscreteParameter, ) +from baybe.recommenders.meta.sequential import TwoPhaseMetaRecommender from baybe.recommenders.pure.bayesian import ( BotorchRecommender, SequentialGreedyRecommender, @@ -284,3 +285,12 @@ def test_migrated_metadata_attribute(): NumericalDiscreteParameter("p", [0, 1]) ) subspace.metadata + + +def test_deprecated_meta_recommender_methods(): + """Calling the deprecated methods of meta recommender raises an error.""" + recommender = TwoPhaseMetaRecommender() + with pytest.raises(DeprecationError, match="has become obsolete"): + recommender.get_current_recommender() + with pytest.raises(DeprecationError, match="has become obsolete"): + recommender.get_next_recommender() From 7a9f123a8c75ef72a6c376c7651693de0c054a27 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 10 Jan 2025 15:33:13 +0100 Subject: [PATCH 19/26] Add missing class docstring --- baybe/recommenders/meta/sequential.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index f24534f32..aaf4348af 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -91,6 +91,8 @@ def __str__(self) -> str: @define class _BaseSequentialMetaRecommender(MetaRecommender): + """Base class for sequential meta recommenders.""" + is_stateful: ClassVar[bool] = True # TODO: These should **not** be exposed via the constructor but the workaround From 7c980eb1bda2030e7d652e8a4572c84351ca2a7d Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Fri, 10 Jan 2025 17:31:45 +0100 Subject: [PATCH 20/26] Add `get_inner_recommender` method --- CHANGELOG.md | 1 + baybe/recommenders/meta/base.py | 55 +++++++++++++++++++++------------ tests/test_deprecations.py | 4 +-- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56e034484..4d58033b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `remain_switched` option to `TwoPhaseMetaRecommender` - `BatchSizeAdaptiveMetaRecommender` for selecting recommenders based on batch size - `is_stateful` class variable to `MetaRecommenders` +- `get_inner_recommender` method to `MetaRecommender` for leaving the meta level ### Changed - `SubstanceParameter` encodings are now computed exclusively with the diff --git a/baybe/recommenders/meta/base.py b/baybe/recommenders/meta/base.py index 3a1cda89e..6f8e49ff8 100644 --- a/baybe/recommenders/meta/base.py +++ b/baybe/recommenders/meta/base.py @@ -4,7 +4,6 @@ from abc import ABC, abstractmethod from typing import Any, ClassVar -import cattrs import pandas as pd from attrs import define from typing_extensions import override @@ -41,20 +40,46 @@ def select_recommender( on the method arguments. """ + def get_inner_recommender( + self, + batch_size: int | None = None, + searchspace: SearchSpace | None = None, + objective: Objective | None = None, + measurements: pd.DataFrame | None = None, + pending_experiments: pd.DataFrame | None = None, + ) -> RecommenderProtocol: + """Follow the meta-recommender chain to the first non-meta recommender. + + See :meth:`baybe.recommenders.base.RecommenderProtocol.recommend` for details + on the method arguments. + """ + recommender: MetaRecommender | RecommenderProtocol = self + while isinstance(recommender, MetaRecommender): + recommender = recommender.select_recommender( + batch_size, searchspace, objective, measurements, pending_experiments + ) + return recommender + def get_current_recommender(self) -> PureRecommender: - """Deprecated! Use :meth:`select_recommender` instead.""" # noqa: D401 + """Deprecated! Use :meth:`select_recommender` or :meth:`get_inner_recommender` + instead. + """ # noqa raise DeprecationError( - f"'{MetaRecommender.__name__}.get_current_recommender' has become " - f"obsolete. Use " - f"'{MetaRecommender.__name__}.{self.select_recommender.__name__}' instead." + f"'{MetaRecommender.__name__}.get_current_recommender' has been deprecated." + f"Use '{MetaRecommender.__name__}.{self.select_recommender.__name__}' or " + f"'{MetaRecommender.__name__}.{self.get_inner_recommender.__name__}' " + f"instead." ) def get_next_recommender(self) -> PureRecommender: - """Deprecated! Use :meth:`select_recommender` instead.""" # noqa: D401 + """Deprecated! Use :meth:`select_recommender` or :meth:`get_inner_recommender` + instead. + """ # noqa raise DeprecationError( - f"'{MetaRecommender.__name__}.get_next_recommender' has become " - f"obsolete. Use " - f"'{MetaRecommender.__name__}.{self.select_recommender.__name__}' instead." + f"'{MetaRecommender.__name__}.get_current_recommender' has been deprecated." + f"Use '{MetaRecommender.__name__}.{self.select_recommender.__name__}' or " + f"'{MetaRecommender.__name__}.{self.get_inner_recommender.__name__}' " + f"instead." ) @override @@ -96,17 +121,7 @@ def recommend( # Register (un-)structure hooks -converter.register_unstructure_hook( - MetaRecommender, - lambda x: unstructure_base( - x, - # TODO: Remove once deprecation got expired: - overrides=dict( - allow_repeated_recommendations=cattrs.override(omit=True), - allow_recommending_already_measured=cattrs.override(omit=True), - ), - ), -) +converter.register_unstructure_hook(MetaRecommender, unstructure_base) converter.register_structure_hook( MetaRecommender, get_base_structure_hook(MetaRecommender) ) diff --git a/tests/test_deprecations.py b/tests/test_deprecations.py index 39f38e712..bf56faf97 100644 --- a/tests/test_deprecations.py +++ b/tests/test_deprecations.py @@ -290,7 +290,7 @@ def test_migrated_metadata_attribute(): def test_deprecated_meta_recommender_methods(): """Calling the deprecated methods of meta recommender raises an error.""" recommender = TwoPhaseMetaRecommender() - with pytest.raises(DeprecationError, match="has become obsolete"): + with pytest.raises(DeprecationError, match="has been deprecated."): recommender.get_current_recommender() - with pytest.raises(DeprecationError, match="has become obsolete"): + with pytest.raises(DeprecationError, match="has been deprecated"): recommender.get_next_recommender() From c427f396d45f684183fd3609e46d2c3effd0e0cb Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Thu, 16 Jan 2025 09:18:53 +0100 Subject: [PATCH 21/26] Make BaseSequentialMetaRecommender public --- baybe/recommenders/meta/sequential.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index aaf4348af..8bbaf8669 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -90,7 +90,7 @@ def __str__(self) -> str: @define -class _BaseSequentialMetaRecommender(MetaRecommender): +class BaseSequentialMetaRecommender(MetaRecommender): """Base class for sequential meta recommenders.""" is_stateful: ClassVar[bool] = True @@ -166,7 +166,7 @@ def recommend( @define -class SequentialMetaRecommender(_BaseSequentialMetaRecommender): +class SequentialMetaRecommender(BaseSequentialMetaRecommender): """A meta recommender that uses a pre-defined sequence of recommenders. A new recommender is taken from the sequence whenever at least one new measurement @@ -233,7 +233,7 @@ def __str__(self) -> str: @define -class StreamingSequentialMetaRecommender(_BaseSequentialMetaRecommender): +class StreamingSequentialMetaRecommender(BaseSequentialMetaRecommender): """A meta recommender that switches between recommenders from an iterable. Similar to :class:`baybe.recommenders.meta.sequential.SequentialMetaRecommender` From 40f4d712446bac2c9108770e990d35fa50e4cad6 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Thu, 16 Jan 2025 09:30:09 +0100 Subject: [PATCH 22/26] Refine `get_inner_recommender` name and docstrings --- CHANGELOG.md | 2 +- baybe/recommenders/meta/base.py | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d58033b0..1655d8e9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `remain_switched` option to `TwoPhaseMetaRecommender` - `BatchSizeAdaptiveMetaRecommender` for selecting recommenders based on batch size - `is_stateful` class variable to `MetaRecommenders` -- `get_inner_recommender` method to `MetaRecommender` for leaving the meta level +- `get_non_meta_recommender` method to `MetaRecommender` ### Changed - `SubstanceParameter` encodings are now computed exclusively with the diff --git a/baybe/recommenders/meta/base.py b/baybe/recommenders/meta/base.py index 6f8e49ff8..844f0f896 100644 --- a/baybe/recommenders/meta/base.py +++ b/baybe/recommenders/meta/base.py @@ -40,7 +40,7 @@ def select_recommender( on the method arguments. """ - def get_inner_recommender( + def get_non_meta_recommender( self, batch_size: int | None = None, searchspace: SearchSpace | None = None, @@ -48,7 +48,12 @@ def get_inner_recommender( measurements: pd.DataFrame | None = None, pending_experiments: pd.DataFrame | None = None, ) -> RecommenderProtocol: - """Follow the meta-recommender chain to the first non-meta recommender. + """Follow the meta-recommender chain to the selected non-meta recommender. + + Recursively calls :meth:`MetaRecommender.select_recommender` until a + non-meta recommender is encountered, which is then returned. + Effectively, this extracts the recommender responsible for generating + the recommendations for the specified context. See :meth:`baybe.recommenders.base.RecommenderProtocol.recommend` for details on the method arguments. @@ -61,24 +66,24 @@ def get_inner_recommender( return recommender def get_current_recommender(self) -> PureRecommender: - """Deprecated! Use :meth:`select_recommender` or :meth:`get_inner_recommender` - instead. + """Deprecated! Use :meth:`select_recommender` or + :meth:`get_non_meta_recommender` instead. """ # noqa raise DeprecationError( f"'{MetaRecommender.__name__}.get_current_recommender' has been deprecated." f"Use '{MetaRecommender.__name__}.{self.select_recommender.__name__}' or " - f"'{MetaRecommender.__name__}.{self.get_inner_recommender.__name__}' " + f"'{MetaRecommender.__name__}.{self.get_non_meta_recommender.__name__}' " f"instead." ) def get_next_recommender(self) -> PureRecommender: - """Deprecated! Use :meth:`select_recommender` or :meth:`get_inner_recommender` - instead. + """Deprecated! Use :meth:`select_recommender` or + :meth:`get_non_meta_recommender` instead. """ # noqa raise DeprecationError( f"'{MetaRecommender.__name__}.get_current_recommender' has been deprecated." f"Use '{MetaRecommender.__name__}.{self.select_recommender.__name__}' or " - f"'{MetaRecommender.__name__}.{self.get_inner_recommender.__name__}' " + f"'{MetaRecommender.__name__}.{self.get_non_meta_recommender.__name__}' " f"instead." ) From 2d309123732d56ca372e9ff85c14d6e08b8759f6 Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Thu, 16 Jan 2025 09:50:24 +0100 Subject: [PATCH 23/26] Validate first interval of BatchSizeAdaptiveMetaRecommender --- baybe/recommenders/meta/adaptive.py | 12 +++++++++-- .../test_meta_recommender_validation.py | 21 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/validation/test_meta_recommender_validation.py diff --git a/baybe/recommenders/meta/adaptive.py b/baybe/recommenders/meta/adaptive.py index 94eb54990..d1179c64a 100644 --- a/baybe/recommenders/meta/adaptive.py +++ b/baybe/recommenders/meta/adaptive.py @@ -27,12 +27,20 @@ class BatchSizeAdaptiveMetaRecommender(MetaRecommender): """The partition mapping batch size intervals to recommenders. """ @partition.validator - def _validate_partitioning(self, _, value): - if (lr := len(self.recommenders)) != (lp := len(value)): + def _validate_partitioning(self, _, partition: Partition): + if (lr := len(self.recommenders)) != (lp := len(partition)): raise ValueError( f"The number of recommenders (given: {lr}) must be equal to the number " f"of intervals defined by the partition (given: {lp})." ) + if (thres := partition.thresholds[0]) < 1: + raise ValueError( + f"The first interval of the specified partition ends at {thres}, " + f"which is irrelevant for a " + f"'{BatchSizeAdaptiveMetaRecommender.__name__}' since the minimum " + f"possible batch size is 1. Please provide a partition whose first " + f"interval includes 1." + ) @override def select_recommender( diff --git a/tests/validation/test_meta_recommender_validation.py b/tests/validation/test_meta_recommender_validation.py new file mode 100644 index 000000000..179682ba9 --- /dev/null +++ b/tests/validation/test_meta_recommender_validation.py @@ -0,0 +1,21 @@ +"""Validation tests for meta recommenders.""" + +import pytest +from pytest import param + +from baybe.recommenders.meta.adaptive import BatchSizeAdaptiveMetaRecommender +from baybe.recommenders.pure.nonpredictive.sampling import RandomRecommender + + +@pytest.mark.parametrize( + ("thresholds", "match"), + [ + param([-1.0], "ends at -1.0", id="unused_interval"), + param([1, 2], "equal to the number of intervals", id="inconsistent_lengths"), + ], +) +def test_batch_size_adaptive_meta_recommender(thresholds, match): + """Providing invalid constructor arguments raises an error.""" + rec = RandomRecommender() + with pytest.raises(ValueError, match=match): + BatchSizeAdaptiveMetaRecommender([rec, rec], thresholds) From 0af5c2cf080b7844eb6da2c3d514cb2c116870ee Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Thu, 16 Jan 2025 11:13:56 +0100 Subject: [PATCH 24/26] Fix/improve docstrings --- CHANGELOG.md | 2 +- baybe/recommenders/meta/adaptive.py | 7 ++++++- baybe/recommenders/meta/base.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1655d8e9c..3f3603318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Functionality for persisting benchmarking results on S3 from a manual pipeline run - `remain_switched` option to `TwoPhaseMetaRecommender` - `BatchSizeAdaptiveMetaRecommender` for selecting recommenders based on batch size -- `is_stateful` class variable to `MetaRecommenders` +- `is_stateful` class variable to `MetaRecommender` - `get_non_meta_recommender` method to `MetaRecommender` ### Changed diff --git a/baybe/recommenders/meta/adaptive.py b/baybe/recommenders/meta/adaptive.py index d1179c64a..9e6922af8 100644 --- a/baybe/recommenders/meta/adaptive.py +++ b/baybe/recommenders/meta/adaptive.py @@ -14,7 +14,12 @@ @define class BatchSizeAdaptiveMetaRecommender(MetaRecommender): - """A meta recommender that selects recommenders according to the batch size.""" + """A meta recommender that selects recommenders according to the batch size. + + Recommender selection is done by grouping batch sizes into intervals using + a :class:`~baybe.utils.interval.Partition`, where each interval is mapped to a + specific recommender. + """ recommenders: list[RecommenderProtocol] = field( converter=list, validator=deep_iterable(instance_of(RecommenderProtocol)) diff --git a/baybe/recommenders/meta/base.py b/baybe/recommenders/meta/base.py index 844f0f896..4d27b18cf 100644 --- a/baybe/recommenders/meta/base.py +++ b/baybe/recommenders/meta/base.py @@ -48,7 +48,7 @@ def get_non_meta_recommender( measurements: pd.DataFrame | None = None, pending_experiments: pd.DataFrame | None = None, ) -> RecommenderProtocol: - """Follow the meta-recommender chain to the selected non-meta recommender. + """Follow the meta recommender chain to the selected non-meta recommender. Recursively calls :meth:`MetaRecommender.select_recommender` until a non-meta recommender is encountered, which is then returned. From e7b929b76e407b776031fc015666b08125a652af Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Thu, 16 Jan 2025 11:43:59 +0100 Subject: [PATCH 25/26] Add hypothesis strategy for Partition --- baybe/utils/interval.py | 2 ++ tests/hypothesis_strategies/utils.py | 12 +++++++++++- tests/validation/utils/test_partition_validation.py | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/baybe/utils/interval.py b/baybe/utils/interval.py index 098990de2..e674bc0ea 100644 --- a/baybe/utils/interval.py +++ b/baybe/utils/interval.py @@ -163,6 +163,8 @@ class Partition: @thresholds.validator def _validate_thresholds(self, _, value): + if not np.all(np.isfinite(value)): + raise ValueError("Thresholds must be finite.") if not all(x < y for x, y in pairwise(value)): raise ValueError("Thresholds must be strictly monotonically increasing.") diff --git a/tests/hypothesis_strategies/utils.py b/tests/hypothesis_strategies/utils.py index fb57555da..1bbc51b5c 100644 --- a/tests/hypothesis_strategies/utils.py +++ b/tests/hypothesis_strategies/utils.py @@ -4,8 +4,9 @@ import hypothesis.extra.numpy as hnp import hypothesis.strategies as st +import numpy as np -from baybe.utils.interval import Interval +from baybe.utils.interval import Interval, Partition from .basic import finite_floats @@ -65,3 +66,12 @@ def intervals( raise RuntimeError("This line should be unreachable.") return Interval.create(bounds) + + +partitions = st.builds( + Partition, + thresholds=st.lists(finite_floats(), min_size=1) + .map(sorted) + .filter(lambda x: np.all(np.diff(x) > 0)), +) +"""Generate :class:`baybe.utils.interval.Partition`.""" diff --git a/tests/validation/utils/test_partition_validation.py b/tests/validation/utils/test_partition_validation.py index 2a18f01b4..e1c1030a1 100644 --- a/tests/validation/utils/test_partition_validation.py +++ b/tests/validation/utils/test_partition_validation.py @@ -9,6 +9,8 @@ @pytest.mark.parametrize( ("thresholds", "error", "match"), [ + param([float("nan")], ValueError, "must be finite", id="nan"), + param([float("inf")], ValueError, "must be finite", id="inf"), param(1, TypeError, None, id="not_iterable"), param([], ValueError, None, id="too_short"), param([1, 0], ValueError, "monotonically increasing", id="decreasing"), From 33f0dd9ece34ad52afadd7b212fd717b13bf54df Mon Sep 17 00:00:00 2001 From: AdrianSosic Date: Thu, 16 Jan 2025 11:49:21 +0100 Subject: [PATCH 26/26] Rename attribute of StreamingSequentialMetaRecommender --- baybe/recommenders/meta/sequential.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/baybe/recommenders/meta/sequential.py b/baybe/recommenders/meta/sequential.py index 8bbaf8669..ff86bd881 100644 --- a/baybe/recommenders/meta/sequential.py +++ b/baybe/recommenders/meta/sequential.py @@ -264,12 +264,12 @@ class StreamingSequentialMetaRecommender(BaseSequentialMetaRecommender): _last_recommender: RecommenderProtocol | None = field(init=False, default=None) """The recommender returned from the last call.""" - _step_of_last_recommender: int | None = field(init=False, default=None) + _position_of_latest_recommender: int | None = field(init=False, default=None) """The position of the latest recommender fetched from the iterable.""" @override def _get_recommender_at_current_step(self) -> RecommenderProtocol: - if self._step != self._step_of_last_recommender: + if self._step != self._position_of_latest_recommender: try: self._last_recommender = next(self._iterator) except StopIteration as ex: @@ -277,7 +277,7 @@ def _get_recommender_at_current_step(self) -> RecommenderProtocol: f"A total of {self._step+1} recommender(s) was/were requested but " f"the provided iterator provided only {self._step} element(s). " ) from ex - self._step_of_last_recommender = self._step + self._position_of_latest_recommender = self._step # By now, the first recommender has been fetched assert self._last_recommender is not None