From 7b49f35f75680dc5770f04fae17adf595a1229b5 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Mon, 22 Jan 2024 10:54:06 +0100 Subject: [PATCH 1/9] Revert "Merge branch 'master' of https://github.com/automl/DEHB" This reverts commit dc5525a1e431dc44a58fe3d16ee075ccd53e80b8, reversing changes made to b4f8467b9d1accab800a111918fa9eaf8a9ce2dc. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 8d07545..c04db57 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ [![Coverage Status](https://coveralls.io/repos/github/automl/DEHB/badge.svg)](https://coveralls.io/github/automl/DEHB) [![PyPI](https://img.shields.io/pypi/v/dehb)](https://pypi.org/project/dehb/) [![Static Badge](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20-blue)](https://pypi.org/project/dehb/) -[![arXiv](https://img.shields.io/badge/arXiv-2105.09821-b31b1b.svg)](https://arxiv.org/abs/2105.09821) ### Installation ```bash # from pypi From 652fe0baf25aa00b9b238bf0e8bbc54cbba8515b Mon Sep 17 00:00:00 2001 From: Bronzila Date: Mon, 22 Jan 2024 10:54:16 +0100 Subject: [PATCH 2/9] Revert "Merge pull request #69 from automl/feat/ask_tell" This reverts commit 1ff61a013af0b9ed3e88bc130277a46adc066000, reversing changes made to a7f6bd924bd04bd8f9d957665b23e43fb069cfbe. --- mkdocs.yml | 4 + src/dehb/optimizers/de.py | 13 +-- src/dehb/optimizers/dehb.py | 175 +++++++++------------------- src/dehb/utils/bracket_manager.py | 13 --- src/dehb/utils/config_repository.py | 38 ++---- tests/test_config_repository.py | 70 +---------- tests/test_dehb.py | 94 +-------------- 7 files changed, 77 insertions(+), 330 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 94b6e28..e94bf8c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -74,6 +74,9 @@ markdown_extensions: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.emoji: + emoji_index: !!python/name:materialx.emoji.twemoji + emoji_generator: !!python/name:materialx.emoji.to_svg plugins: - search @@ -81,6 +84,7 @@ plugins: - mkdocstrings: default_handler: python enable_inventory: true + custom_templates: docs/_templates handlers: python: paths: [src] diff --git a/src/dehb/optimizers/de.py b/src/dehb/optimizers/de.py index be0e660..d1c40a2 100644 --- a/src/dehb/optimizers/de.py +++ b/src/dehb/optimizers/de.py @@ -163,7 +163,7 @@ def vector_to_configspace(self, vector: np.ndarray) -> ConfigSpace.Configuration ''' # creates a ConfigSpace object dict with all hyperparameters present, the inactive too new_config = ConfigSpace.util.impute_inactive_values( - self.cs.get_default_configuration() + self.cs.sample_configuration() ).get_dictionary() # iterates over all hyperparameters and normalizes each based on its type for i, hyper in enumerate(self.cs.get_hyperparameters()): @@ -304,17 +304,12 @@ def f_objective(self, x, fidelity=None, **kwargs): raise NotImplementedError("An objective function needs to be passed.") if self.encoding: x = self.map_to_original(x) - - # Only convert config if configspace is used + configuration has not been converted yet if self.configspace: - if not isinstance(x, ConfigSpace.Configuration): - # converts [0, 1] vector to a ConfigSpace object - config = self.vector_to_configspace(x) - else: - config = x + # converts [0, 1] vector to a ConfigSpace object + config = self.vector_to_configspace(x) else: + # can insert custom scaling/transform function here config = x.copy() - if fidelity is not None: # to be used when called by multi-fidelity based optimizers res = self.f(config, fidelity=fidelity, **kwargs) else: diff --git a/src/dehb/optimizers/dehb.py b/src/dehb/optimizers/dehb.py index 6ac3f04..612b496 100644 --- a/src/dehb/optimizers/dehb.py +++ b/src/dehb/optimizers/dehb.py @@ -245,18 +245,14 @@ def _f_objective(self, job_info): res = self.de[fidelity].f_objective(config, fidelity, **kwargs) info = res["info"] if "info" in res else {} run_info = { - "job_info": { - "config": config, - "config_id": config_id, - "fidelity": fidelity, - "parent_id": parent_id, - "bracket_id": bracket_id, - }, - "result": { - "fitness": res["fitness"], - "cost": res["cost"], - "info": info, - }, + "fitness": res["fitness"], + "cost": res["cost"], + "config": config, + "config_id": config_id, + "fidelity": fidelity, + "parent_id": parent_id, + "bracket_id": bracket_id, + "info": info, } if "gpu_devices" in job_info: @@ -546,10 +542,7 @@ def _acquire_config(self, bracket, fidelity): return config, config_id, parent_id def _get_next_job(self): - """Loads a configuration and fidelity to be evaluated next. - - Returns: - dict: Dicitonary containing all necessary information of the next job. + """ Loads a configuration and fidelity to be evaluated next by a free worker """ bracket = None if len(self.active_brackets) == 0 or \ @@ -571,50 +564,16 @@ def _get_next_job(self): # fidelity that the SH bracket allots fidelity = bracket.get_next_job_fidelity() config, config_id, parent_id = self._acquire_config(bracket, fidelity) - - # transform config to proper representation - if self.configspace: - # converts [0, 1] vector to a ConfigSpace object - config = self.de[fidelity].vector_to_configspace(config) - # notifies the Bracket Manager that a single config is to run for the fidelity chosen job_info = { "config": config, "config_id": config_id, "fidelity": fidelity, "parent_id": parent_id, - "bracket_id": bracket.bracket_id, + "bracket_id": bracket.bracket_id } - - # pass information of job submission to Bracket Manager - for bracket in self.active_brackets: - if bracket.bracket_id == job_info['bracket_id']: - # registering is IMPORTANT for Bracket Manager to perform SH - bracket.register_job(job_info['fidelity']) - break return job_info - def ask(self, n_configs: int=1): - """Get the next configuration to run from the optimizer. - - The retrieved configuration can then be evaluated by the user. - After evaluation use `tell` to report the results back to the optimizer. - For more information, please refer to the description of `tell`. - - Args: - n_configs (int, optional): Number of configs to ask for. Defaults to 1. - - Returns: - dict or list of dict: Job info(s) of next configuration to evaluate. - """ - if n_configs == 1: - return self._get_next_job() - - jobs = [] - for _ in range(n_configs): - jobs.append(self._get_next_job()) - return jobs - def _get_gpu_id_with_low_load(self): candidates = [] for k, v in self.gpu_usage.items(): @@ -635,7 +594,7 @@ def submit_job(self, job_info, **kwargs): """ Asks a free worker to run the objective function on config and fidelity """ job_info["kwargs"] = self.shared_data if self.shared_data is not None else kwargs - # submit to Dask client + # submit to to Dask client if self.n_workers > 1 or isinstance(self.client, Client): if self.single_node_with_gpus: # managing GPU allocation for the job to be submitted @@ -647,6 +606,13 @@ def submit_job(self, job_info, **kwargs): # skipping scheduling to Dask worker to avoid added overheads in the synchronous case self.futures.append(self._f_objective(job_info)) + # pass information of job submission to Bracket Manager + for bracket in self.active_brackets: + if bracket.bracket_id == job_info['bracket_id']: + # registering is IMPORTANT for Bracket Manager to perform SH + bracket.register_job(job_info['fidelity']) + break + def _fetch_results_from_workers(self): """ Iterate over futures and collect results from finished workers """ @@ -670,20 +636,40 @@ def _fetch_results_from_workers(self): else: # Dask not invoked in the synchronous case run_info = future - # tell result - self.tell(run_info["job_info"], run_info["result"]) + # update bracket information + fitness, cost = run_info["fitness"], run_info["cost"] + info = run_info["info"] if "info" in run_info else dict() + fidelity, parent_id = run_info["fidelity"], run_info["parent_id"] + config, config_id = run_info["config"], run_info["config_id"] + bracket_id = run_info["bracket_id"] + for bracket in self.active_brackets: + if bracket.bracket_id == bracket_id: + # bracket job complete + bracket.complete_job(fidelity) # IMPORTANT to perform synchronous SH + + self.config_repository.tell_result(config_id, fidelity, fitness, cost, info) + + # carry out DE selection + if fitness <= self.de[fidelity].fitness[parent_id]: + self.de[fidelity].population[parent_id] = config + self.de[fidelity].population_ids[parent_id] = config_id + self.de[fidelity].fitness[parent_id] = fitness + # updating incumbents + if self.de[fidelity].fitness[parent_id] < self.inc_score: + self._update_incumbents( + config=self.de[fidelity].population[parent_id], + score=self.de[fidelity].fitness[parent_id], + info=info + ) + # book-keeping + self._update_trackers( + traj=self.inc_score, runtime=cost, history=( + config.tolist(), float(fitness), float(cost), float(fidelity), info + ) + ) # remove processed future self.futures = np.delete(self.futures, [i for i, _ in done_list]).tolist() - def _adjust_budgets(self, fevals=None, brackets=None): - # only update budgets if it is not the first run - if fevals is not None and len(self.traj) > 0: - fevals = len(self.traj) + fevals - elif brackets is not None and self.iteration_counter > -1: - brackets = self.iteration_counter + brackets - - return fevals, brackets - def _is_run_budget_exhausted(self, fevals=None, brackets=None, total_cost=None): """ Checks if the DEHB run should be terminated or continued """ @@ -759,54 +745,6 @@ def _verbosity_runtime(self, fevals, brackets, total_cost): "{}/{} {}".format(remaining[0], remaining[1], remaining[2]) ) - def tell(self, job_info: dict, result: dict): - """Feed a result back to the optimizer. - - In order to correctly interpret the results, the `job_info` dict, retrieved by `ask`, - has to be given. Moreover, the `result` dict has to contain the keys `fitness` and `cost`. - It is also possible to add the field `info` to the `result` in order to store additional, - user-specific information. - - Args: - job_info (dict): Job info returned by ask(). - result (dict): Result dictionary with mandatory keys `fitness` and `cost`. - """ - # update bracket information - fitness, cost = result["fitness"], result["cost"] - info = result["info"] if "info" in result else dict() - fidelity, parent_id = job_info["fidelity"], job_info["parent_id"] - config, config_id = job_info["config"], job_info["config_id"] - bracket_id = job_info["bracket_id"] - for bracket in self.active_brackets: - if bracket.bracket_id == bracket_id: - # bracket job complete - bracket.complete_job(fidelity) # IMPORTANT to perform synchronous SH - - self.config_repository.tell_result(config_id, fidelity, fitness, cost, info) - - # get hypercube representation from config repo - if self.configspace: - config = self.config_repository.get(config_id) - - # carry out DE selection - if fitness <= self.de[fidelity].fitness[parent_id]: - self.de[fidelity].population[parent_id] = config - self.de[fidelity].population_ids[parent_id] = config_id - self.de[fidelity].fitness[parent_id] = fitness - # updating incumbents - if self.de[fidelity].fitness[parent_id] < self.inc_score: - self._update_incumbents( - config=self.de[fidelity].population[parent_id], - score=self.de[fidelity].fitness[parent_id], - info=info - ) - # book-keeping - self._update_trackers( - traj=self.inc_score, runtime=cost, history=( - config.tolist(), float(fitness), float(cost), float(fidelity), info - ) - ) - @logger.catch def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus=False, verbose=False, debug=False, save_intermediate=True, save_history=True, name=None, **kwargs): @@ -823,12 +761,6 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus 2) Number of Successive Halving brackets run under Hyperband (brackets) 3) Total computational cost (in seconds) aggregated by all function evaluations (total_cost) """ - # check if run has already been called before - if self.start is not None: - logger.warning("DEHB has already been run. Calling 'run' twice could lead to unintended" - + " behavior. Please restart DEHB with an increased compute budget" - + " instead of calling 'run' twice.") - # checks if a Dask client exists if len(kwargs) > 0 and self.n_workers > 1 and isinstance(self.client, Client): # broadcasts all additional data passed as **kwargs to all client workers @@ -842,8 +774,7 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus if self.single_node_with_gpus: self.distribute_gpus() - self.start = self.start = time.time() - fevals, brackets = self._adjust_budgets(fevals, brackets) + self.start = time.time() if verbose: print("\nLogging at {} for optimization starting at {}\n".format( os.path.join(os.getcwd(), self.log_filename), @@ -855,11 +786,11 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus if self._is_run_budget_exhausted(fevals, brackets, total_cost): break if self.is_worker_available(): - job_info = self.ask() + job_info = self._get_next_job() if brackets is not None and job_info["bracket_id"] >= brackets: # ignore submission and only collect results # when brackets are chosen as run budget, an extra bracket is created - # since iteration_counter is incremented in ask() and then checked + # since iteration_counter is incremented in _get_next_job() and then checked # in _is_run_budget_exhausted(), therefore, need to skip suggestions # coming from the extra allocated bracket # _is_run_budget_exhausted() will not return True until all the lower brackets @@ -919,6 +850,4 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus self.logger.info("{}".format(self.inc_config)) self._save_incumbent(name) self._save_history(name) - # reset waiting jobs of active bracket to allow for continuation - self.active_brackets[0].reset_waiting_jobs() return np.array(self.traj), np.array(self.runtime), np.array(self.history, dtype=object) diff --git a/src/dehb/utils/bracket_manager.py b/src/dehb/utils/bracket_manager.py index 598b92a..2642223 100644 --- a/src/dehb/utils/bracket_manager.py +++ b/src/dehb/utils/bracket_manager.py @@ -125,19 +125,6 @@ def is_waiting(self): """ return np.any([self._is_rung_waiting(i) > 0 for i, _ in enumerate(self.fidelities)]) - def reset_waiting_jobs(self): - """Resets all waiting jobs and updates the current_rung pointer accordingly.""" - for i, fidelity in enumerate(self.fidelities): - pending = self.sh_bracket[fidelity] - done = self._sh_bracket[fidelity] - waiting = np.abs(self.n_configs[i] - pending - done) - - # update current_rung pointer to the lowest rung with waiting jobs - if waiting > 0 and self.current_rung > i: - self.current_rung = i - # reset waiting jobs - self.sh_bracket[fidelity] += waiting - def __repr__(self): cell_width = 10 cell = "{{:^{}}}".format(cell_width) diff --git a/src/dehb/utils/config_repository.py b/src/dehb/utils/config_repository.py index 2d2bab1..126b28f 100644 --- a/src/dehb/utils/config_repository.py +++ b/src/dehb/utils/config_repository.py @@ -82,26 +82,28 @@ def announce_population(self, population: np.ndarray, fidelity=None) -> np.ndarr population_ids.append(conf_id) return np.array(population_ids) - def announce_fidelity(self, config_id: int, fidelity: float): + def announce_fidelity(self, config_id: int, fidelity: float) -> bool: """Announce the evaluation of a new fidelity for a given config. This function may only be used if the config already exists in the repository. - Note: This function is currently unused, but might be used later in order to - allow for continuation. Args: config_id (int): ID of Configuration fidelity (float): Fidelity on which the config will be evaluated + + Returns: + bool: Success/Failure of operation """ - try: - config_item = self.configs[config_id] - except IndexError as e: - raise IndexError("Config with the given ID can not be found.") from e + if config_id >= len(self.configs) or config_id < 0: + # TODO: Error message + return False + config_item = self.configs[config_id] result_item = { fidelity: ResultItem(np.inf, -1, {}), } config_item.results[fidelity] = result_item + return True def tell_result(self, config_id: int, fidelity: float, score: float, cost: float, info: dict): """Logs the achieved performance, cost etc. of a specific configuration-fidelity pair. @@ -113,10 +115,7 @@ def tell_result(self, config_id: int, fidelity: float, score: float, cost: float cost (float): Cost, given by objective function info (dict): Run info, given by objective function """ - try: - config_item = self.configs[config_id] - except IndexError as e: - raise IndexError("Config with the given ID can not be found.") from e + config_item = self.configs[config_id] # If configuration has been promoted, there is no fidelity information yet if fidelity not in config_item.results: @@ -125,19 +124,4 @@ def tell_result(self, config_id: int, fidelity: float, score: float, cost: float # ResultItem already given for specified fidelity --> update entries config_item.results[fidelity].score = score config_item.results[fidelity].cost = cost - config_item.results[fidelity].info = info - - def get(self, config_id: int) -> np.ndarray: - """Get the configuration with the given ID. - - Args: - config_id (int): ID of config - - Returns: - np.ndarray: Config in hypercube representation - """ - try: - config_item = self.configs[config_id] - except IndexError as e: - raise IndexError("Config with the given ID can not be found.") from e - return config_item.config \ No newline at end of file + config_item.results[fidelity].info = info \ No newline at end of file diff --git a/tests/test_config_repository.py b/tests/test_config_repository.py index 76cc1ae..f63e870 100644 --- a/tests/test_config_repository.py +++ b/tests/test_config_repository.py @@ -1,37 +1,21 @@ import typing import numpy as np -import pytest from src.dehb.utils import ConfigRepository class TestConfigAnnouncing(): """Class that bundles all tests for announcing configurations to the repository.""" - def test_single_config_fidelity(self): - """Tests announcing single config with a specified fidelity.""" + def test_single_config(self): + """Tests announcing single config.""" repo = ConfigRepository() config = np.array([0.5]) - config_id = repo.announce_config(config, 2.) + config_id = repo.announce_config(config, 2) assert len(repo.configs) == 1 assert config_id == 0 assert repo.configs[config_id].config == config - # result entry properly given - assert repo.configs[config_id].results[2.] is not None - - def test_single_config_no_fidelity(self): - """Tests announcing single config with a specified fidelity.""" - repo = ConfigRepository() - config = np.array([0.5]) - - config_id = repo.announce_config(config) - - assert len(repo.configs) == 1 - assert config_id == 0 - assert repo.configs[config_id].config == config - # result entry properly given - assert repo.configs[config_id].results[0.] is not None def test_population(self): """Tests announcing a whole population.""" @@ -47,50 +31,4 @@ def test_population(self): assert len(repo.configs) == 10 for conf_id in config_ids: - assert repo.configs[conf_id].config == pop[conf_id] - -class TestGetConfig(): - """Class that bundles all tests regarding retrieving of configs via config ID.""" - def test_get_successful(self): - """Test that get retrieves the right configuration.""" - repo = ConfigRepository() - config = np.array([0.5]) - - config_id = repo.announce_config(config) - - retrieved_config = repo.get(config_id) - - assert config == retrieved_config - - def test_get_failure(self): - """Test to verify that get returns the right error if config ID is unkown.""" - repo = ConfigRepository() - config = np.array([0.5]) - - config_id = repo.announce_config(config) - - with pytest.raises(IndexError): - repo.get(config_id + 1) - -class TestTellResult(): - """This class bundles all tests regarding the `tell_result` method.""" - def test_tell_result_successful(self): - repo = ConfigRepository() - config = np.array([0.5]) - - fidelity = 2.0 - config_id = repo.announce_config(config, fidelity) - score = 1 - cost = 2 - info = { - "test": 123, - } - repo.tell_result(config_id, fidelity, score, cost, info) - - config_item = repo.configs[config_id] - results = config_item.results - - assert len(results) == 1 - assert results[fidelity].score == score - assert results[fidelity].cost == cost - assert results[fidelity].info == info \ No newline at end of file + assert repo.configs[conf_id].config == pop[conf_id] \ No newline at end of file diff --git a/tests/test_dehb.py b/tests/test_dehb.py index d0a8e5c..277e959 100644 --- a/tests/test_dehb.py +++ b/tests/test_dehb.py @@ -37,7 +37,7 @@ def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_fideli Returns: _type_: _description_ """ - dim = len(configspace.get_hyperparameters()) if configspace else 1 + dim = len(configspace.get_hyperparameters()) return DEHB(f=objective_function, cs=configspace, dimensions=dim, min_fidelity=min_fidelity, max_fidelity=max_fidelity, eta=eta, n_workers=1) @@ -147,94 +147,4 @@ def test_single_bracket(self): # Note: The final + 1 is due to the inner workings of DEHB. If the run budget is exhausted, # we keep evolving new configurations without evaluating them, since we are only waiting to # to fetch all results started ahead of the budget exhaustion. - assert len(dehb.config_repository.configs) == num_initial_configs + 9 + 1 - -class TestAskTell: - """Class that bundles all tests regarding the ask and tell functionality of DEHB.""" - def test_all_fields_available(self): - """Verifies, that all fields needed are present in job info returned by ask.""" - cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, - objective_function=objective_function) - conf = dehb.ask() - assert "config" in conf - assert "bracket_id" in conf - assert "config_id" in conf - assert "fidelity" in conf - - def test_format_configspace(self): - """Verifies, that the returned config by ask() is of type Configuration - if a configspace is passed. - """ - cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, - objective_function=objective_function) - job_info = dehb.ask() - assert isinstance(job_info["config"], ConfigSpace.Configuration) - - def test_format_no_configspace(self): - """Verifies, that the returned config by ask() is of type Configuration - if a configspace is passed. - """ - dehb = create_toy_optimizer(configspace=None, min_fidelity=3, max_fidelity=27, eta=3, - objective_function=objective_function) - job_info = dehb.ask() - assert isinstance(job_info["config"], np.ndarray) - - def test_ask_multiple(self): - """Verifies, that ask can return multiple configs.""" - cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, - objective_function=objective_function) - job_infos = dehb.ask(2) - - assert len(job_infos) == 2 - assert job_infos[0]["config"] != job_infos[1]["config"] - - def test_ask_twice_different(self): - """Verifies, that ask can return multiple configs.""" - cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, - objective_function=objective_function) - job_info_a = dehb.ask() - job_info_b = dehb.ask() - assert job_info_a != job_info_b - - def test_tell_successful(self): - """Verifies, that tell successfully saves results.""" - cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, - objective_function=objective_function) - job_info = dehb.ask() - id = job_info["config_id"] - fid = job_info["fidelity"] - conf = job_info["config"] - - # before telling, entry should be empty - saved_score = dehb.config_repository.configs[id].results[fid].score - assert saved_score == np.inf - - result = objective_function(conf, fid) - dehb.tell(job_info, result) - - # after telling, score should be saved - saved_score = dehb.config_repository.configs[id].results[fid].score - assert saved_score == result["fitness"] - - def test_tell_error(self): - """Verifies, that tell throws an error if config ID is non-existent.""" - cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, - objective_function=objective_function) - # get config - job_info = dehb.ask() - # adjust config id to non existing id - job_info["config_id"] = 1337 - # create random result item - result = { - "fitness": 42, - "cost": 123 - } - # telling with wrong config_id should throw an error - with pytest.raises(IndexError): - dehb.tell(job_info, result) \ No newline at end of file + assert len(dehb.config_repository.configs) == num_initial_configs + 9 + 1 \ No newline at end of file From 8f446bcd665bb3b7ff06ec233e52cf0a64a39871 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Mon, 22 Jan 2024 11:02:58 +0100 Subject: [PATCH 3/9] Fix docs pipeline --- mkdocs.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index e94bf8c..94b6e28 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -74,9 +74,6 @@ markdown_extensions: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_code_format - - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg plugins: - search @@ -84,7 +81,6 @@ plugins: - mkdocstrings: default_handler: python enable_inventory: true - custom_templates: docs/_templates handlers: python: paths: [src] From 5f71db9a3c661a766f51abce80659ec87f0656a6 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Mon, 5 Feb 2024 11:20:49 +0100 Subject: [PATCH 4/9] Revert "Merge branch 'development' into master" This reverts commit 9ed23f6a45cb678e6e15c3009f3ac4b0dd6f6eaa, reversing changes made to dc5525a1e431dc44a58fe3d16ee075ccd53e80b8. --- README.md | 18 +- examples/00_interfacing_DEHB.ipynb | 90 +++--- ...1_Optimizing_RandomForest_using_DEHB.ipynb | 129 ++++---- .../02_using DEHB_without_ConfigSpace.ipynb | 42 +-- examples/03_pytorch_mnist_hpo.py | 30 +- src/dehb/optimizers/de.py | 142 +++------ src/dehb/optimizers/dehb.py | 291 ++++++++---------- src/dehb/utils/__init__.py | 3 +- src/dehb/utils/bracket_manager.py | 104 +++---- src/dehb/utils/config_repository.py | 127 -------- tests/test_config_repository.py | 34 -- tests/test_de.py | 2 +- tests/test_dehb.py | 70 +---- utils/README.md | 4 +- 14 files changed, 408 insertions(+), 678 deletions(-) delete mode 100644 src/dehb/utils/config_repository.py delete mode 100644 tests/test_config_repository.py diff --git a/README.md b/README.md index c04db57..604fa6d 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ pip install -e DEHB # -e stands for editable, lets you modify the code and reru To run PyTorch example: (*note additional requirements*) ```bash python examples/03_pytorch_mnist_hpo.py \ - --min_fidelity 1 \ - --max_fidelity 3 \ + --min_budget 1 \ + --max_budget 3 \ --runtime 60 \ --verbose ``` @@ -62,8 +62,8 @@ to it by that DEHB run. To run the PyTorch MNIST example on a single node using 2 workers: ```bash python examples/03_pytorch_mnist_hpo.py \ - --min_fidelity 1 \ - --max_fidelity 3 \ + --min_budget 1 \ + --max_budget 3 \ --runtime 60 \ --n_workers 2 \ --single_node_with_gpus \ @@ -96,8 +96,8 @@ bash utils/run_dask_setup.sh \ # Make sure to sleep to allow the workers to setup properly sleep 5 python examples/03_pytorch_mnist_hpo.py \ - --min_fidelity 1 \ - --max_fidelity 3 \ + --min_budget 1 \ + --max_budget 3 \ --runtime 60 \ --scheduler_file dask_dump/scheduler.json \ --verbose @@ -111,9 +111,9 @@ and were found to be *generally* useful across all cases tested. However, the parameters are still available for tuning to a specific problem. The Hyperband components: -* *min\_fidelity*: Needs to be specified for every DEHB instantiation and is used in determining -the fidelity spacing for the problem at hand. -* *max\_fidelity*: Needs to be specified for every DEHB instantiation. Represents the full-fidelity +* *min\_budget*: Needs to be specified for every DEHB instantiation and is used in determining +the budget spacing for the problem at hand. +* *max\_budget*: Needs to be specified for every DEHB instantiation. Represents the full-budget evaluation or the actual black-box setting. * *eta*: (default=3) Sets the aggressiveness of Hyperband's aggressive early stopping by retaining 1/eta configurations every round diff --git a/examples/00_interfacing_DEHB.ipynb b/examples/00_interfacing_DEHB.ipynb index 807b124..350087c 100644 --- a/examples/00_interfacing_DEHB.ipynb +++ b/examples/00_interfacing_DEHB.ipynb @@ -40,7 +40,7 @@ "\n", "DEHB also uses Hyperband along with DE, to allow for cheaper approximations of the actual evaluations of $x$. Let $f(x)$ be the validation error of training a multilayer perceptron (MLP) on the complete training set. Multi-fidelity algorithms such as Hyperband, allow for cheaper approximations along a possible *fidelity*. For the MLP, a subset of the dataset maybe a cheaper approximation to the full data set evaluation. Whereas the fidelity can be quantifies as the fraction of the dataset used to evaluate the configuration $x$, instead of the full dataset. Such approximations can allow sneak-peek into the black-box, potentially revealing certain landscape feature of *f(x)*, thus rendering it a *gray*-box and not completely opaque and black! \n", "\n", - "The $z$ parameter is the fidelity parameter to the black-box function. If $z \\in [fidelity_{min}, fidelity_{max}]$, then $f(x, fidelity_{max})$ would be equivalent to the black-box case of $f(x)$.\n", + "The $z$ parameter is the fidelity parameter to the black-box function. If $z \\in [budget_{min}, budget_{max}]$, then $f(x, budget_{max})$ would be equivalent to the black-box case of $f(x)$.\n", "\n", "![boxes](imgs/black-gray-box.png)" ] @@ -62,7 +62,7 @@ "source": [ "def target_function(\n", " x: Union[ConfigSpace.Configuration, List, np.array], \n", - " fidelity: Union[int, float] = None,\n", + " budget: Union[int, float] = None,\n", " **kwargs\n", ") -> Dict:\n", " \"\"\" Target/objective function to optimize\n", @@ -70,7 +70,7 @@ " Parameters\n", " ----------\n", " x : configuration that DEHB wants to evaluate\n", - " fidelity : parameter determining cheaper evaluations\n", + " budget : parameter determining cheaper evaluations\n", " \n", " Returns\n", " -------\n", @@ -83,7 +83,7 @@ " # remove the code snippet below\n", " start = time.time()\n", " y = np.random.uniform() # placeholder response of evaluation\n", - " time.sleep(fidelity) # simulates runtime (mostly proportional to fidelity)\n", + " time.sleep(budget) # simulates runtime (mostly proportional to fidelity)\n", " cost = time.time() - start\n", " \n", " # result dict passed to DE/DEHB as function evaluation output\n", @@ -171,9 +171,8 @@ { "data": { "text/plain": [ - "Configuration(values={\n", - " 'x0': 8.107160631154175,\n", - "})" + "Configuration:\n", + " x0, Value: 3.716302229868112" ] }, "execution_count": 5, @@ -199,7 +198,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Defining fidelity range for the target function" + "### Defining fidelity/budget range for the target function" ] }, { @@ -208,7 +207,7 @@ "metadata": {}, "outputs": [], "source": [ - "min_fidelity, max_fidelity = (0.1, 3) " + "min_budget, max_budget = (0.1, 3) " ] }, { @@ -245,8 +244,8 @@ " f=target_function,\n", " dimensions=dimensions,\n", " cs=cs,\n", - " min_fidelity=min_fidelity,\n", - " max_fidelity=max_fidelity,\n", + " min_budget=min_budget,\n", + " max_budget=max_budget,\n", " output_path=\"./temp\",\n", " n_workers=1 # set to >1 to utilize parallel workers\n", ")\n", @@ -282,9 +281,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Configuration(values={\n", - " 'x0': 4.152073449922892,\n", - "})\n" + "Configuration:\n", + " x0, Value: 4.060258498267547\n", + "\n" ] } ], @@ -309,14 +308,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m2023-10-22 20:03:06.057\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", + "2021-10-22 14:45:56.117 | INFO | dehb.optimizers.dehb:reset:107 - \n", + "\n", + "RESET at 10/22/21 14:45:56 CEST\n", "\n", - "RESET at 10/22/23 20:03:06 CEST\n", "\n", - "\u001b[0m\n", - "(Configuration(values={\n", - " 'x0': 8.96840263375364,\n", - "}), 0.05819975786653586)\n" + "(Configuration:\n", + " x0, Value: 3.724555206841792\n", + ", 0.0938589687572785)\n" ] } ], @@ -344,14 +343,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m2023-10-22 20:03:11.073\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", + "2021-10-22 14:45:58.567 | INFO | dehb.optimizers.dehb:reset:107 - \n", + "\n", + "RESET at 10/22/21 14:45:58 CEST\n", "\n", - "RESET at 10/22/23 20:03:11 CEST\n", "\n", - "\u001b[0m\n", - "(Configuration(values={\n", - " 'x0': 8.708444163420975,\n", - "}), 0.0710929937087792)\n" + "(Configuration:\n", + " x0, Value: 4.341818535733585\n", + ", 3.653636256717441e-05)\n" ] } ], @@ -382,9 +381,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "(Configuration(values={\n", - " 'x0': 8.454086817115218,\n", - "}), 0.016305791635409683)\n" + "(Configuration:\n", + " x0, Value: 4.610766436763522\n", + ", 0.007774399252232556)\n" ] } ], @@ -393,8 +392,8 @@ " f=target_function,\n", " dimensions=dimensions,\n", " cs=cs,\n", - " min_fidelity=min_fidelity,\n", - " max_fidelity=max_fidelity,\n", + " min_budget=min_budget,\n", + " max_budget=max_budget,\n", " output_path=\"./temp\",\n", " n_workers=2\n", ")\n", @@ -414,9 +413,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Configuration(values={\n", - " 'x0': 8.454086817115218,\n", - "})\n" + "Configuration:\n", + " x0, Value: 4.610766436763522\n", + "\n" ] } ], @@ -433,10 +432,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "0.016305791635409683 0.016305791635409683\n", - "Configuration(values={\n", - " 'x0': 8.454086817115218,\n", - "})\n" + "0.007774399252232556 0.007774399252232556\n", + "Configuration:\n", + " x0, Value: 4.610766436763522\n", + "\n" ] } ], @@ -455,7 +454,7 @@ "\n", "As detailed above, the problem definition needs to be input to DEHB as the following information:\n", "* the *target_function* (`f`) that is the primary black-box function to optimize\n", - "* the fidelity range of `min_fidelity` and `max_fidelity` that allows the cheaper, faster gray-box optimization of `f`\n", + "* the fidelity range of `min_budget` and `max_budget` that allows the cheaper, faster gray-box optimization of `f`\n", "* the search space or the input domain of the function `f`, that can be represented as a `ConfigSpace` object and passed to DEHB at initialization\n", "\n", "\n", @@ -466,20 +465,13 @@ "\n", "DEHB will terminate once its chosen runtime budget is exhausted, and report the incumbent found. DEHB, as an *anytime* algorithm, constantly writes to disk a lightweight `json` file with the best found configuration and its score seen till that point." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "dask", "language": "python", - "name": "python3" + "name": "dask" }, "language_info": { "codemirror_mode": { @@ -491,7 +483,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.6.9" } }, "nbformat": 4, diff --git a/examples/01_Optimizing_RandomForest_using_DEHB.ipynb b/examples/01_Optimizing_RandomForest_using_DEHB.ipynb index e5bd359..c35427b 100644 --- a/examples/01_Optimizing_RandomForest_using_DEHB.ipynb +++ b/examples/01_Optimizing_RandomForest_using_DEHB.ipynb @@ -37,7 +37,7 @@ "* `min_samples_split`\n", "* `max_features`\n", "* `min_samples_leaf`\n", - "while the `n_estimators` hyperparameter to the Random Forest is chosen to be a fidelity parameter instead. Lesser number of trees ($<10$) in the Random Forest may not allow adequate ensembling for the grouped prediction to be significantly better than the individual tree predictions. Whereas a large number of trees (~$100$) often give accurate predictions but is naturally slower to train and predict on account of more trees to train. Therefore, a smaller `n_estimators` can be used as a cheaper approximation of the actual fidelity of `n_estimators=100`." + "while the `n_estimators` hyperparameter to the Random Forest is chosen to be a fidelity parameter instead. Lesser number of trees ($<10$) in the Random Forest may not allow adequate ensembling for the grouped prediction to be significantly better than the individual tree predictions. Whereas a large number of trees (~$100$) often give accurate predictions but is naturally slower to train and predict on account of more trees to train. Therefore, a smaller `n_estimators` can be used as a cheaper approximation of the actual budget of `n_estimators=100`." ] }, { @@ -53,7 +53,7 @@ "metadata": {}, "outputs": [], "source": [ - "min_fidelity, max_fidelity = 2, 50" + "min_budget, max_budget = 2, 50" ] }, { @@ -147,7 +147,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now the primary black/gray-box interface to the Random Forest model needs to be built for DEHB to query. As given in the `00_interfacing_DEHB` notebook, this function will have a signature akin to: `target_function(config, fidelity)`, and return a two-element tuple of the `score` and `cost`. It must be noted that DEHB **minimizes** and therefore the `score` being returned by this `target_function` should account for it." + "Now the primary black/gray-box interface to the Random Forest model needs to be built for DEHB to query. As given in the `00_interfacing_DEHB` notebook, this function will have a signature akin to: `target_function(config, budget)`, and return a two-element tuple of the `score` and `cost`. It must be noted that DEHB **minimizes** and therefore the `score` being returned by this `target_function` should account for it." ] }, { @@ -273,23 +273,23 @@ "metadata": {}, "outputs": [], "source": [ - "def target_function(config, fidelity, **kwargs):\n", + "def target_function(config, budget, **kwargs):\n", " # Extracting support information\n", " seed = kwargs[\"seed\"]\n", " train_X = kwargs[\"train_X\"]\n", " train_y = kwargs[\"train_y\"]\n", " valid_X = kwargs[\"valid_X\"]\n", " valid_y = kwargs[\"valid_y\"]\n", - " max_fidelity = kwargs[\"max_fidelity\"]\n", + " max_budget = kwargs[\"max_budget\"]\n", " \n", - " if fidelity is None:\n", - " fidelity = max_fidelity\n", + " if budget is None:\n", + " budget = max_budget\n", " \n", " start = time.time()\n", " # Building model \n", " model = RandomForestClassifier(\n", " **config.get_dictionary(),\n", - " n_estimators=int(fidelity),\n", + " n_estimators=int(budget),\n", " bootstrap=True,\n", " random_state=seed,\n", " )\n", @@ -308,7 +308,7 @@ " \"cost\": cost,\n", " \"info\": {\n", " \"test_score\": test_accuracy,\n", - " \"fidelity\": fidelity\n", + " \"budget\": budget\n", " }\n", " }\n", " return result" @@ -340,8 +340,8 @@ " f=target_function, \n", " cs=cs, \n", " dimensions=dimensions, \n", - " min_fidelity=min_fidelity, \n", - " max_fidelity=max_fidelity,\n", + " min_budget=min_budget, \n", + " max_budget=max_budget,\n", " n_workers=1,\n", " output_path=\"./temp\"\n", ")" @@ -363,7 +363,7 @@ " train_y=train_y,\n", " valid_X=valid_X,\n", " valid_y=valid_y,\n", - " max_fidelity=dehb.max_fidelity\n", + " max_budget=dehb.max_budget\n", ")" ] }, @@ -376,16 +376,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "454 454 454\n", + "473 473 473\n", "\n", "Last evaluated configuration, \n", "Configuration(values={\n", - " 'max_depth': 6,\n", - " 'max_features': 0.6215565437234547,\n", - " 'min_samples_leaf': 8,\n", - " 'min_samples_split': 4,\n", - "})got a score of -1.0, was evaluated at a fidelity of 16.67 and took 0.016 seconds to run.\n", - "The additional info attached: {'test_score': 1.0, 'fidelity': 16.666666666666664}\n" + " 'max_depth': 7,\n", + " 'max_features': 0.669059250229961,\n", + " 'min_samples_leaf': 2,\n", + " 'min_samples_split': 3,\n", + "})\n", + "got a score of -1.0, was evaluated at a budget of 50.00 and took 0.048 seconds to run.\n", + "The additional info attached: {'test_score': 1.0, 'budget': 50.0}\n" ] } ], @@ -394,12 +395,12 @@ "\n", "# Last recorded function evaluation\n", "last_eval = history[-1]\n", - "config, score, cost, fidelity, _info = last_eval\n", + "config, score, cost, budget, _info = last_eval\n", "\n", "print(\"Last evaluated configuration, \")\n", "print(dehb.vector_to_configspace(config), end=\"\")\n", - "print(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"\n", - " \"took {:.3f} seconds to run.\".format(score, fidelity, cost))\n", + "print(\"got a score of {}, was evaluated at a budget of {:.2f} and \"\n", + " \"took {:.3f} seconds to run.\".format(score, budget, cost))\n", "print(\"The additional info attached: {}\".format(_info))" ] }, @@ -419,29 +420,29 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m2023-10-22 20:04:30.731\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", + "\u001b[32m2023-06-22 12:00:41.016\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m107\u001b[0m - \u001b[1m\n", "\n", - "RESET at 10/22/23 20:04:30 CEST\n", + "RESET at 06/22/23 12:00:40 CEST\n", "\n", "\u001b[0m\n", - "\u001b[32m2023-10-22 20:04:41.051\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", + "\u001b[32m2023-06-22 12:00:51.085\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m107\u001b[0m - \u001b[1m\n", "\n", - "RESET at 10/22/23 20:04:41 CEST\n", + "RESET at 06/22/23 12:00:51 CEST\n", "\n", "\u001b[0m\n", - "\u001b[32m2023-10-22 20:04:51.128\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", + "\u001b[32m2023-06-22 12:01:01.182\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m107\u001b[0m - \u001b[1m\n", "\n", - "RESET at 10/22/23 20:04:51 CEST\n", + "RESET at 06/22/23 12:01:01 CEST\n", "\n", "\u001b[0m\n", - "\u001b[32m2023-10-22 20:05:01.200\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", + "\u001b[32m2023-06-22 12:01:11.238\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m107\u001b[0m - \u001b[1m\n", "\n", - "RESET at 10/22/23 20:05:01 CEST\n", + "RESET at 06/22/23 12:01:11 CEST\n", "\n", "\u001b[0m\n", - "\u001b[32m2023-10-22 20:05:11.273\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", + "\u001b[32m2023-06-22 12:01:21.293\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m107\u001b[0m - \u001b[1m\n", "\n", - "RESET at 10/22/23 20:05:11 CEST\n", + "RESET at 06/22/23 12:01:21 CEST\n", "\n", "\u001b[0m\n" ] @@ -465,14 +466,14 @@ " train_y=train_y,\n", " valid_X=valid_X,\n", " valid_y=valid_y,\n", - " max_fidelity=dehb.max_fidelity\n", + " max_budget=dehb.max_budget\n", " )\n", " best_config = dehb.vector_to_configspace(dehb.inc_config)\n", " \n", " # Creating a model using the best configuration found\n", " model = RandomForestClassifier(\n", " **best_config.get_dictionary(),\n", - " n_estimators=int(max_fidelity),\n", + " n_estimators=int(max_budget),\n", " bootstrap=True,\n", " random_state=seed,\n", " )\n", @@ -515,39 +516,44 @@ "output_type": "stream", "text": [ "Configuration(values={\n", - " 'max_depth': 7,\n", - " 'max_features': 0.7162350418245509,\n", + " 'max_depth': 13,\n", + " 'max_features': 0.5412753369058052,\n", + " 'min_samples_leaf': 12,\n", + " 'min_samples_split': 14,\n", + "})\n", + " got an accuracy of 1.0 on the test set.\n", + "\n", + "Configuration(values={\n", + " 'max_depth': 6,\n", + " 'max_features': 0.6764411582074702,\n", " 'min_samples_leaf': 1,\n", - " 'min_samples_split': 18,\n", - "}) got an accuracy of 1.0 on the test set.\n", + " 'min_samples_split': 27,\n", + "})\n", + " got an accuracy of 1.0 on the test set.\n", "\n", "Configuration(values={\n", - " 'max_depth': 11,\n", - " 'max_features': 0.564056444856198,\n", + " 'max_depth': 5,\n", + " 'max_features': 0.5862915814751853,\n", " 'min_samples_leaf': 2,\n", - " 'min_samples_split': 7,\n", - "}) got an accuracy of 1.0 on the test set.\n", + " 'min_samples_split': 22,\n", + "})\n", + " got an accuracy of 1.0 on the test set.\n", "\n", "Configuration(values={\n", - " 'max_depth': 9,\n", - " 'max_features': 0.7477652209361112,\n", - " 'min_samples_leaf': 1,\n", - " 'min_samples_split': 7,\n", - "}) got an accuracy of 1.0 on the test set.\n", + " 'max_depth': 14,\n", + " 'max_features': 0.5346143393392929,\n", + " 'min_samples_leaf': 5,\n", + " 'min_samples_split': 9,\n", + "})\n", + " got an accuracy of 1.0 on the test set.\n", "\n", "Configuration(values={\n", - " 'max_depth': 9,\n", - " 'max_features': 0.6510861760309854,\n", + " 'max_depth': 4,\n", + " 'max_features': 0.5541455312635835,\n", " 'min_samples_leaf': 4,\n", - " 'min_samples_split': 24,\n", - "}) got an accuracy of 1.0 on the test set.\n", - "\n", - "Configuration(values={\n", - " 'max_depth': 6,\n", - " 'max_features': 0.5989756936409275,\n", - " 'min_samples_leaf': 2,\n", - " 'min_samples_split': 4,\n", - "}) got an accuracy of 1.0 on the test set.\n", + " 'min_samples_split': 10,\n", + "})\n", + " got an accuracy of 1.0 on the test set.\n", "\n" ] } @@ -557,13 +563,6 @@ " print(\"{} got an accuracy of {} on the test set.\".format(config, score))\n", " print()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/examples/02_using DEHB_without_ConfigSpace.ipynb b/examples/02_using DEHB_without_ConfigSpace.ipynb index 79c987d..11264c6 100644 --- a/examples/02_using DEHB_without_ConfigSpace.ipynb +++ b/examples/02_using DEHB_without_ConfigSpace.ipynb @@ -61,7 +61,7 @@ "dimensions = len(param_space)\n", "\n", "# Declaring the fidelity range\n", - "min_fidelity, max_fidelity = 2, 50\n", + "min_budget, max_budget = 2, 50\n", "\n", "\n", "def transform_space(param_space, configuration):\n", @@ -164,27 +164,27 @@ " return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset\n", "\n", "\n", - "def target_function(config, fidelity, **kwargs):\n", + "def target_function(config, budget, **kwargs):\n", " # Extracting support information\n", " seed = kwargs[\"seed\"]\n", " train_X = kwargs[\"train_X\"]\n", " train_y = kwargs[\"train_y\"]\n", " valid_X = kwargs[\"valid_X\"]\n", " valid_y = kwargs[\"valid_y\"]\n", - " max_fidelity = kwargs[\"max_fidelity\"]\n", + " max_budget = kwargs[\"max_budget\"]\n", " \n", " # Mapping [0, 1]-vector to Sklearn parameters\n", " param_space = kwargs[\"param_space\"]\n", " config = transform_space(param_space, config)\n", " \n", - " if fidelity is None:\n", - " fidelity = max_fidelity\n", + " if budget is None:\n", + " budget = max_budget\n", " \n", " start = time.time()\n", " # Building model \n", " model = RandomForestClassifier(\n", " **config,\n", - " n_estimators=int(fidelity),\n", + " n_estimators=int(budget),\n", " bootstrap=True,\n", " random_state=seed,\n", " )\n", @@ -203,7 +203,7 @@ " \"cost\": cost,\n", " \"info\": {\n", " \"test_score\": test_accuracy,\n", - " \"fidelity\": fidelity\n", + " \"budget\": budget\n", " }\n", " }\n", " return result\n", @@ -238,8 +238,8 @@ "dehb = DEHB(\n", " f=target_function, \n", " dimensions=dimensions, \n", - " min_fidelity=min_fidelity, \n", - " max_fidelity=max_fidelity,\n", + " min_budget=min_budget, \n", + " max_budget=max_budget,\n", " n_workers=1,\n", " output_path=\"./temp\"\n", ")" @@ -260,7 +260,7 @@ " train_y=train_y,\n", " valid_X=valid_X,\n", " valid_y=valid_y,\n", - " max_fidelity=dehb.max_fidelity,\n", + " max_budget=dehb.max_budget,\n", " param_space=param_space\n", ")" ] @@ -274,9 +274,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Incumbent score: -0.9611111111111111\n", + "Incumbent score: -0.9685185185185186\n", "Incumbent configuration:\n", - "{'max_depth': 9, 'min_samples_split': 3, 'max_features': 0.3990411414400532, 'min_samples_leaf': 1}\n" + "{'max_depth': 10, 'min_samples_split': 3, 'max_features': 0.24012458257841524, 'min_samples_leaf': 2}\n" ] } ], @@ -301,14 +301,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Test accuracy: 0.9944444444444445\n" + "Test accuracy: 1.0\n" ] } ], "source": [ "model = RandomForestClassifier(\n", " **transform_space(param_space, dehb.inc_config),\n", - " n_estimators=int(max_fidelity),\n", + " n_estimators=int(max_budget),\n", " bootstrap=True,\n", " random_state=seed,\n", ")\n", @@ -334,12 +334,14 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGwCAYAAABFFQqPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABKRklEQVR4nO3de1xUdf4/8NcMMDOoMKYoeAHxgimCoIAIGnhhQ9dMyl+hWSC2Vq6kRlliCqkpWmpY+k1zTe1ikttmpebGTuIVQ0FM8m4aeBkQU0bBuM35/WGMO8uoDAznDMzr+XjM4wFnPnPmPbNtvPqc9/l8ZIIgCCAiIiKyQXKpCyAiIiKSCoMQERER2SwGISIiIrJZDEJERERksxiEiIiIyGYxCBEREZHNYhAiIiIim2UvdQHWSK/X4/Lly3BycoJMJpO6HCIiIqoDQRBw8+ZNdOzYEXJ53eZ6GIRMuHz5Mtzd3aUug4iIiOqhoKAAnTt3rtNYBiETnJycANz5Ip2dnSWuhoiIiOpCp9PB3d3d8He8LhiETKi5HObs7MwgRERE1MSY09bCZmkiIiKyWQxCREREZLMYhIiIiMhmMQgRERGRzbKKILRq1Sp4enpCpVIhODgYWVlZ9xz7yy+/YOzYsfD09IRMJkNqamqtMSkpKQgKCoKTkxPat2+PqKgonDp1qhE/ARERETVFkgehtLQ0JCQkIDk5GTk5OfDz80NkZCSKiopMji8rK0O3bt2wePFiuLm5mRyze/duTJ06FQcPHkR6ejoqKyvx6KOPorS0tDE/ChERETUxMkEQBCkLCA4ORlBQEFauXAngzqrO7u7uePnllzFr1qz7vtbT0xMzZszAjBkz7jvu6tWraN++PXbv3o2wsLAH1qTT6aBWq1FSUsLb54mIiJqI+vz9lnRGqKKiAtnZ2YiIiDAck8vliIiIQGZmpsXep6SkBADQpk0bk8+Xl5dDp9MZPYiIiKj5kzQIFRcXo7q6Gq6urkbHXV1dodVqLfIeer0eM2bMwKBBg+Dj42NyTEpKCtRqteHB7TWIiIhsg+Q9Qo1t6tSpyMvLw+bNm+85JjExESUlJYZHQUGBiBUSERGRVCTdYsPFxQV2dnYoLCw0Ol5YWHjPRmhzxMfHY9u2bdizZ899N19TKpVQKpUNfj8iIiJqWiSdEVIoFAgICIBGozEc0+v10Gg0CAkJqfd5BUFAfHw8vv76a/z444/o2rWrJcolIiKiZkbyTVcTEhIQGxuLwMBADBgwAKmpqSgtLUVcXBwAICYmBp06dUJKSgqAOw3Wx48fN/x86dIl5ObmolWrVujRoweAO5fDNm3ahG+++QZOTk6GfiO1Wg1HR0cJPuUdZRVV+L20wuRzTkoHqFs4iFwRERGRbZP89nkAWLlyJd59911otVr4+/vj/fffR3BwMABgyJAh8PT0xIYNGwAAFy5cMDnDEx4ejoyMDAD33nV2/fr1mDhx4gPraazb5789ehnTvjhi8jl7uQyf/S0YA7u1tdj7ERER2ZL6/P2WfEYIuNPLEx8fb/K5mnBTw9PTEw/KblaQ7Uyyk8mgtK99NbKyWo8qvYC8SyUMQkRERCKyiiBkK0b17YBRfTvUOj5zy1Fsyb6IymrrDHBERETNVbO/fb4pcPhzlqiyWi9xJURERLaFQcgKOMjv9DQxCBEREYmLQcgKONjd+Z+hgkGIiIhIVAxCVsBwaayKPUJERERiYhCyAjUzQrw0RkREJC4GISugsGOPEBERkRQYhKwAe4SIiIikwSBkBWqCUBXXESIiIhIVg5AV4DpCRERE0mAQsgLsESIiIpIGg5AVuNsjxEtjREREYmIQsgKG2+erOCNEREQkJgYhK8B1hIiIiKTBIGQFFPbsESIiIpICg5AVYI8QERGRNBiErIC9nJfGiIiIpMAgZAV4aYyIiEgaDEJWgHeNERERSYNByAqwR4iIiEgaDEJWgLfPExERSYNByAooGISIiIgkwSBkBRz+bJbm7vNERETiYhCyAnd7hPQQBIYhIiIisTAIWYGaIAQAVXoGISIiIrEwCFkBxX8FIfYJERERiYdByAo42MkMP1dWcUaIiIhILAxCVsBOLoPszyxUwRkhIiIi0TAIWQGZTMa1hIiIiCTAIGQluJYQERGR+BiErIS9HTdeJSIiEpvkQWjVqlXw9PSESqVCcHAwsrKy7jn2l19+wdixY+Hp6QmZTIbU1NRaY/bs2YPRo0ejY8eOkMlk2Lp1a+MVb0GGtYTYLE1ERCQaSYNQWloaEhISkJycjJycHPj5+SEyMhJFRUUmx5eVlaFbt25YvHgx3NzcTI4pLS2Fn58fVq1a1ZilWxwvjREREYnPXso3X758OSZPnoy4uDgAwOrVq7F9+3Z8/PHHmDVrVq3xQUFBCAoKAgCTzwPAyJEjMXLkyMYrupE48NIYERGR6CSbEaqoqEB2djYiIiLuFiOXIyIiApmZmaLWUl5eDp1OZ/QQ239vs0FERETikCwIFRcXo7q6Gq6urkbHXV1dodVqRa0lJSUFarXa8HB3dxf1/YG7QYgbrxIREYlH8mZpa5CYmIiSkhLDo6CgQPQaHOzZI0RERCQ2yXqEXFxcYGdnh8LCQqPjhYWF92yEbixKpRJKpVLU9/xfCvYIERERiU6yGSGFQoGAgABoNBrDMb1eD41Gg5CQEKnKkszdHiFeGiMiIhKLpHeNJSQkIDY2FoGBgRgwYABSU1NRWlpquIssJiYGnTp1QkpKCoA7DdbHjx83/Hzp0iXk5uaiVatW6NGjBwDg1q1bOHv2rOE9zp8/j9zcXLRp0wYeHh4if8K6M2yxUcUZISIiIrFIGoSio6Nx9epVJCUlQavVwt/fHzt37jQ0UOfn50MuvztpdfnyZfTr18/w+9KlS7F06VKEh4cjIyMDAHD48GEMHTrUMCYhIQEAEBsbiw0bNjT+h6on7jVGREQkPkmDEADEx8cjPj7e5HM14aaGp6cnBOH+l46GDBnywDHWSGHPHiEiIiKx8a4xK8EeISIiIvExCFkJXhojIiISH4OQlTBsscFmaSIiItEwCFkJzggRERGJj0HISrBHiIiISHwMQlaCM0JERETiYxCyEtxig4iISHwMQlbi7owQL40RERGJhUHISnD3eSIiIvExCFkJ9ggRERGJj0HISrBHiIiISHwMQlbCcPt8FXuEiIiIxMIgZCV4aYyIiEh8DEJWgs3SRERE4mMQshLsESIiIhIfg5CV4BYbRERE4mMQshL2NT1C3H2eiIhINAxCVsKBl8aIiIhExyBkJRS8a4yIiEh0DEJWgnuNERERiY9ByErcbZbmjBAREZFYGISshML+To9QFYMQERGRaBiErAQvjREREYmPQchK8NIYERGR+BiErMR/7zUmCJwVIiIiEgODkJWouX1eEIBqPYMQERGRGBiErITDn83SAPuEiIiIxMIgZCVqLo0B7BMiIiISC4OQlbCX//eMEIMQERGRGBiErIRMJuN+Y0RERCJjELIihjvHqtgjREREJAarCEKrVq2Cp6cnVCoVgoODkZWVdc+xv/zyC8aOHQtPT0/IZDKkpqY2+JzWgmsJERERiUvyIJSWloaEhAQkJycjJycHfn5+iIyMRFFRkcnxZWVl6NatGxYvXgw3NzeLnNNaOHAHeiIiIlFJHoSWL1+OyZMnIy4uDt7e3li9ejVatGiBjz/+2OT4oKAgvPvuuxg3bhyUSqVFzmktFOwRIiIiEpWkQaiiogLZ2dmIiIgwHJPL5YiIiEBmZqZo5ywvL4dOpzN6SMHBnjNCREREYpI0CBUXF6O6uhqurq5Gx11dXaHVakU7Z0pKCtRqteHh7u5er/duKG68SkREJC7JL41Zg8TERJSUlBgeBQUFktTBHiEiIiJx2Uv55i4uLrCzs0NhYaHR8cLCwns2QjfGOZVK5T37jcTEHiEiIiJxSTojpFAoEBAQAI1GYzim1+uh0WgQEhJiNecUi+H2ea4jREREJApJZ4QAICEhAbGxsQgMDMSAAQOQmpqK0tJSxMXFAQBiYmLQqVMnpKSkALjTDH38+HHDz5cuXUJubi5atWqFHj161Omc1oqXxoiIiMQleRCKjo7G1atXkZSUBK1WC39/f+zcudPQ7Jyfnw+5/O7E1eXLl9GvXz/D70uXLsXSpUsRHh6OjIyMOp3TWvGuMSIiInHJBEHgdZj/odPpoFarUVJSAmdnZ9He928bD+E/J4qwZKwvooM8RHtfIiKi5qA+f79515gVubvFBrMpERGRGBiErIi9YdNVXhojIiISA4OQFXHg7fNERESiYhCyIgreNUZERCQqBiErwh4hIiIicTEIWRGuI0RERCQuBiEr4mD/Z48Qm6WJiIhEwSBkRWp6hKr0vDRGREQkBgYhK3K3R4gzQkRERGJgELIiDlxHiIiISFQMQlaE6wgRERGJi0HIiigMm66yR4iIiEgM9dp9Pj8/H7/99hvKysrQrl079OnTB0ql0tK12Rz2CBEREYmrzkHowoUL+PDDD7F582ZcvHgR/71pvUKhwCOPPIIXXngBY8eOhVzOiab64DpCRERE4qpTYpk2bRr8/Pxw/vx5vP322zh+/DhKSkpQUVEBrVaLHTt2YPDgwUhKSkLfvn1x6NChxq67WWKPEBERkbjqNCPUsmVL/Prrr2jbtm2t59q3b49hw4Zh2LBhSE5Oxs6dO1FQUICgoCCLF9vc3b1rjD1CREREYqhTEEpJSanzCUeMGFHvYmwde4SIiIjEZXYzT3JyMn777bfGqMXm8dIYERGRuMwOQt988w26d++O4cOHY9OmTSgvL2+MumySgs3SREREojI7COXm5uLQoUPo06cPpk+fDjc3N0yZMoUN0hbgwHWEiIiIRFWv+9z79euH999/H5cvX8a6detw8eJFDBo0CH379sWKFStQUlJi6TptgqFHiFtsEBERiaJBC/4IgoDKykpUVFRAEAQ89NBDWLlyJdzd3ZGWlmapGm1GTY9QlZ5BiIiISAz1CkLZ2dmIj49Hhw4d8Morr6Bfv344ceIEdu/ejTNnzmDhwoWYNm2apWtt9u72CPHSGBERkRjMDkK+vr4YOHAgzp8/j3Xr1qGgoACLFy9Gjx49DGPGjx+Pq1evWrRQW8Dd54mIiMRl9l5jTz/9NCZNmoROnTrdc4yLiwv0vLxjtppmaa4jREREJA6zg9DcuXMbow4C1xEiIiISm9mXxsaOHYslS5bUOv7OO+/gqaeeskhRtqqmR0gvANV69gkRERE1NrOD0J49e/DXv/611vGRI0diz549FinKVtX0CAGcFSIiIhKD2UHo1q1bUCgUtY47ODhAp9NZpChb9d9BiH1CREREja9ed42ZWiNo8+bN8Pb2tkhRtqqmRwjgnWNERERiqFez9JNPPolz585h2LBhAACNRoMvvvgCW7ZssXiBtkQmk8FeLkOVXuBaQkRERCIwOwiNHj0aW7duxaJFi/DPf/4Tjo6O6Nu3L/7zn/8gPDy8MWq0KQ52clTpq9kjREREJIJ6rSw9atQo7N+/H6WlpSguLsaPP/7YoBC0atUqeHp6QqVSITg4GFlZWfcdv2XLFvTq1QsqlQq+vr7YsWOH0fOFhYWYOHEiOnbsiBYtWmDEiBE4c+ZMvesTU83lMfYIERERNb4G7TVmCWlpaUhISEBycjJycnLg5+eHyMhIFBUVmRx/4MABjB8/Hs8//zyOHDmCqKgoREVFIS8vD8Cd/c+ioqLw66+/4ptvvsGRI0fQpUsXREREoLS0VMyPVi8Kww70DEJERESNTSYIglnNKNXV1Xjvvffw5ZdfIj8/HxUVFUbP//7772YVEBwcjKCgIKxcuRIAoNfr4e7ujpdffhmzZs2qNT46OhqlpaXYtm2b4djAgQPh7++P1atX4/Tp03j44YeRl5eHPn36GM7p5uaGRYsW4W9/+1utc5aXl6O8vNzwu06ng7u7O0pKSuDs7GzW52mokBQNrpT8gW0vD4ZPJ7Wo701ERNSU6XQ6qNVqs/5+mz0jNG/ePCxfvhzR0dEoKSlBQkICnnzyScjlcrz11ltmnauiogLZ2dmIiIi4W5BcjoiICGRmZpp8TWZmptF4AIiMjDSMrwk0KpXK6JxKpRL79u0zec6UlBSo1WrDw93d3azPYUk1t9Dz0hgREVHjMzsIff7551i7di1effVV2NvbY/z48fjHP/6BpKQkHDx40KxzFRcXo7q6Gq6urkbHXV1dodVqTb5Gq9Xed3yvXr3g4eGBxMREXL9+HRUVFViyZAkuXryIK1eumDxnYmIiSkpKDI+CggKzPoclGbbZ4O3zREREjc7sIKTVauHr6wsAaNWqFUpKSgAAjz32GLZv327Z6urBwcEB//rXv3D69Gm0adMGLVq0wK5duzBy5EjI5aY/rlKphLOzs9FDKoYd6Hn7PBERUaMzOwh17tzZMLPSvXt3/PDDDwCAQ4cOQalUmnUuFxcX2NnZobCw0Oh4YWEh3NzcTL7Gzc3tgeMDAgKQm5uLGzdu4MqVK9i5cyeuXbuGbt26mVWfFNgsTUREJB6zg9ATTzwBjUYDAHj55Zcxd+5ceHl5ISYmBpMmTTLrXAqFAgEBAYbzAXcamzUaDUJCQky+JiQkxGg8AKSnp5scr1ar0a5dO5w5cwaHDx/GmDFjzKpPCuwRIiIiEo/ZCyouXrzY8HN0dDS6dOmCAwcOwMvLC6NHjza7gISEBMTGxiIwMBADBgxAamoqSktLERcXBwCIiYlBp06dkJKSAgCYPn06wsPDsWzZMowaNQqbN2/G4cOH8dFHHxnOuWXLFrRr1w4eHh44duwYpk+fjqioKDz66KNm1yc2Q48QgxAREVGjMysIVVZW4sUXX8TcuXPRtWtXAHduXR84cGC9C4iOjsbVq1eRlJQErVYLf39/7Ny509AQnZ+fb9TbExoaik2bNmHOnDmYPXs2vLy8sHXrVvj4+BjGXLlyBQkJCSgsLESHDh0QExODuXPn1rtGMd3tEWIQIiIiamxmryOkVquRm5trCELNUX3WIbCU5zccguZkEd4Z2xdPB0l3Gz8REVFTI8o6QlFRUdi6dau5L6M6sucWG0RERKIxu0fIy8sL8+fPx/79+xEQEICWLVsaPT9t2jSLFWeLeGmMiIhIPGYHoXXr1qF169bIzs5Gdna20XMymYxBqIEUDEJERESiMTsInT9/vjHqoD9xQUUiIiLxSL77PBlzsP+zR4hbbBARETU6s2eEHrRo4scff1zvYujujFCVnkGIiIiosZkdhK5fv270e2VlJfLy8nDjxg0MGzbMYoXZKgUvjREREYnG7CD09ddf1zqm1+sxZcoUdO/e3SJF2TLDFhu8NEZERNToLNIjJJfLkZCQgPfee88Sp7NpvH2eiIhIPBZrlj537hyqqqosdTqbVdMszSBERETU+My+NJaQkGD0uyAIuHLlCrZv347Y2FiLFWar2CNEREQkHrOD0JEjR4x+l8vlaNeuHZYtW/bAO8rowQw9QpwRIiIianRmB6Fdu3Y1Rh30J0OPEJuliYiIGp3ZPULnz5/HmTNnah0/c+YMLly4YImabJqDHXuEiIiIxGJ2EJo4cSIOHDhQ6/hPP/2EiRMnWqImm8YtNoiIiMRjdhA6cuQIBg0aVOv4wIEDkZuba4mabBp7hIiIiMRjdhCSyWS4efNmreMlJSWorq62SFG2jJfGiIiIxGN2EAoLC0NKSopR6KmurkZKSgoGDx5s0eJskYM9F1QkIiISi9l3jS1ZsgRhYWF4+OGH8cgjjwAA9u7dC51Ohx9//NHiBdoawzpCVewRIiIiamxmByFvb2/8/PPPWLlyJY4ePQpHR0fExMQgPj4ebdq0aYwabUpNj9ClG7cRvynHcNy9TQu89ujDsJPLpCqNiIio2TE7CAFAx44dsWjRIkvXQgDaOSkBALfKq7Dt5ytGzwV5PoRhvVylKIuIiKhZMjsIrV+/Hq1atcJTTz1ldHzLli0oKyvjNhsN1NWlJdbHBeFCcanh2H9OFGL/2WvYc7qYQYiIiMiCzA5CKSkpWLNmTa3j7du3xwsvvMAgZAFDH24PPHz39w5qxz+D0FXpiiIiImqGzL5rLD8/H127dq11vEuXLsjPz7dIUWQstEdb2Mll+LW4FAW/l0ldDhERUbNhdhBq3749fv7551rHjx49irZt21qkKDLmrHJAP/fWAIC9Z4qlLYaIiKgZMTsIjR8/HtOmTcOuXbtQXV2N6upq/Pjjj5g+fTrGjRvXGDUSgLCe7QCAl8eIiIgsyOweoQULFuDChQsYPnw47O3vvFyv1yMmJoZ3kjWiR7xcsDz9NPafK0ZVtR72dmZnWCIiIvofZgchhUKBtLQ0LFiwwLCOkK+vL7p06dIY9dGf+nZujdYtHHCjrBJHL95AQBeu2URERNRQ9VpHCAB69uyJnj17WrIWug87uQyDerhg+89XsPt0MYMQERGRBdQrCF28eBHffvst8vPzUVFRYfTc8uXLLVIY1RbmdScI7T1zFQl/YQglIiJqKLODkEajweOPP45u3brh5MmT8PHxwYULFyAIAvr3798YNdKfahqmjxbcQElZJdQtHCSuiIiIqGkzu+M2MTERr732Go4dOwaVSoWvvvoKBQUFCA8Pr7XadF2tWrUKnp6eUKlUCA4ORlZW1n3Hb9myBb169YJKpYKvry927Nhh9PytW7cQHx+Pzp07w9HREd7e3li9enW9arMmHdSO8GrfCnoB2HeWt9ETERE1lNlB6MSJE4iJiQEA2Nvb4/bt22jVqhXmz5+PJUuWmF1AWloaEhISkJycjJycHPj5+SEyMhJFRUUmxx84cADjx4/H888/jyNHjiAqKgpRUVHIy8szjElISMDOnTvx2Wef4cSJE5gxYwbi4+Px7bffml2ftXnE686s0N4zvI2eiIioocwOQi1btjT0BXXo0AHnzp0zPFdcbP4sxfLlyzF58mTExcUZZm5atGiBjz/+2OT4FStWYMSIEZg5cyZ69+6NBQsWoH///li5cqVhzIEDBxAbG4shQ4bA09MTL7zwAvz8/O4501ReXg6dTmf0sFZhPV0A3FlPSBAEiashIiJq2swOQgMHDsS+ffsAAH/961/x6quvYuHChZg0aRIGDhxo1rkqKiqQnZ2NiIiIuwXJ5YiIiEBmZqbJ12RmZhqNB4DIyEij8aGhofj2229x6dIlCIKAXbt24fTp03j00UdNnjMlJQVqtdrwcHd3N+tziCm4a1so7OW4XPIHzl0tffALiIiI6J7MDkLLly9HcHAwAGDevHkYPnw40tLS4OnpiXXr1pl1ruLiYlRXV8PV1XhHdVdXV2i1WpOv0Wq1Dxz/wQcfwNvbG507d4ZCocCIESOwatUqhIWFmTxnYmIiSkpKDI+CggKzPoeYHBV2GOB559Z5rjJNRETUMGbfNdatWzfDzy1btrTKJuQPPvgABw8exLfffosuXbpgz549mDp1Kjp27FhrNgkAlEollEqlBJXWT1hPF+w7W4w9Z65i0uDaG+ASERFR3dR7QUVLcHFxgZ2dHQoLC42OFxYWws3NzeRr3Nzc7jv+9u3bmD17Nr7++muMGjUKANC3b1/k5uZi6dKlJoNQU3OnYfokDv56DeVV1VDa20ldEhERUZMk6YZVCoUCAQEB0Gg0hmN6vR4ajQYhISEmXxMSEmI0HgDS09MN4ysrK1FZWQm53Pij2dnZQa/XW/gTSKOXmxPaOynxR6Uehy9cl7ocIiKiJkvynTsTEhKwdu1abNy4ESdOnMCUKVNQWlqKuLg4AEBMTAwSExMN46dPn46dO3di2bJlOHnyJN566y0cPnwY8fHxAABnZ2eEh4dj5syZyMjIwPnz57FhwwZ88skneOKJJyT5jJYmk8kMt9GzT4iIiKj+JL00BgDR0dG4evUqkpKSoNVq4e/vj507dxoaovPz841md0JDQ7Fp0ybMmTMHs2fPhpeXF7Zu3QofHx/DmM2bNyMxMRETJkzA77//ji5dumDhwoV46aWXRP98jSWspwu+yrmIPWeKkfjg4URERGSCTGjAYjT79+9HYGBgk2o0rgudTge1Wo2SkhI4OztLXY5J126VI3DhfyAIQNabw9HeSSV1SURERJKqz9/vBl0aGzlyJC5dutSQU1A9tW2lhE9HNQBg72lut0FERFQfDQpCXNlYWo943VllmtttEBER1Y/kzdJUfzW70e89Uwy9nqGUiIjIXA0KQmvWrKm1yjOJp7/HQ2ipsMO10gocv2K9+6MRERFZqwYFoWeeeQYtW7a0VC1kJoW9HCHd2wIA9vDyGBERkdl4aayJq7k8xvWEiIiIzMcg1MSF/bmwYvZv11FaXiVxNURERE0Lg1AT16VtC7i3cURltYCDv16TuhwiIqImhUGoiZPJZIZZIV4eIyIiMo/FgtD169fxySefWOp0ZIaafcf2nuHCikREROawWBDKz883bJRK4grt0RZ2chl+LS5Fwe9lUpdDRETUZNR501Wd7v7r1Ny8ebPBxVD9OKsc0N+jNQ5duI49Z65iQnAXqUsiIiJqEuochFq3bg2ZTHbP5wVBuO/z1Lge8WqHQxeuY+/pYgYhIiKiOqpzEHJycsKbb76J4OBgk8+fOXMGL774osUKI/OE9WyH5emnsf9cMaqq9bC3Yx88ERHRg9Q5CPXv3x8AEB4ebvL51q1bcxNWCfl2UqN1CwfcKKtEbsENBHq2kbokIiIiq1fnaYNnnnkGKpXqns+7ubkhOTnZIkWR+ezkMgzqcWc3+j28e4yIiKhOZAKncWrR6XRQq9UoKSmBs7Oz1OXU2ZeHCvD6Vz+jU2tHDO3VTpT3bNdKhSlDukNhz0txREQkrfr8/a7zpTGyfmE920EuAy7duI3PDuaL9r5OKntMGtxVtPcjIiKylDoFoc2bN2PcuHF1OmFBQQHy8/MxaNCgBhVG5nNTq/CP2EAcLSgR5f1+u1aKrbmX8dGeXzFhoAeU9naivC8REZGl1CkIffjhh5g3bx7i4uIwevRo9O7d2+j5kpIS7N+/H5999hnS09Oxbt26RimWHmxYL1cM6+UqynuVV1Uj89dr0Or+wL9yLmH8AA9R3peIiMhS6tTYsXv3bixZsgTp6enw8fGBs7MzvLy84Ovri86dO6Nt27aYNGkSPDw8kJeXh8cff7yx6yYroLS3w+RHugEAPsw4h6pqvcQVERERmcfsZuni4mLs27cPv/32G27fvg0XFxf069cP/fr1g1zePBpmm2qztBTKKqoweMku/F5agfei/fBEv85Sl0RERDZKlGZpFxcXREVFmfsyaqZaKOzx/OCuePffp/B/u85hjF8nyOVcYZyIiJqG5jGFQ5J6LqQLnFT2OFN0Cz8c10pdDhERUZ0xCFGDOascEBviCQBYuessVxgnIqImg0GILGLS4K5wdLBD3iUddp++KnU5REREdcIgRBbRpqUCE4Lv3D6/atdZiashIiKqG7OD0Pz581FWVlbr+O3btzF//nyLFEVN0+SwblDYyXHownX89Os1qcshIiJ6ILOD0Lx583Dr1q1ax8vKyjBv3jyLFEVNk6uzCk8F3rl9fiVnhYiIqAkwOwgJggCZrPbt0UePHkWbNm0sUhQ1XS+Fd4edXIa9Z4pxtOCG1OUQERHdV52D0EMPPYQ2bdpAJpOhZ8+eaNOmjeGhVqvxl7/8BU8//XRj1kpNgHubFhjj3xEAZ4WIiMj61TkIpaamYvny5RAEAfPmzcN7771neKxevRr79u3DqlWr6lXEqlWr4OnpCZVKheDgYGRlZd13/JYtW9CrVy+oVCr4+vpix44dRs/LZDKTj3fffbde9ZF5/j6kB2QyIP14IU5qdVKXQ0REdE9mb7Gxe/duDBo0CPb2Zi9KbVJaWhpiYmKwevVqBAcHIzU1FVu2bMGpU6fQvn37WuMPHDiAsLAwpKSk4LHHHsOmTZuwZMkS5OTkwMfHBwCg1Rov6vf999/j+eefx9mzZ9GtW7cH1sQtNhpu6uc52H7sChzsZLCvw9YrchkwZUh3xA/zEqE6IiJqjurz99vsIAQA586dw/r163Hu3DmsWLEC7du3x/fffw8PDw/06dPHrHMFBwcjKCgIK1euBADo9Xq4u7vj5ZdfxqxZs2qNj46ORmlpKbZt22Y4NnDgQPj7+2P16tUm3yMqKgo3b96ERqOpU00MQg13SnsTj6/ch/Kqum/E2tO1FX54JbwRqyIiouZMlL3Gdu/ejZEjR2LQoEHYs2cPFi5ciPbt2+Po0aNYt24d/vnPf9b5XBUVFcjOzkZiYqLhmFwuR0REBDIzM02+JjMzEwkJCUbHIiMjsXXrVpPjCwsLsX37dmzcuPGedZSXl6O8vNzwu07HyzkN9bCbEw7NiUBJWeUDxx67VIK/f56DCjNCExERkSWYfdfYrFmz8PbbbyM9PR0KhcJwfNiwYTh48KBZ5youLkZ1dTVcXV2Njru6uta6vFVDq9WaNX7jxo1wcnLCk08+ec86UlJSoFarDQ93d3ezPgeZ5qxygHubFg98dGrtCAAMQkREJDqzg9CxY8fwxBNP1Drevn17FBcXW6QoS/r4448xYcIEqFSqe45JTExESUmJ4VFQUCBihaSwv/OPYUU19ygjIiJxmX1prHXr1rhy5Qq6du1qdPzIkSPo1KmTWedycXGBnZ0dCgsLjY4XFhbCzc3N5Gvc3NzqPH7v3r04deoU0tLS7luHUqmEUqk0q3ayHEMQqqqWuBIiIrI1Zs8IjRs3Dm+88Qa0Wi1kMhn0ej3279+P1157DTExMWadS6FQICAgwKiJWa/XQ6PRICQkxORrQkJCajU9p6enmxy/bt06BAQEwM/Pz6y6SFwKu5oZIV4aIyIicZkdhBYtWoRevXrB3d0dt27dgre3N8LCwhAaGoo5c+aYXUBCQgLWrl2LjRs34sSJE5gyZQpKS0sRFxcHAIiJiTFqpp4+fTp27tyJZcuW4eTJk3jrrbdw+PBhxMfHG51Xp9Nhy5Yt+Nvf/mZ2TSQupWFGiEGIiIjEZdalMUEQoNVq8f777yMpKQnHjh3DrVu30K9fP3h51W/9l+joaFy9ehVJSUnQarXw9/fHzp07DQ3R+fn5kP/XOjShoaHYtGkT5syZg9mzZ8PLywtbt241rCFUY/PmzRAEAePHj69XXSSemktjegGoqtbD3s7sfE5ERFQvZq0jpNfroVKp8Msvv9Q7+DQFXEdIXKXlVeiT/G8AwPH5kWihsMxinUREZFvq8/fbrP/0lsvl8PLywrVr1+pVIJEpNTNCAFBZxTvHiIhIPGZfg1i8eDFmzpyJvLy8xqiHbJC9XAaZ7M7P5dW8c4yIiMRj9jWImJgYlJWVwc/PDwqFAo6OjkbP//777xYrjmyDTCaDwk6O8io9G6aJiEhUZgeh1NTURiiDbJ3CnkGIiIjEZ3YQio2NbYw6yMYp7eW4Ca4lRERE4jI7CN1rQ1KZTAalUmm0/xhRXTnYcS0hIiISX7222JDVdLaa0LlzZ0ycOBHJyclG6/8Q3U/NnWOVnBEiIiIRmR2ENmzYgDfffBMTJ07EgAEDAABZWVnYuHEj5syZg6tXr2Lp0qVQKpWYPXu2xQum5qlmm41yzggREZGIzA5CGzduxLJly/D0008bjo0ePRq+vr5Ys2YNNBoNPDw8sHDhQgYhqjMFt9kgIiIJmH3t6sCBA+jXr1+t4/369UNmZiYAYPDgwcjPz294dWQzGISIiEgKZgchd3d3rFu3rtbxdevWwd3dHQBw7do1PPTQQw2vjmyGA3egJyIiCZh9aWzp0qV46qmn8P333yMoKAgAcPjwYZw8eRL//Oc/AQCHDh1CdHS0ZSulZo070BMRkRTMDkKPP/44Tp48iTVr1uD06dMAgJEjR2Lr1q3w9PQEAEyZMsWiRVLzV9MszbvGiIhITPXa5rtr165YvHixpWshG8YeISIikkK9FvrZu3cvnn32WYSGhuLSpUsAgE8//RT79u2zaHFkO2qCEG+fJyIiMZkdhL766itERkbC0dEROTk5KC8vBwCUlJRg0aJFFi+QbIOCzdJERCQBs4PQ22+/jdWrV2Pt2rVwcHAwHB80aBBycnIsWhzZDgdeGiMiIgmYHYROnTqFsLCwWsfVajVu3LhhiZrIBim41xgREUnA7CDk5uaGs2fP1jq+b98+dOvWzSJFke1Rcq8xIiKSgNlBaPLkyZg+fTp++uknyGQyXL58GZ9//jlee+013jZP9ca7xoiISApm3z4/a9Ys6PV6DB8+HGVlZQgLC4NSqcRrr72Gl19+uTFqJBvAZmkiIpKC2UFIJpPhzTffxMyZM3H27FncunUL3t7eaNWqFW7fvg1HR8fGqJOaOd4+T0REUqjXOkIAoFAo4O3tjQEDBsDBwQHLly9H165dLVkb2RAHNksTEZEE6hyEysvLkZiYiMDAQISGhmLr1q0AgPXr16Nr165477338MorrzRWndTMsUeIiIikUOdLY0lJSVizZg0iIiJw4MABPPXUU4iLi8PBgwexfPlyPPXUU7Czs2vMWqkZU/CuMSIikkCdg9CWLVvwySef4PHHH0deXh769u2LqqoqHD16FDKZrDFrJBtg2H2eQYiIiERU50tjFy9eREBAAADAx8cHSqUSr7zyCkMQWQQXVCQiIinUOQhVV1dDoVAYfre3t0erVq0apSiyPewRIiIiKdT50pggCJg4cSKUSiUA4I8//sBLL72Eli1bGo3717/+ZdkKySbU3DXG2+eJiEhMdQ5CsbGxRr8/++yzFi+GbJeCPUJERCSBOgeh9evXN2YdZON4aYyIiKRQ7wUVLWnVqlXw9PSESqVCcHAwsrKy7jt+y5Yt6NWrF1QqFXx9fbFjx45aY06cOIHHH38carUaLVu2RFBQEPLz8xvrI1AD1TRL8/Z5IiISk+RBKC0tDQkJCUhOTkZOTg78/PwQGRmJoqIik+MPHDiA8ePH4/nnn8eRI0cQFRWFqKgo5OXlGcacO3cOgwcPRq9evZCRkYGff/4Zc+fOhUqlEutjkZmUnBEiIiIJyARBEKQsIDg4GEFBQVi5ciUAQK/Xw93dHS+//DJmzZpVa3x0dDRKS0uxbds2w7GBAwfC398fq1evBgCMGzcODg4O+PTTT+tVk06ng1qtRklJCZydnet1DjLPheJSDFmagZYKO/wyf4TU5RARURNUn7/fks4IVVRUIDs7GxEREYZjcrkcERERyMzMNPmazMxMo/EAEBkZaRiv1+uxfft29OzZE5GRkWjfvj2Cg4MNW4KYUl5eDp1OZ/QgcbFZmoiIpCBpECouLkZ1dTVcXV2Njru6ukKr1Zp8jVarve/4oqIi3Lp1C4sXL8aIESPwww8/4IknnsCTTz6J3bt3mzxnSkoK1Gq14eHu7m6BT0fmuLvFhgC9XtJJSiIisiGS9whZml5/Z0ZhzJgxeOWVV+Dv749Zs2bhscceM1w6+1+JiYkoKSkxPAoKCsQsmXA3CAGcFSIiIvHU+fb5xuDi4gI7OzsUFhYaHS8sLISbm5vJ17i5ud13vIuLC+zt7eHt7W00pnfv3ti3b5/JcyqVSsNCkSSNmrvGgDt3jqkcuIEvERE1PklnhBQKBQICAqDRaAzH9Ho9NBoNQkJCTL4mJCTEaDwApKenG8YrFAoEBQXh1KlTRmNOnz6NLl26WPgTkKX8dxDinWNERCQWSWeEACAhIQGxsbEIDAzEgAEDkJqaitLSUsTFxQEAYmJi0KlTJ6SkpAAApk+fjvDwcCxbtgyjRo3C5s2bcfjwYXz00UeGc86cORPR0dEICwvD0KFDsXPnTnz33XfIyMiQ4iNSHcjlMtjLZajSC7w0RkREopE8CEVHR+Pq1atISkqCVquFv78/du7caWiIzs/Ph1x+d7YgNDQUmzZtwpw5czB79mx4eXlh69at8PHxMYx54oknsHr1aqSkpGDatGl4+OGH8dVXX2Hw4MGifz6qO4W9HFUV1ZwRIiIi0Ui+jpA14jpC0vCf/wNulFUi/ZUweLk6SV0OERE1MU1uHSGi/6bgDvRERCQyBiGyGnfXEmIQIiIicTAIkdXgDvRERCQ2BiGyGjWXxnjXGBERiYVBiKwGZ4SIiEhsDEJkNQwzQgxCREQkEgYhshrcgZ6IiMTGIERWg5fGiIhIbAxCZDUc2CxNREQiYxAiq8EZISIiEhuDEFkNJZuliYhIZAxCZDU4I0RERGJjECKrwbvGiIhIbAxCZDW4sjQREYmNQYishgMvjRERkcgYhMhqcGVpIiISG4MQWQ02SxMRkdgYhMhqKNksTUREImMQIqvBGSEiIhIbgxBZjZoeoUrOCBERkUgYhMhq1Ow1Vs4ZISIiEgmDEFkNXhojIiKxMQiR1eDK0kREJDYGIbIanBEiIiKxMQiR1eDu80REJDYGIbIaNVts8K4xIiISC4MQWQ1usUFERGJjECKrwWZpIiISG4MQWY2aIMR1hIiISCwMQmQ1eGmMiIjExiBEVuO/N10VBEHiaoiIyBZYRRBatWoVPD09oVKpEBwcjKysrPuO37JlC3r16gWVSgVfX1/s2LHD6PmJEydCJpMZPUaMGNGYH4EsoGaLDUEAqvUMQkRE1PgkD0JpaWlISEhAcnIycnJy4Ofnh8jISBQVFZkcf+DAAYwfPx7PP/88jhw5gqioKERFRSEvL89o3IgRI3DlyhXD44svvhDj41AD1PQIAWyYJiIicUgehJYvX47JkycjLi4O3t7eWL16NVq0aIGPP/7Y5PgVK1ZgxIgRmDlzJnr37o0FCxagf//+WLlypdE4pVIJNzc3w+Ohhx4S4+NQAxgFIfYJERGRCCQNQhUVFcjOzkZERIThmFwuR0REBDIzM02+JjMz02g8AERGRtYan5GRgfbt2+Phhx/GlClTcO3atXvWUV5eDp1OZ/Qg8dnLZZDJ7vzMIERERGKQNAgVFxejuroarq6uRsddXV2h1WpNvkar1T5w/IgRI/DJJ59Ao9FgyZIl2L17N0aOHInq6mqT50xJSYFarTY83N3dG/jJqD5kMpnhzjHeQk9ERGKwl7qAxjBu3DjDz76+vujbty+6d++OjIwMDB8+vNb4xMREJCQkGH7X6XQMQxJR2MtRXqVnjxAREYlC0hkhFxcX2NnZobCw0Oh4YWEh3NzcTL7Gzc3NrPEA0K1bN7i4uODs2bMmn1cqlXB2djZ6kDRqZoS43xgREYlB0iCkUCgQEBAAjUZjOKbX66HRaBASEmLyNSEhIUbjASA9Pf2e4wHg4sWLuHbtGjp06GCZwqnRGLbZ4KUxIiISgeR3jSUkJGDt2rXYuHEjTpw4gSlTpqC0tBRxcXEAgJiYGCQmJhrGT58+HTt37sSyZctw8uRJvPXWWzh8+DDi4+MBALdu3cLMmTNx8OBBXLhwARqNBmPGjEGPHj0QGRkpyWekumMQIiIiMUneIxQdHY2rV68iKSkJWq0W/v7+2Llzp6EhOj8/H3L53bwWGhqKTZs2Yc6cOZg9eza8vLywdetW+Pj4AADs7Ozw888/Y+PGjbhx4wY6duyIRx99FAsWLIBSqZTkM1LdcZsNIiISk0zgXga16HQ6qNVqlJSUsF9IZKPe34tfLuswf0wfBHTh2k9EZJpn25ZoqZT8v+XJytTn7zf/KSKrUrPfWNI3v0hcCRFZs06tHbHn9aGwk8ukLoWaOAYhsipPB7qjUFeOKj0vjRGRaYW6cly6cRs3yirQthVbHqhhGITIqowb4IFxAzykLoOIrFjft/4N3R9VuHG7kkGIGkzyu8aIiIjM0bqFAgBwo6xS4kqoOWAQIiKiJqV1CwcAQMntCokroeaAQYiIiJoUteOdIMQZIbIEBiEiImpSGITIkhiEiIioSbl7aYxBiBqOQYiIiJqU1o53mqUZhMgSGISIiKhJqZkRulHGZmlqOAYhIiJqUgw9QpwRIgtgECIioiaFzdJkSQxCRETUpNQsqKjjjBBZAIMQERE1KYYeIQYhsgAGISIialJaO95tltbrBYmroaaOQYiIiJoU5z+DkF4AblVUSVwNNXUMQkRE1KSoHOygcrjz56uEDdPUQAxCRETU5NQsqsg7x6ihGISIiKjJ4TYbZCkMQkRE1OTcXVSRq0tTwzAIERFRk8NFFclSGISIiKjJ4aUxshQGISIianJqVpfmxqvUUAxCRETU5NRcGuOMEDUUgxARETU5hm022CNEDcQgRERETc7du8YYhKhhGISIiKjJqVlQkStLU0MxCBERUZNzdwd6NktTwzAIERFRk8N1hMhSGISIiKjJqZkRKq/S44/KaomroaaMQYiIiJqcVkp72MllAHgLPTUMgxARETU5MpmMl8fIIqwiCK1atQqenp5QqVQIDg5GVlbWfcdv2bIFvXr1gkqlgq+vL3bs2HHPsS+99BJkMhlSU1MtXDUREUmptSEIsWGa6k/yIJSWloaEhAQkJycjJycHfn5+iIyMRFFRkcnxBw4cwPjx4/H888/jyJEjiIqKQlRUFPLy8mqN/frrr3Hw4EF07NixsT8GERGJTN2CawlRw0kehJYvX47JkycjLi4O3t7eWL16NVq0aIGPP/7Y5PgVK1ZgxIgRmDlzJnr37o0FCxagf//+WLlypdG4S5cu4eWXX8bnn38OBweH+9ZQXl4OnU5n9CAiIutWMyPEtYSoISQNQhUVFcjOzkZERIThmFwuR0REBDIzM02+JjMz02g8AERGRhqN1+v1eO655zBz5kz06dPngXWkpKRArVYbHu7u7vX8REREJBbuN0aWIGkQKi4uRnV1NVxdXY2Ou7q6QqvVmnyNVqt94PglS5bA3t4e06ZNq1MdiYmJKCkpMTwKCgrM/CRERCQ2ww70XFSRGsBe6gIsLTs7GytWrEBOTg5kMlmdXqNUKqFUKhu5MiIisiTeNUaWIGkQcnFxgZ2dHQoLC42OFxYWws3NzeRr3Nzc7jt+7969KCoqgoeHh+H56upqvPrqq0hNTcWFCxcs+yGIiEgSNYsqakv+wMXrZRJXQ+ZydLBD21bST0JIGoQUCgUCAgKg0WgQFRUF4E5/j0ajQXx8vMnXhISEQKPRYMaMGYZj6enpCAkJAQA899xzJnuInnvuOcTFxTXK5yAiIvHVBCHNySJoTpq+05is1+N+HfH++H5SlyH9pbGEhATExsYiMDAQAwYMQGpqKkpLSw2hJSYmBp06dUJKSgoAYPr06QgPD8eyZcswatQobN68GYcPH8ZHH30EAGjbti3atm1r9B4ODg5wc3PDww8/LO6HIyKiRjOwW1t4tGmBQt0fUpdC9WBvV7f2lcYmeRCKjo7G1atXkZSUBK1WC39/f+zcudPQEJ2fnw+5/G5Pd2hoKDZt2oQ5c+Zg9uzZ8PLywtatW+Hj4yPVRyAiIgl0UDtiz+tDpS6DmjiZIAiC1EVYG51OB7VajZKSEjg7O0tdDhEREdVBff5+S76gIhEREZFUGISIiIjIZjEIERERkc1iECIiIiKbxSBERERENotBiIiIiGwWgxARERHZLAYhIiIislkMQkRERGSzGISIiIjIZjEIERERkc1iECIiIiKbxSBERERENste6gKskSAIAO7sYktERERNQ83f7Zq/43XBIGTCzZs3AQDu7u4SV0JERETmunnzJtRqdZ3GygRzYpON0Ov1uHz5MpycnCCTySx2Xp1OB3d3dxQUFMDZ2dli56V743cuDX7v4uN3Lg1+7+K733cuCAJu3ryJjh07Qi6vW/cPZ4RMkMvl6Ny5c6Od39nZmf+HERm/c2nwexcfv3Np8HsX372+87rOBNVgszQRERHZLAYhIiIislkMQiJSKpVITk6GUqmUuhSbwe9cGvzexcfvXBr83sVn6e+czdJERERkszgjRERERDaLQYiIiIhsFoMQERER2SwGISIiIrJZDEIiWbVqFTw9PaFSqRAcHIysrCypS2rWUlJSEBQUBCcnJ7Rv3x5RUVE4deqU1GXZlMWLF0Mmk2HGjBlSl9LsXbp0Cc8++yzatm0LR0dH+Pr64vDhw1KX1WxVV1dj7ty56Nq1KxwdHdG9e3csWLDArP2t6MH27NmD0aNHo2PHjpDJZNi6davR84IgICkpCR06dICjoyMiIiJw5swZs9+HQUgEaWlpSEhIQHJyMnJycuDn54fIyEgUFRVJXVqztXv3bkydOhUHDx5Eeno6Kisr8eijj6K0tFTq0mzCoUOHsGbNGvTt21fqUpq969evY9CgQXBwcMD333+P48ePY9myZXjooYekLq3ZWrJkCT788EOsXLkSJ06cwJIlS/DOO+/ggw8+kLq0ZqW0tBR+fn5YtWqVyeffeecdvP/++1i9ejV++ukntGzZEpGRkfjjjz/MeyOBGt2AAQOEqVOnGn6vrq4WOnbsKKSkpEhYlW0pKioSAAi7d++WupRm7+bNm4KXl5eQnp4uhIeHC9OnT5e6pGbtjTfeEAYPHix1GTZl1KhRwqRJk4yOPfnkk8KECRMkqqj5AyB8/fXXht/1er3g5uYmvPvuu4ZjN27cEJRKpfDFF1+YdW7OCDWyiooKZGdnIyIiwnBMLpcjIiICmZmZElZmW0pKSgAAbdq0kbiS5m/q1KkYNWqU0T/z1Hi+/fZbBAYG4qmnnkL79u3Rr18/rF27VuqymrXQ0FBoNBqcPn0aAHD06FHs27cPI0eOlLgy23H+/HlotVqjf8+o1WoEBweb/beVm642suLiYlRXV8PV1dXouKurK06ePClRVbZFr9djxowZGDRoEHx8fKQup1nbvHkzcnJycOjQIalLsRm//vorPvzwQyQkJGD27Nk4dOgQpk2bBoVCgdjYWKnLa5ZmzZoFnU6HXr16wc7ODtXV1Vi4cCEmTJggdWk2Q6vVAoDJv601z9UVgxA1e1OnTkVeXh727dsndSnNWkFBAaZPn4709HSoVCqpy7EZer0egYGBWLRoEQCgX79+yMvLw+rVqxmEGsmXX36Jzz//HJs2bUKfPn2Qm5uLGTNmoGPHjvzOmyBeGmtkLi4usLOzQ2FhodHxwsJCuLm5SVSV7YiPj8e2bduwa9cudO7cWepymrXs7GwUFRWhf//+sLe3h729PXbv3o33338f9vb2qK6ulrrEZqlDhw7w9vY2Ota7d2/k5+dLVFHzN3PmTMyaNQvjxo2Dr68vnnvuObzyyitISUmRujSbUfP30xJ/WxmEGplCoUBAQAA0Go3hmF6vh0ajQUhIiISVNW+CICA+Ph5ff/01fvzxR3Tt2lXqkpq94cOH49ixY8jNzTU8AgMDMWHCBOTm5sLOzk7qEpulQYMG1Voa4vTp0+jSpYtEFTV/ZWVlkMuN/3za2dlBr9dLVJHt6dq1K9zc3Iz+tup0Ovz0009m/23lpTERJCQkIDY2FoGBgRgwYABSU1NRWlqKuLg4qUtrtqZOnYpNmzbhm2++gZOTk+GasVqthqOjo8TVNU9OTk61erBatmyJtm3bsjerEb3yyisIDQ3FokWL8PTTTyMrKwsfffQRPvroI6lLa7ZGjx6NhQsXwsPDA3369MGRI0ewfPlyTJo0SerSmpVbt27h7Nmzht/Pnz+P3NxctGnTBh4eHpgxYwbefvtteHl5oWvXrpg7dy46duyIqKgo897IQne20QN88MEHgoeHh6BQKIQBAwYIBw8elLqkZg2Aycf69eulLs2m8PZ5cXz33XeCj4+PoFQqhV69egkfffSR1CU1azqdTpg+fbrg4eEhqFQqoVu3bsKbb74plJeXS11as7Jr1y6T/x6PjY0VBOHOLfRz584VXF1dBaVSKQwfPlw4deqU2e8jEwQuhUlERES2iT1CREREZLMYhIiIiMhmMQgRERGRzWIQIiIiIpvFIEREREQ2i0GIiIiIbBaDEBEREdksBiEiIiKyWQxCRDYiIyMDMpkMN27cAABs2LABrVu3rvPrPT09kZqaarF6LHU+S9dVVxMnTjR/Kf9m4MKFC5DJZMjNzZW6FCKLYBAisjKrV6+Gk5MTqqqqDMdu3boFBwcHDBkyxGhsTbg5d+6cyFWK717B7dChQ3jhhRdEr2fFihXYsGGD6O9LRJbFIERkZYYOHYpbt27h8OHDhmN79+6Fm5sbfvrpJ/zxxx+G47t27YKHhwe6d+8uRalWoV27dmjRooXo76tWq82aUSMi68QgRGRlHn74YXTo0AEZGRmGYxkZGRgzZgy6du2KgwcPGh0fOnQoAODTTz9FYGAgnJyc4ObmhmeeeQZFRUVmvfd3332HoKAgqFQquLi44Iknnrjn2Pz8fIwZMwatWrWCs7Mznn76aRQWFtb7fP/4xz/QunVraDSaWs9lZGQgLi4OJSUlkMlkkMlkeOuttwDUvjQmk8mwZs0aPPbYY2jRogV69+6NzMxMnD17FkOGDEHLli0RGhpaaxbtm2++Qf/+/aFSqdCtWzfMmzfPaFbuf/3vpbEhQ4Zg2rRpeP3119GmTRu4ubkZaryXjIwMDBgwAC1btkTr1q0xaNAg/Pbbb3Wu6caNG3jxxRfh6uoKlUoFHx8fbNu2zfD8V199hT59+kCpVMLT0xPLli0zen9PT08sWrQIkyZNgpOTEzw8PGrtWp+VlYV+/fpBpVIhMDAQR44cMXr++vXrmDBhAtq1awdHR0d4eXlh/fr19/3cRNaEQYjICg0dOhS7du0y/L5r1y4MGTIE4eHhhuO3b9/GTz/9ZAhClZWVWLBgAY4ePYqtW7fiwoULmDhxYp3fc/v27XjiiSfw17/+FUeOHIFGo8GAAQNMjtXr9RgzZgx+//137N69G+np6fj1118RHR1dr/O98847mDVrFn744QcMHz681vOhoaFITU2Fs7Mzrly5gitXruC1116752dZsGABYmJikJubi169euGZZ57Biy++iMTERBw+fBiCICA+Pt4wfu/evYiJicH06dNx/PhxrFmzBhs2bMDChQvr+vUBADZu3IiWLVvip59+wjvvvIP58+cjPT3d5NiqqipERUUhPDwcP//8MzIzM/HCCy9AJpPVqSa9Xo+RI0di//79+Oyzz3D8+HEsXrwYdnZ2AIDs7Gw8/fTTGDduHI4dO4a33noLc+fOrXU5b9myZYaA8/e//x1TpkzBqVOnANy5JPvYY4/B29sb2dnZeOutt2p973PnzsXx48fx/fff48SJE/jwww/h4uJi1vdGJCmz96snoka3du1aoWXLlkJlZaWg0+kEe3t7oaioSNi0aZMQFhYmCIIgaDQaAYDw22+/mTzHoUOHBADCzZs3BUEQhF27dgkAhOvXrwuCIAjr168X1Gq1YXxISIgwYcKEe9bUpUsX4b333hMEQRB++OEHwc7OTsjPzzc8/8svvwgAhKysLLPO9/rrrwsdOnQQ8vLy7vud/G+9puoSBEEAIMyZM8fwe2ZmpgBAWLduneHYF198IahUKsPvw4cPFxYtWmR03k8//VTo0KHDPeuJjY0VxowZY/g9PDxcGDx4sNGYoKAg4Y033jD5+mvXrgkAhIyMDJPPP6imf//734JcLhdOnTpl8vXPPPOM8Je//MXo2MyZMwVvb2/D7126dBGeffZZw+96vV5o37698OGHHwqCIAhr1qwR2rZtK9y+fdsw5sMPPxQACEeOHBEEQRBGjx4txMXFmayBqCngjBCRFRoyZAhKS0tx6NAh7N27Fz179kS7du0QHh5u6BPKyMhAt27d4OHhAeDODMDo0aPh4eEBJycnhIeHA7hzCasucnNzTc7GmHLixAm4u7vD3d3dcMzb2xutW7fGiRMn6ny+ZcuWYe3atdi3bx/69OlTp/eui759+xp+dnV1BQD4+voaHfvjjz+g0+kAAEePHsX8+fPRqlUrw2Py5Mm4cuUKysrK6vW+ANChQ4d7Xp5s06YNJk6ciMjISIwePRorVqzAlStXDM8/qKbc3Fx07twZPXv2NHn+EydOYNCgQUbHBg0ahDNnzqC6utpkzTKZDG5uboaaT5w4gb59+0KlUhnGhISEGJ1zypQp2Lx5M/z9/fH666/jwIED9/uKiKwOgxCRFerRowc6d+6MXbt2YdeuXYZQ07FjR7i7u+PAgQPYtWsXhg0bBgAoLS1FZGQknJ2d8fnnn+PQoUP4+uuvAQAVFRV1ek9HR0eLfoa6nO+RRx5BdXU1vvzyS4u+t4ODg+HnmktNpo7p9XoAdy4BzZs3D7m5uYbHsWPHcObMGaMQYM771rxPzXuYsn79emRmZiI0NBRpaWno2bOnoQfsQTVZ6n8vc2v+XyNHjsRvv/2GV155BZcvX8bw4cPve9mSyNowCBFZqaFDhyIjIwMZGRlGt82HhYXh+++/R1ZWlqE/6OTJk7h27RoWL16MRx55BL169TK7Ubpv374mG5VN6d27NwoKClBQUGA4dvz4cdy4cQPe3t51Pt+AAQPw/fffY9GiRVi6dOl9xyoUCqOZDEvq378/Tp06hR49etR6yOWN+6/Jfv36ITExEQcOHICPjw82bdpUp5r69u2Lixcv4vTp0ybP27t3b+zfv9/o2P79+9GzZ09DH9GD9O7dGz///LPRnYr/3axfo127doiNjcVnn32G1NTUWg3XRNbMXuoCiMi0oUOHYurUqaisrDTMCAFAeHg44uPjUVFRYQhCHh4eUCgU+OCDD/DSSy8hLy8PCxYsMOv9kpOTMXz4cHTv3h3jxo1DVVUVduzYgTfeeKPW2IiICPj6+mLChAlITU1FVVUV/v73vyM8PByBgYFmnS80NBQ7duzAyJEjYW9vjxkzZpisz9PTE7du3YJGo4Gfnx9atGhhsdvmk5KS8Nhjj8HDwwP/7//9P8jlchw9ehR5eXl4++23LfIe/+v8+fP46KOP8Pjjj6Njx444deoUzpw5g5iYmDrVFB4ejrCwMIwdOxbLly9Hjx49cPLkSchkMowYMQKvvvoqgoKCsGDBAkRHRyMzMxMrV67E//3f/9W5xmeeeQZvvvkmJk+ejMTERFy4cKFWYE1KSkJAQAD69OmD8vJybNu2Db1797bod0XUmDgjRGSlhg4ditu3b6NHjx6GPhfgThC6efOm4TZ74M5/kW/YsAFbtmyBt7c3Fi9e/MAZlv81ZMgQbNmyBd9++y38/f0xbNgwZGVlmRwrk8nwzTff4KGHHkJYWBgiIiLQrVs3pKWl1et8gwcPxvbt2zFnzhx88MEHJseEhobipZdeQnR0NNq1a4d33nnHrM93P5GRkdi2bRt++OEHBAUFYeDAgXjvvffQpUsXi73H/2rRogVOnjyJsWPHomfPnnjhhRcwdepUvPjii3Wu6auvvkJQUBDGjx8Pb29vvP7664ZZs/79++PLL7/E5s2b4ePjg6SkJMyfP9+sOwlbtWqF7777DseOHUO/fv3w5ptvYsmSJUZjFAoFEhMT0bdvX4SFhcHOzg6bN29u+BdEJBKZIAiC1EUQERERSYEzQkRERGSzGISIiIjIZjEIERERkc1iECIiIiKbxSBERERENotBiIiIiGwWgxARERHZLAYhIiIislkMQkRERGSzGISIiIjIZjEIERERkc36/2BmNJKCJBmEAAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhk0lEQVR4nO3de5ycZX338c93ZzOb3U3YYBIQs0DABGggQSEgchJBFB4rqAUlqBy0Yn1AsEoVqaXI82qrPi0oh9oioghVVASNNpXGgKCikHDKAQRSTkkIJAFCQhKS7O6vf9z3JsOwh5mQe+7Zne/79corc59mfhPIfnNd131flyICMzOzSjXlXYCZmQ0tDg4zM6uKg8PMzKri4DAzs6o4OMzMrCrNeRewvYwbNy4mTpyYdxlmZkPKvffeuyoixldzzbAJjokTJzJv3ry8yzAzG1IkPVXtNe6qMjOzqjg4zMysKpkGh6TjJD0iabGkC/o4fqSk+yR1STqpZP9bJP1B0iJJ8yV9OMs6zcyscpkFh6QCcBVwPDAFmCFpStlpTwNnAD8o278eOC0i9gWOA74haUxWtZqZWeWyHBw/GFgcEY8DSLoROBF4qPeEiHgyPdZTemFEPFry+hlJK4DxwOoM6zUzswpk2VU1AVhSsr003VcVSQcDReB/+jh2lqR5kuatXLlymws1M7PK1fXguKRdgOuBMyOip/x4RFwdEdMjYvr48VXdhmxmZtsoy66qZcCuJdud6b6KSNoB+E/gbyPij9u5ti3Wb+ri337zmsYMI4sFzjh0Im3FYfOoi5nZdpHlT8W5wGRJe5AExinAqZVcKKkI3AJ8PyJuyq5E2LCpmytuX/yqfb1LlEzeaTTHTtk5y483MxtyMguOiOiSdA5wK1AAro2IRZIuAeZFxExJB5EExI7A+yR9Jb2T6kPAkcBYSWekb3lGRDywvescO6qFJ/7pva/a98Sqdbzzn3/Dyxs3b++PMzMb8jLth4mIWcCssn0XlbyeS9KFVX7dDcANWdY2kPZiAYCXN3bnVYKZWd2q68HxvLS3JHm6fmNXzpWYmdUfB0cfWkckLY51m9ziMDMr5+DoQ1OTaCsW3OIwM+uDg6MfbcVmtzjMzPrg4OjHqJYC69ziMDN7DQdHP9qKzazf5OAwMyvn4OhHe0uBdb4d18zsNRwc/XCLw8ysbw6OfrS3FDw4bmbWBwdHP9qLzR4cNzPrg4OjH+0tDg4zs744OPrRViywflM30TtVrpmZAQ6OfrW3NNPVE2zqfs36UWZmDc3B0Y+2dIZc35JrZvZqDo5+9M6Q63EOM7NXc3D0oz1dMna9b8k1M3sVB0c/2lp6p1Z3i8PMrJSDox9bWhwe4zAzexUHRz/aW3qXj3WLw8yslIOjH1vHOBwcZmalHBz92DrG4a4qM7NSDo5+bB3jcIvDzKyUg6MfrSPc4jAz64uDox9NTaK96OVjzczKOTgG0NbixZzMzMo5OAaQtDjcVWVmVsrBMQAvH2tm9loOjgG0t7jFYWZWzsExgPaWZs9VZWZWxsExAK87bmb2Wg6OAfQuH2tmZls5OAbQ3uIWh5lZOQfHANpbCqzb1E1E5F2KmVndcHAMoK3YTHdPsLGrJ+9SzMzqhoNjAO3FZL4qj3OYmW3l4BhAW0syQ67HOczMtso0OCQdJ+kRSYslXdDH8SMl3SepS9JJZcdOl/RY+uv0LOvsz9bFnNziMDPrlVlwSCoAVwHHA1OAGZKmlJ32NHAG8IOya98A/D3wNuBg4O8l7ZhVrf3x8rFmZq+VZYvjYGBxRDweEZuAG4ETS0+IiCcjYj5QPvr8HmB2RLwQES8Cs4HjMqy1T+0tXj7WzKxclsExAVhSsr003bfdrpV0lqR5kuatXLlymwvtT1s6OO75qszMthrSg+MRcXVETI+I6ePHj9/u7791jMMtDjOzXlkGxzJg15LtznRf1tduN20tXj7WzKxclsExF5gsaQ9JReAUYGaF194KvFvSjumg+LvTfTU1yrfjmpm9RmbBERFdwDkkP/AfBn4cEYskXSLpBABJB0laCpwM/LukRem1LwD/jyR85gKXpPtqamRzAQnWOzjMzLZozvLNI2IWMKts30Ulr+eSdEP1de21wLVZ1jeYpibRNqLgriozsxIVBUfaXfQmYAPwZEQ0zORNbS1ePtbMrFS/wSGpAzgbmAEUgZXASGBnSX8E/jUibq9JlTka1dLs23HNzEoM1OK4Cfg+cERErC49IOlA4GOS9oyI72RYX+7aigUPjpuZleg3OCLi2AGO3Qvcm0lFdaa96HXHzcxKDXpXlaSbJb1X0pB+WHBbtbV4+Vgzs1KVhMG/AqcCj0n6qqS9M66prrQXvXysmVmpQYMjIn4dER8BDgCeBH4t6S5JZ0oakXWBeWtvKXhw3MysREXdT5LGkkx//pfA/cA3SYJkdmaV1Yk2j3GYmb3KoM9xSLoF2Bu4HnhfRCxPD/1I0rwsi6sH7ekYR0QgKe9yzMxyV8kDgJf397xGREzfzvXUnbZiM909wcauHkaOKORdjplZ7irpqpoiaUzvRjrx4P/NrqT60p6uyeE7q8zMEpUExydLHwBMV+T7ZGYV1Zl2z5BrZvYqlQRHQSWd++la4sXsSqovW4LDA+RmZkBlYxy/IhkI//d0+1Ppvobg5WPNzF6tkuD4IklYfDrdng1ck1lFdaa3xeEZcs3MEoMGRzqF+rfSXw2nd91xtzjMzBKVPMcxGfgnYArJtOoARMSeGdZVN9p71x334LiZGVDZ4Ph3SVobXcA7SaZavyHLoupJW9FdVWZmpSoJjtaImAMoIp6KiIuB92ZbVv3Y0uLwcxxmZkBlg+Mb0ynVH5N0DrAMGJVtWfWjdUQBCda7q8rMDKisxXEe0AacCxwIfBQ4Pcui6omkdDEntzjMzGCQFkf6sN+HI+J84GXgzJpUVWe8fKyZ2VYDtjgiohs4vEa11K32Frc4zMx6VTLGcb+kmcBPgHW9OyPi5syqqjNtxYLHOMzMUpUEx0jgeeDokn0BNExwtHsxJzOzLSp5crwhxzVKtbcUWPXyprzLMDOrC5U8Of5dkhbGq0TExzOpqA61tTSz7oX1eZdhZlYXKumq+mXJ65HAB4BnsimnPrUXC6z3XFVmZkBlXVU/Ld2W9EPgd5lVVIfaPMZhZrZFJQ8AlpsM7LS9C6lno1qaWb+pm4jX9NiZmTWcSsY41vLqMY5nSdboaBhtLQW6e4KNXT2MHFHIuxwzs1xV0lU1uhaF1LPeNTkuvGUBLc1JcHS0juCvj528ZdvMrFFU0uL4AHBbRLyUbo8BjoqIn2VbWv2Y2tnBhDGt/PaxVQBEBKte3kTnjq189JDdc67OzKy2NFi/vaQHIuItZfvuj4i3ZllYtaZPnx7z5s2ryWdFBH/xrbt4bs1Gbj//KIrN2zJUZGaWP0n3RsT0aq6p5CdeX+dUchvvsCWJc4+ZzLLVG/jpfUvzLsfMrKYqCY55ki6V9Ob016XAvVkXVu/esdd49u/s4KrbF7O5uyfvcszMaqaS4PgMsAn4EXAj8ApwdpZFDQWSOO9dk1n64gZuuX9Z3uWYmdXMoMEREesi4oKImB4RB0XEhRGxbrDrACQdJ+kRSYslXdDH8RZJP0qP3y1pYrp/hKTrJC2Q9LCkL1X9zWrgnXvvxH4TduCq2xfT5VaHmTWIQYND0uz0Tqre7R0l3VrBdQXgKuB4YAowQ9KUstM+AbwYEZOAy4CvpftPBloiYirJqoOf6g2VeiKJc4+ezFPPr2fmgw01C4uZNbBKuqrGRcTq3o2IeJHKnhw/GFgcEY9HxCaSbq4Ty845EbgufX0TcIwkkTxw2C6pGWgl6SpbU8Fn1tyxU3bmz3bZgStvW0x3j58sN7Phr5Lg6JG0W++GpN3pY7bcPkwAlpRsL0339XlORHQBLwFjSUJkHbAceBr454h4ofwDJJ0laZ6keStXrqygpO0vaXVM4vFV6/jlfLc6zGz4qyQ4/hb4naTrJd0A3AlkPeZwMNANvAnYA/i8pD3LT4qIq9Oxl+njx4/PuKT+vWffN7L3zqO5wq0OM2sAlQyO/wo4gK13VR0YEYOOcQDLgF1LtjvTfX2ek3ZLdZCsNngq8KuI2BwRK4DfA1U9oFJLTU3iM8dMYvGKl/mvhcvzLsfMLFOVPvLcDawgGWeYIunICq6ZC0yWtIekInAKMLPsnJnA6enrk0imNgmS7qmjASS1A4cAf6qw1lwcv98uTNppFFfMWUyPWx1mNoxVclfVX5J0T90KfCX9/eLBrkvHLM5Jz38Y+HFELJJ0iaQT0tO+A4yVtBj4HNB7y+5VwChJi0gC6LsRMb+aL1ZrhSbxmaMn8chza7l10bN5l2NmlplK5qpaABwE/DEi3iJpH+AfI+KDtSiwUrWcq6o/3T3BsZfeQbG5iVnnHkFTk3Ktx8xsMFnNVfVKRLySfkBLRPwJ2HtbChzuCk3inKMn8adn1/Lrh5/Luxwzs0xUEhxL0wcAfwbMlvRz4KksixrKTtj/Tew+to3Lb3vMKwaa2bBUyV1VH4iI1RFxMfB3JOMS78+4riGrudDE2e+cxMJla7j9kRV5l2Nmtt1VtZBERNwRETPTJ8GtHx946wR2fUMr35yz2K0OMxt2vAJRBkYUmjj7qEk8uGQ1dzyazxPtZmZZcXBk5IMHdDJhTCvfnOOxDjMbXqpayU/Sn0fEL7MqZjgpNjfx6aPezJd/tpDzfzKf0SPzXTRx5IgCZx42kZ13GJlrHWY29FX70+wSwMFRoZOnd/KTeUuY/VD+DwSu39TNLx58hus+fhCTdhqddzlmNoRVGxx+oq0KLc0Ffn7O4XmXAcDCZS9xxnfv4aR/+wPfOX06B+7+hrxLMrMhqtoxjk9lUoVlbr8JHdz86cMY0zqCU799N7Mf8gOKZrZtqr0d956sCrHs7Ta2jZ9++lD2eeNoPnX9PH5w99N5l2RmQ5DvqmowY0e18MOzDuEde43nwlsWcNnsR33Xl5lVxcHRgNqKzVx92nROPrCTb855jAtvWUBXd0/eZZnZELFN94hK2ied7NCGqBGFJr5+0jR23mEkV96+mJVrN3LFjANoLRbyLs3M6ty2Plzw38Bug55ldU0S579nb3buGMlFP1/Iqdf8kX94/1SKzZU1RCeMaXXQmDWgfoND0uX9HQLGZFKN5eJjh+zO+FEtnHvj/fyfy39b8XXv2Gs813384AwrM7N6NFCL40zg88DGPo7NyKYcy8tx+72RWecewUPL11R0/rW/e4JnX3ol46rMrB4NFBxzgYURcVf5AUkXZ1aR5WbSTqOYtNOois79zZ9WcM+TL2RckZnVo4GC4ySgz39SRsQe2ZRjQ0VrscCGTd15l2FmOeg3OCLC/5y0frUVC6x3cJg1pH5vn5H0C0nvkzSij2N7SrpE0sezLc/qVVuxmQ2bu+np8cODZo1moK6qTwKfA74h6QVgJTASmAj8D3BlRPw88wqtLrWlt+G+0tVNWzHfKePNrLYG6qp6FvgC8AVJE4FdgA3AoxGxvjblWb3qDY51Gx0cZo2mor/xEfEk8GSmldiQ0pqGhQfIzRqP56qybdLb4li/uSvnSsys1hwctk22BIdbHGYNZ9DgkHReJfussfSOa6zf6OAwazSVtDhO72PfGdu5DhtitrY43FVl1mgGmuRwBnAqsIekmSWHRgN+OLDB9c6Ku2GzWxxmjWagu6ruApYD44B/Kdm/FpifZVFW/zzGYda4BnqO4yngKeDtknYHJkfEryW1Aq0kAWINassYh4PDrOFUMjj+SeAm4N/TXZ3AzzKsyYaALS2OjR7jMGs0lQyOnw0cBqwBiIjHgJ2yLMrq34hCEyMKYr3HOMwaTiXBsTEiNvVuSGoGPLOd0TrCU6ubNaJKguMOSRcCrZKOBX4C/CLbsmwoaCs2+3ZcswZUSXB8kWRm3AXAp4BZwJezLMqGhraWAuvc4jBrOANOciipACyKiH2Ab1f75pKOA74JFIBrIuKrZcdbgO8DBwLPAx9OJ1RE0jSSAfkdgB7goIjwItd1pM2rAJo1pAFbHBHRDTwiabdq3zgNnauA44EpwAxJU8pO+wTwYkRMAi4DvpZe2wzcAPxVROwLHAVsrrYGy1bbCHdVmTWiSqZV3xFYJOkeYF3vzog4YZDrDgYWR8TjAJJuBE4EHio550Tg4vT1TcCVkgS8G5gfEQ+mn/V8BXVajbUWC6xev2nwE81sWKkkOP5uG997ArCkZHsp8Lb+zomILkkvAWOBvYCQdCswHrgxIr5e/gGSzgLOAthtt6obRfY6tRULPLPaXVVmjWbQ4IiIO2pRSJlm4HDgIGA9MEfSvRExp6y2q4GrAaZPn+5bhGssuavKwWHWaCp5cnytpDVlv5ZIukXSngNcugzYtWS7M93X5znpuEYHySD5UuDOiFiVLlM7Czig8q9ltdBWLHiMw6wBVXI77jeAvyHpVuoEzgd+ANwIXDvAdXOByZL2kFQETgFmlp0zk63Ttp8E3BYRAdwKTJXUlgbKO3j12IjVgSQ43OIwazSVjHGcEBH7l2xfLemBiPhi+mBgn9Ixi3NIQqAAXBsRiyRdAsyLiJnAd4DrJS0mmar9lPTaFyVdShI+AcyKiP/cpm9omWktFtjY1UN3T1BoUt7lmFmNVBIc6yV9iOSuJ0haBr3PUww4rhARs0i6mUr3XVTy+hXg5H6uvYHkllyrU20la3KMaqnkfyUzGw4q6ar6CPAxYAXwXPr6o+n06udkWJvVua3Lx3qcw6yRVHJX1ePA+/o5/LvtW44NJV7MyawxVXJX1V6S5khamG5Pk+S5qszBYdagKumq+jbwJdIpPyJiPukgtjW21rSrasNmd1WZNZJKgqMtIu4p2+efFLalxbFuo1scZo2kkuBYJenNpHdQSToJWJ5pVTYktI5wV5VZI6rkHsqzSab12EfSMuAJkjutrMG1t7iryqwRVXpX1bsktZO0UNaTjHE8lXFtVuc8OG7WmPrtqpK0g6QvSboyXTJ2Pcn0IIuBD9WqQKtfrb0PADo4zBrKQC2O64EXgT8AnwT+FhDwgYh4IPvSrN61jfDguFkjGig49oyIqQCSriEZEN/Ny7dar+ZCE8VCE+s9xmHWUAa6q2rLUq3pErJLHRpWrq3F646bNZqBWhz7S1qTvhbQmm4LiIjYIfPqrO61jfDU6maNpt/giIhCLQuxoam16BaHWaOp5AFAs361FZtZ51UAzRqKg8Nel1avAmjWcBwc9rq0u6vKrOE4OOx1aSs2s95dVWYNxcFhr4u7qswaj4PDXpc2B4dZw6lkdlyzfrUWC6x9ZTOf/P68vEtpSIdPGsfph07MuwxrMA4Oe12OnDye3y9exdIXN+RdSsNZufYV7nniBU57++5IyrscayAODntdDps0jl9+5oi8y2hIP7j7aS68ZQFLXtjAbmPb8i7HGojHOMyGqGmdHQDMX7Y630Ks4Tg4zIaovXYeTbHQxIKlL+VdijUYB4fZEFVsbuLPdhnNfAeH1ZiDw2wIm9Y5hoXLXqKnJ/IuxRqIg8NsCJva2cHajV088fy6vEuxBuLgMBvCegfIPc5hteTgMBvCJo0fxcgRTR7nsJpycJgNYc2FJvZ9UwcLfEuu1ZCDw2yImzqhg4XL1tDtAXKrEQeH2RA3rbODDZu7+Z+VL+ddijUIB4fZELflCXKPc1iNODjMhrg9xo2ivVhgwdLVeZdiDcLBYTbEFZrEvhM6mL/MLQ6rjUyDQ9Jxkh6RtFjSBX0cb5H0o/T43ZImlh3fTdLLks7Psk6zoW7ahA4eemYNm7t78i7FGkBmwSGpAFwFHA9MAWZImlJ22ieAFyNiEnAZ8LWy45cC/5VVjWbDxdTODjZ29fDYcx4gt+xl2eI4GFgcEY9HxCbgRuDEsnNOBK5LX98EHKN0RRpJ7weeABZlWKPZsDCtcwyAn+ewmsgyOCYAS0q2l6b7+jwnIrqAl4CxkkYBXwS+kmF9ZsPGxLFtjB7Z7DurrCbqdXD8YuCyiBiw3S3pLEnzJM1buXJlbSozq0OSmNbZwQIPkFsNZBkcy4BdS7Y70319niOpGegAngfeBnxd0pPAZ4ELJZ1T/gERcXVETI+I6ePHj9/uX8BsKJk6YQwPL1/Dxq7uvEuxYS7L4JgLTJa0h6QicAows+ycmcDp6euTgNsicURETIyIicA3gH+MiCszrNVsyJvW2cHm7uCRZ9fmXYoNc5kFRzpmcQ5wK/Aw8OOIWCTpEkknpKd9h2RMYzHwOeA1t+yaWWWmTvAT5FYbzVm+eUTMAmaV7buo5PUrwMmDvMfFmRRnNsx07tjKjm0jvDaHZa5eB8fNrEqSmNo5xk+QW+YcHGbDyLQJHTz63Fpe2ewBcsuOg8NsGJna2UF3T/DQ8jV5l2LDmIPDbBjxGuRWCw4Os2HkjTuMZNyoFt9ZZZlycJgNI1ufIF+ddyk2jDk4zIaZqRM6WLziZdZt7Mq7FBumHBxmw8z+u3bQE3iA3DLj4DAbZvbzE+SWMQeH2TCz0+iR7NIxkvleg9wy4uAwG4amTujwLbmWGQeH2TA0rbODx1etY80rm/MuxYYhB4fZMDQ1XUp2oeetsgxkOjuumeWjd4r1z974AB2tI3KuxrK2zy47cMWMt9bs8xwcZsPQG9qLnHfMZB5b4UWdGsGuO7bW9PMcHGbD1F8fu1feJdgw5TEOMzOrioPDzMyq4uAwM7OqODjMzKwqDg4zM6uKg8PMzKri4DAzs6o4OMzMrCqKiLxr2C4krQSe2sbLxwGrtmM5Q42/v7+/v3/j2jsiRldzwbB5cjwixm/rtZLmRcT07VnPUOLv7+/v79/Y37/aa9xVZWZmVXFwmJlZVRwciavzLiBn/v6Nzd+/sVX9/YfN4LiZmdWGWxxmZlYVB4eZmVWloYND0nGSHpG0WNIFeddTS5J2lXS7pIckLZJ0Xt415UFSQdL9kn6Zdy15kDRG0k2S/iTpYUlvz7umWpL01+n//wsl/VDSyLxrypKkayWtkLSwZN8bJM2W9Fj6+46DvU/DBoekAnAVcDwwBZghaUq+VdVUF/D5iJgCHAKc3WDfv9d5wMN5F5GjbwK/ioh9gP1poD8LSROAc4HpEbEfUABOybeqzH0POK5s3wXAnIiYDMxJtwfUsMEBHAwsjojHI2ITcCNwYs411UxELI+I+9LXa0l+YEzIt6raktQJvBe4Ju9a8iCpAzgS+A5ARGyKiNW5FlV7zUCrpGagDXgm53oyFRF3Ai+U7T4RuC59fR3w/sHep5GDYwKwpGR7KQ32g7OXpInAW4G7cy6l1r4BfAHoybmOvOwBrAS+m3bXXSOpPe+iaiUilgH/DDwNLAdeioj/zreqXOwcEcvT188COw92QSMHhwGSRgE/BT4bEWvyrqdWJP05sCIi7s27lhw1AwcA34qItwLrqKCbYrhI+/JPJAnQNwHtkj6ab1X5iuT5jEGf0Wjk4FgG7Fqy3ZnuaxiSRpCExn9ExM1511NjhwEnSHqSpJvyaEk35FtSzS0FlkZEb0vzJpIgaRTvAp6IiJURsRm4GTg055ry8JykXQDS31cMdkEjB8dcYLKkPSQVSQbFZuZcU81IEknf9sMRcWne9dRaRHwpIjojYiLJf/vbIqKh/rUZEc8CSyTtne46Bngox5Jq7WngEElt6d+HY2igmwNKzAROT1+fDvx8sAuGzey41YqILknnALeS3E1xbUQsyrmsWjoM+BiwQNID6b4LI2JWfiVZDj4D/Ef6j6fHgTNzrqdmIuJuSTcB95HcZXg/w3z6EUk/BI4CxklaCvw98FXgx5I+QbI0xYcGfR9POWJmZtVo5K4qMzPbBg4OMzOrioPDzMyq4uAwM7OqODjMzKwqDg7LnKTLJH22ZPtWSdeUbP+LpM8NcP33JJ2Uvv6NpOkDnHuGpCu3ocaJpTOGDnDOqSXb0yVdXu1nVVDLX0k6bXu/b5a29c/dhiYHh9XC70mfyJXUBIwD9i05fihwVw51VWsisCU4ImJeRJy7vT8kIv4tIr6/vd/XbHtxcFgt3AX0rvOwL7AQWCtpR0ktwJ8B90m6SNLcdG2Eq9OnefuVrqdyn6QHJc3p4/hESbdJmi9pjqTd0v07S7olve5BSYeWXbdnOunfQWVv+VXgCEkPpOs4HNW7joekiyVdJ+m3kp6S9EFJX5e0QNKv0uldkHSgpDsk3Zu2vHbpo+6LJZ2fvv6NpK9JukfSo5KO6OP8XSTdmda1sPccSe+W9If0z+gn6bxkSDpI0l3pd79H0mhJIyV9N633fknvTM89Q9LN6Xd4TNLXSz73zLSme0geKO3df3Jax4OS7hzov6ENTQ4Oy1xEPAN0pT+4DwX+QDIT79uB6cCCdGr7KyPioHRthFbgz/t7T0njgW8DfxER+wMn93HaFcB1ETEN+A+gt1vpcuCO9LoDgC0zBiiZfuOnwBkRMbfs/S4AfhsRb4mIy/r4vDcDRwMnADcAt0fEVGAD8N40PK4AToqIA4FrgX/o7zuWaI6Ig4HPkjzpW+5U4NaIeAvJmhoPSBoHfBl4V0QcAMwDPpc+If4j4Lz0+78rre9skjnupgIzgOu0dVGjtwAfBqYCH1ayCNguwFdIAuNwkjVtel0EvCd9/xMq+H42xDTslCNWc3eRhMahwKUkU9gfCrxE0pUF8E5JXyBZF+ENJD/Qf9HP+x0C3BkRTwBERPkaA5AE0wfT19cDvf9aPho4Lb2uG3hJyUyp40nm6flgRGzLnE3/FRGbJS0gmcbmV+n+BSTdXHsD+wGz08ZUgWQ678H0TkB5b/o+5eYC16bB9LOIeEDSO0h+mP8+/awiSWDvDSzvDcXeGZElHU4SakTEnyQ9BeyVvv+ciHgpPe8hYHeS7sbfRMTKdP+PSs7/PfA9ST8uqd2GEQeH1UrvOMdUkq6qJcDngTUk60GMBP6VZDW2JZIuBmq9jOdLJBPfHc62Tfa3ESAieiRtjq3z+fSQ/F0TsCgiql2edWP6ezd9/J2NiDslHUmyKNX3JF0KvAjMjogZpedKmlrlZ5d+fr81lNXzV5LeltZzr6QDI+L5bfhcq1PuqrJauYuk6+mFiOhOWwhjSFoFd7E1JFalffEnDfJ+fwSOlLQHJOsm9/OZvUuBfgT4bfp6DvDp9LqCkpXwADYBHwBOU8ndUyXWAqMHqWsgjwDjla7rLWmEpH0HuWZQknYHnouIb5OsZngAyZ/PYZImpee0S9orrWGX3vGbdHyjmeTP5iPpvr2A3dJz+3M38A5JY9OWzpauQklvjoi7I+IikoWidu3vTWxocovDamUBSffGD8r2jYqIVQCSvk3SGnmWpPulXxGxUtJZwM1K7tRaARxbdtpnSFozf0PyA6x35tfzgKuVzAbaTRIiy9P3XadkkafZkl6OiNKp9ucD3ZIeJFm7+f4qvj8RsUnJbcWXp2HVTLIK4eudlfko4G8kbQZeBk5L/3zOAH6o5AYEgC9HxKOSPgxcIamVZHzjXSStvW+l3WxdJGM8G9XP/QkRsTxtFf4BWA08UHL4/0uaTNLCmgM8+Dq/n9UZz45rZmZVcVeVmZlVxcFhZmZVcXCYmVlVHBxmZlYVB4eZmVXFwWFmZlVxcJiZWVX+FxfOQx1HEspKAAAAAElFTkSuQmCC\n", "text/plain": [ - "
" + "
" ] }, - "metadata": {}, + "metadata": { + "needs_background": "light" + }, "output_type": "display_data" } ], @@ -354,9 +356,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "dask", "language": "python", - "name": "python3" + "name": "dask" }, "language_info": { "codemirror_mode": { @@ -368,7 +370,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.6.9" } }, "nbformat": 4, diff --git a/examples/03_pytorch_mnist_hpo.py b/examples/03_pytorch_mnist_hpo.py index 14fba45..c7eef27 100644 --- a/examples/03_pytorch_mnist_hpo.py +++ b/examples/03_pytorch_mnist_hpo.py @@ -6,7 +6,7 @@ this space can be passed to an object of class Model() which can instantiate a CNN architecture from it. The objective_function() is the target function that DEHB minimizes for this problem. This function instantiates an architecture, an optimizer, as defined by a configuration and performs the -training and evaluation (on the validation set) as per the fidelity passed. +training and evaluation (on the validation set) as per the budget passed. The argument `runtime` can be passed to DEHB as a wallclock budget for running the optimisation. This tutorial also briefly refers to the different methods of interfacing DEHB with the Dask @@ -167,7 +167,7 @@ def evaluate(model, device, data_loader, acc=False): return loss -def train_and_evaluate(config, max_fidelity, verbose=False, **kwargs): +def train_and_evaluate(config, max_budget, verbose=False, **kwargs): device = kwargs["device"] batch_size = config["batch_size"] train_set = kwargs["train_set"] @@ -176,7 +176,7 @@ def train_and_evaluate(config, max_fidelity, verbose=False, **kwargs): test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False) model = Model(config).to(device) optimizer = optim.Adadelta(model.parameters(), lr=config["lr"]) - for epoch in range(1, int(max_fidelity)+1): + for epoch in range(1, int(max_budget)+1): train(model, device, train_loader, optimizer) accuracy = evaluate(model, device, test_loader, acc=True) if verbose: @@ -184,7 +184,7 @@ def train_and_evaluate(config, max_fidelity, verbose=False, **kwargs): return accuracy -def objective_function(config, fidelity, **kwargs): +def objective_function(config, budget, **kwargs): """ The target function to minimize for HPO""" device = kwargs["device"] @@ -204,7 +204,7 @@ def objective_function(config, fidelity, **kwargs): optimizer = optim.Adadelta(model.parameters(), lr=config["lr"]) start = time.time() # measuring wallclock time - for epoch in range(1, int(fidelity)+1): + for epoch in range(1, int(budget)+1): train(model, device, train_loader, optimizer) loss = evaluate(model, device, valid_loader) cost = time.time() - start @@ -216,7 +216,7 @@ def objective_function(config, fidelity, **kwargs): res = { "fitness": loss, "cost": cost, - "info": {"test_loss": test_loss, "fidelity": fidelity} + "info": {"test_loss": test_loss, "budget": budget} } return res @@ -228,11 +228,11 @@ def input_arguments(): parser.add_argument('--seed', type=int, default=123, metavar='S', help='random seed (default: 123)') parser.add_argument('--refit_training', action='store_true', default=False, - help='Refit with incumbent configuration on full training data and fidelity') - parser.add_argument('--min_fidelity', type=float, default=None, - help='Minimum fidelity (epoch length)') - parser.add_argument('--max_fidelity', type=float, default=None, - help='Maximum fidelity (epoch length)') + help='Refit with incumbent configuration on full training data and budget') + parser.add_argument('--min_budget', type=float, default=None, + help='Minimum budget (epoch length)') + parser.add_argument('--max_budget', type=float, default=None, + help='Maximum budget (epoch length)') parser.add_argument('--eta', type=int, default=3, help='Parameter for Hyperband controlling early stopping aggressiveness') parser.add_argument('--output_path', type=str, default="./pytorch_mnist_dehb", @@ -250,7 +250,7 @@ def input_arguments(): parser.add_argument('--verbose', action="store_true", default=False, help='Decides verbosity of DEHB optimization') parser.add_argument('--runtime', type=float, default=300, - help='Total time in seconds as fidelity to run DEHB') + help='Total time in seconds as budget to run DEHB') args = parser.parse_args() return args @@ -300,8 +300,8 @@ def main(): # DEHB optimisation block # ########################### np.random.seed(args.seed) - dehb = DEHB(f=objective_function, cs=cs, dimensions=dimensions, min_fidelity=args.min_fidelity, - max_fidelity=args.max_fidelity, eta=args.eta, output_path=args.output_path, + dehb = DEHB(f=objective_function, cs=cs, dimensions=dimensions, min_budget=args.min_budget, + max_budget=args.max_budget, eta=args.eta, output_path=args.output_path, # if client is not None and of type Client, n_workers is ignored # if client is None, a Dask client with n_workers is set up client=client, n_workers=args.n_workers) @@ -325,7 +325,7 @@ def main(): root='./data', train=True, download=True, transform=transform ) incumbent = dehb.vector_to_configspace(dehb.inc_config) - acc = train_and_evaluate(incumbent, args.max_fidelity, verbose=True, + acc = train_and_evaluate(incumbent, args.max_budget, verbose=True, train_set=train_set, test_set=test_set, device=device) dehb.logger.info("Test accuracy of {:.3f} for the best found configuration: ".format(acc)) dehb.logger.info(incumbent) diff --git a/src/dehb/optimizers/de.py b/src/dehb/optimizers/de.py index d1c40a2..d1227f5 100644 --- a/src/dehb/optimizers/de.py +++ b/src/dehb/optimizers/de.py @@ -1,20 +1,17 @@ import os -from typing import List - +import numpy as np import ConfigSpace import ConfigSpace.util -import numpy as np +from typing import List from distributed import Client -from ..utils import ConfigRepository - class DEBase(): '''Base class for Differential Evolution ''' def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=None, - mutation_factor=None, crossover_prob=None, strategy=None, - boundary_fix_type='random', config_repository=None, **kwargs): + mutation_factor=None, crossover_prob=None, strategy=None, budget=None, + boundary_fix_type='random', **kwargs): # Benchmark related variables self.cs = cs self.f = f @@ -29,6 +26,7 @@ def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=None self.mutation_factor = mutation_factor self.crossover_prob = crossover_prob self.strategy = strategy + self.budget = budget self.fix_type = boundary_fix_type # Miscellaneous @@ -41,28 +39,18 @@ def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=None self.output_path = kwargs['output_path'] if 'output_path' in kwargs else './' os.makedirs(self.output_path, exist_ok=True) - if config_repository: - self.config_repository = config_repository - else: - self.config_repository = ConfigRepository() - # Global trackers - self.inc_score : float - self.inc_config : np.ndarray[float] - self.inc_id : int - self.population : np.ndarray[np.ndarray[float]] - self.population_ids :np.ndarray[int] - self.fitness : np.ndarray[float] - self.age : int - self.history : list[object] - self.reset() + self.inc_score = np.inf + self.inc_config = None + self.population = None + self.fitness = None + self.age = None + self.history = [] def reset(self): self.inc_score = np.inf self.inc_config = None - self.inc_id = -1 self.population = None - self.population_ids = None self.fitness = None self.age = None self.history = [] @@ -107,7 +95,6 @@ def init_population(self, pop_size: int) -> List: else: # if no ConfigSpace representation available, uniformly sample from [0, 1] population = np.random.uniform(low=0.0, high=1.0, size=(pop_size, self.dimensions)) - return np.array(population) def sample_population(self, size: int = 3, alt_pop: List = None) -> List: @@ -131,7 +118,7 @@ def sample_population(self, size: int = 3, alt_pop: List = None) -> List: selection = np.random.choice(np.arange(len(self.population)), size, replace=False) return self.population[selection] - def boundary_check(self, vector: np.ndarray) -> np.ndarray: + def boundary_check(self, vector: np.array) -> np.array: ''' Checks whether each of the dimensions of the input vector are within [0, 1]. If not, values of those dimensions are replaced with the type of fix selected. @@ -156,7 +143,7 @@ def boundary_check(self, vector: np.ndarray) -> np.ndarray: vector[violations] = np.clip(vector[violations], a_min=0, a_max=1) return vector - def vector_to_configspace(self, vector: np.ndarray) -> ConfigSpace.Configuration: + def vector_to_configspace(self, vector: np.array) -> ConfigSpace.Configuration: '''Converts numpy array to ConfigSpace object Works when self.cs is a ConfigSpace object and the input vector is in the domain [0, 1]. @@ -194,7 +181,7 @@ def vector_to_configspace(self, vector: np.ndarray) -> ConfigSpace.Configuration ) return new_config - def configspace_to_vector(self, config: ConfigSpace.Configuration) -> np.ndarray: + def configspace_to_vector(self, config: ConfigSpace.Configuration) -> np.array: '''Converts ConfigSpace object to numpy array scaled to [0,1] Works when self.cs is a ConfigSpace object and the input config is a ConfigSpace object. @@ -244,11 +231,10 @@ def run(self): class DE(DEBase): def __init__(self, cs=None, f=None, dimensions=None, pop_size=20, max_age=np.inf, mutation_factor=None, crossover_prob=None, strategy='rand1_bin', - encoding=False, dim_map=None, config_repository=None, **kwargs): + budget=None, encoding=False, dim_map=None, **kwargs): super().__init__(cs=cs, f=f, dimensions=dimensions, pop_size=pop_size, max_age=max_age, mutation_factor=mutation_factor, crossover_prob=crossover_prob, - strategy=strategy, config_repository=config_repository, - **kwargs) + strategy=strategy, budget=budget, **kwargs) if self.strategy is not None: self.mutation_strategy = self.strategy.split('_')[0] self.crossover_strategy = self.strategy.split('_')[1] @@ -299,7 +285,7 @@ def map_to_original(self, vector): new_vector[i] = np.max(np.array(vector)[self.dim_map[i]]) return new_vector - def f_objective(self, x, fidelity=None, **kwargs): + def f_objective(self, x, budget=None, **kwargs): if self.f is None: raise NotImplementedError("An objective function needs to be passed.") if self.encoding: @@ -310,19 +296,18 @@ def f_objective(self, x, fidelity=None, **kwargs): else: # can insert custom scaling/transform function here config = x.copy() - if fidelity is not None: # to be used when called by multi-fidelity based optimizers - res = self.f(config, fidelity=fidelity, **kwargs) + if budget is not None: # to be used when called by multi-fidelity based optimizers + res = self.f(config, budget=budget, **kwargs) else: res = self.f(config, **kwargs) assert "fitness" in res assert "cost" in res return res - def init_eval_pop(self, fidelity=None, eval=True, **kwargs): + def init_eval_pop(self, budget=None, eval=True, **kwargs): '''Creates new population of 'pop_size' and evaluates individuals. ''' self.population = self.init_population(self.pop_size) - self.population_ids = self.config_repository.announce_population(self.population, fidelity) self.fitness = np.array([np.inf for i in range(self.pop_size)]) self.age = np.array([self.max_age] * self.pop_size) @@ -335,29 +320,25 @@ def init_eval_pop(self, fidelity=None, eval=True, **kwargs): for i in range(self.pop_size): config = self.population[i] - config_id = self.population_ids[i] - res = self.f_objective(config, fidelity, **kwargs) + res = self.f_objective(config, budget, **kwargs) self.fitness[i], cost = res["fitness"], res["cost"] info = res["info"] if "info" in res else dict() if self.fitness[i] < self.inc_score: self.inc_score = self.fitness[i] self.inc_config = config - self.inc_id = config_id - self.config_repository.tell_result(config_id, float(fidelity or 0), res["fitness"], res["cost"], info) traj.append(self.inc_score) runtime.append(cost) - history.append((config.tolist(), float(self.fitness[i]), float(fidelity or 0), info)) + history.append((config.tolist(), float(self.fitness[i]), float(budget or 0), info)) return traj, runtime, history - def eval_pop(self, population=None, population_ids=None, fidelity=None, **kwargs): + def eval_pop(self, population=None, budget=None, **kwargs): '''Evaluates a population If population=None, the current population's fitness will be evaluated If population!=None, this population will be evaluated ''' pop = self.population if population is None else population - pop_ids = self.population_ids if population_ids is None else population_ids pop_size = self.pop_size if population is None else len(pop) traj = [] runtime = [] @@ -366,7 +347,7 @@ def eval_pop(self, population=None, population_ids=None, fidelity=None, **kwargs costs = [] ages = [] for i in range(pop_size): - res = self.f_objective(pop[i], fidelity, **kwargs) + res = self.f_objective(pop[i], budget, **kwargs) fitness, cost = res["fitness"], res["cost"] info = res["info"] if "info" in res else dict() if population is None: @@ -374,11 +355,9 @@ def eval_pop(self, population=None, population_ids=None, fidelity=None, **kwargs if fitness <= self.inc_score: self.inc_score = fitness self.inc_config = pop[i] - self.inc_id = pop_ids[i] - self.config_repository.tell_result(pop_ids[i], float(fidelity or 0), info) traj.append(self.inc_score) runtime.append(cost) - history.append((pop[i].tolist(), float(fitness), float(fidelity or 0), info)) + history.append((pop[i].tolist(), float(fitness), float(budget or 0), info)) fitnesses.append(fitness) costs.append(cost) ages.append(self.max_age) @@ -484,7 +463,7 @@ def crossover(self, target, mutant): offspring = self.crossover_exp(target, mutant) return offspring - def selection(self, trials, trial_ids, fidelity=None, **kwargs): + def selection(self, trials, budget=None, **kwargs): '''Carries out a parent-offspring competition given a set of trial population ''' traj = [] @@ -492,16 +471,13 @@ def selection(self, trials, trial_ids, fidelity=None, **kwargs): history = [] for i in range(len(trials)): # evaluation of the newly created individuals - res = self.f_objective(trials[i], fidelity, **kwargs) + res = self.f_objective(trials[i], budget, **kwargs) fitness, cost = res["fitness"], res["cost"] info = res["info"] if "info" in res else dict() - # log result to config repo - self.config_repository.tell_result(trial_ids[i], float(fidelity or 0), fitness, cost, info) # selection -- competition between parent[i] -- child[i] ## equality is important for landscape exploration if fitness <= self.fitness[i]: self.population[i] = trials[i] - self.population_ids[i] = trial_ids[i] self.fitness[i] = fitness # resetting age since new individual in the population self.age[i] = self.max_age @@ -512,28 +488,23 @@ def selection(self, trials, trial_ids, fidelity=None, **kwargs): if self.fitness[i] < self.inc_score: self.inc_score = self.fitness[i] self.inc_config = self.population[i] - self.inc_id = self.population[i] traj.append(self.inc_score) runtime.append(cost) - history.append((trials[i].tolist(), float(fitness), float(fidelity or 0), info)) + history.append((trials[i].tolist(), float(fitness), float(budget or 0), info)) return traj, runtime, history - def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs): + def evolve_generation(self, budget=None, best=None, alt_pop=None, **kwargs): '''Performs a complete DE evolution: mutation -> crossover -> selection ''' trials = [] - trial_ids = [] for j in range(self.pop_size): target = self.population[j] donor = self.mutation(current=target, best=best, alt_pop=alt_pop) trial = self.crossover(target, donor) trial = self.boundary_check(trial) - trial_id = self.config_repository.announce_config(trial, float(fidelity or 0)) trials.append(trial) - trial_ids.append(trial_id) trials = np.array(trials) - trial_ids = np.array(trial_ids) - traj, runtime, history = self.selection(trials, trial_ids, fidelity, **kwargs) + traj, runtime, history = self.selection(trials, budget, **kwargs) return traj, runtime, history def sample_mutants(self, size, population=None): @@ -554,20 +525,20 @@ def sample_mutants(self, size, population=None): return mutants - def run(self, generations=1, verbose=False, fidelity=None, reset=True, **kwargs): + def run(self, generations=1, verbose=False, budget=None, reset=True, **kwargs): # checking if a run exists if not hasattr(self, 'traj') or reset: self.reset() if verbose: print("Initializing and evaluating new population...") - self.traj, self.runtime, self.history = self.init_eval_pop(fidelity=fidelity, **kwargs) + self.traj, self.runtime, self.history = self.init_eval_pop(budget=budget, **kwargs) if verbose: print("Running evolutionary search...") for i in range(generations): if verbose: print("Generation {:<2}/{:<2} -- {:<0.7}".format(i+1, generations, self.inc_score)) - traj, runtime, history = self.evolve_generation(fidelity=fidelity, **kwargs) + traj, runtime, history = self.evolve_generation(budget=budget, **kwargs) self.traj.extend(traj) self.runtime.extend(runtime) self.history.extend(history) @@ -581,7 +552,7 @@ def run(self, generations=1, verbose=False, fidelity=None, reset=True, **kwargs) class AsyncDE(DE): def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=np.inf, mutation_factor=None, crossover_prob=None, strategy='rand1_bin', - async_strategy='immediate', config_repository=None, **kwargs): + budget=None, async_strategy='immediate', **kwargs): '''Extends DE to be Asynchronous with variations Parameters @@ -600,8 +571,7 @@ def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=np.i ''' super().__init__(cs=cs, f=f, dimensions=dimensions, pop_size=pop_size, max_age=max_age, mutation_factor=mutation_factor, crossover_prob=crossover_prob, - strategy=strategy, config_repository=config_repository, - **kwargs) + strategy=strategy, budget=budget, **kwargs) if self.strategy is not None: self.mutation_strategy = self.strategy.split('_')[0] self.crossover_strategy = self.strategy.split('_')[1] @@ -672,9 +642,8 @@ def _sample_population(self, size=3, alt_pop=None, target=None): selection = np.random.choice(np.arange(len(population)), size, replace=False) return population[selection] - def eval_pop(self, population=None, population_ids=None, fidelity=None, **kwargs): + def eval_pop(self, population=None, budget=None, **kwargs): pop = self.population if population is None else population - pop_ids = self.population_ids if population_ids is None else population_ids pop_size = self.pop_size if population is None else len(pop) traj = [] runtime = [] @@ -683,7 +652,7 @@ def eval_pop(self, population=None, population_ids=None, fidelity=None, **kwargs costs = [] ages = [] for i in range(pop_size): - res = self.f_objective(pop[i], fidelity, **kwargs) + res = self.f_objective(pop[i], budget, **kwargs) fitness, cost = res["fitness"], res["cost"] info = res["info"] if "info" in res else dict() if population is None: @@ -691,11 +660,9 @@ def eval_pop(self, population=None, population_ids=None, fidelity=None, **kwargs if fitness <= self.inc_score: self.inc_score = fitness self.inc_config = pop[i] - self.inc_id = pop_ids[i] - self.config_repository.tell_result(pop_ids[i], float(fidelity or 0), fitness, cost, info) traj.append(self.inc_score) runtime.append(cost) - history.append((pop[i].tolist(), float(fitness), float(fidelity or 0), info)) + history.append((pop[i].tolist(), float(fitness), float(budget or 0), info)) fitnesses.append(fitness) costs.append(cost) ages.append(self.max_age) @@ -756,46 +723,40 @@ def sample_mutants(self, size, population=None): return mutants - def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs): + def evolve_generation(self, budget=None, best=None, alt_pop=None, **kwargs): '''Performs a complete DE evolution, mutation -> crossover -> selection ''' traj = [] runtime = [] history = [] - if self.async_strategy == "deferred": + if self.async_strategy == 'deferred': trials = [] - trial_ids = [] for j in range(self.pop_size): target = self.population[j] donor = self.mutation(current=target, best=best, alt_pop=alt_pop) trial = self.crossover(target, donor) trial = self.boundary_check(trial) - trial_id = self.config_repository.announce_config(trial, float(fidelity or 0)) trials.append(trial) - trial_ids.append(trial_id) # selection takes place on a separate trial population only after # one iteration through the population has taken place trials = np.array(trials) - traj, runtime, history = self.selection(trials, trial_ids, fidelity, **kwargs) + traj, runtime, history = self.selection(trials, budget, **kwargs) return traj, runtime, history - elif self.async_strategy == "immediate": + elif self.async_strategy == 'immediate': for i in range(self.pop_size): target = self.population[i] donor = self.mutation(current=target, best=best, alt_pop=alt_pop) trial = self.crossover(target, donor) trial = self.boundary_check(trial) - trial_id = self.config_repository.announce_config(trial, float(fidelity or 0)) # evaluating a single trial population for the i-th individual de_traj, de_runtime, de_history, fitnesses, costs = \ - self.eval_pop(trial.reshape(1, self.dimensions), - np.array([trial_id]), fidelity=fidelity, **kwargs) + self.eval_pop(trial.reshape(1, self.dimensions), budget=budget, **kwargs) # one-vs-one selection ## can replace the i-the population despite not completing one iteration if fitnesses[0] <= self.fitness[i]: self.population[i] = trial - self.population_ids[i] = trial_id self.fitness[i] = fitnesses[0] traj.extend(de_traj) runtime.extend(de_runtime) @@ -805,7 +766,7 @@ def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs): else: # async_strategy == 'random' or async_strategy == 'worst': for count in range(self.pop_size): # choosing target individual - if self.async_strategy == "random": + if self.async_strategy == 'random': i = np.random.choice(np.arange(self.pop_size)) else: # async_strategy == 'worst' i = np.argsort(-self.fitness)[0] @@ -813,11 +774,9 @@ def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs): mutant = self.mutation(current=target, best=best, alt_pop=alt_pop) trial = self.crossover(target, mutant) trial = self.boundary_check(trial) - trial_id = self.config_repository.announce_config(trial, float(fidelity or 0)) # evaluating a single trial population for the i-th individual de_traj, de_runtime, de_history, fitnesses, costs = \ - self.eval_pop(trial.reshape(1, self.dimensions), np.array([trial_id]), - fidelity=fidelity, **kwargs) + self.eval_pop(trial.reshape(1, self.dimensions), budget=budget, **kwargs) # one-vs-one selection ## can replace the i-the population despite not completing one iteration if fitnesses[0] <= self.fitness[i]: @@ -829,21 +788,22 @@ def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs): return traj, runtime, history - def run(self, generations=1, verbose=False, fidelity=None, reset=True, **kwargs): + def run(self, generations=1, verbose=False, budget=None, reset=True, **kwargs): # checking if a run exists - if not hasattr(self, "traj") or reset: + if not hasattr(self, 'traj') or reset: self.reset() if verbose: print("Initializing and evaluating new population...") - self.traj, self.runtime, self.history = self.init_eval_pop(fidelity=fidelity, **kwargs) + self.traj, self.runtime, self.history = self.init_eval_pop(budget=budget, **kwargs) if verbose: print("Running evolutionary search...") for i in range(generations): if verbose: print("Generation {:<2}/{:<2} -- {:<0.7}".format(i+1, generations, self.inc_score)) - traj, runtime, history = self.evolve_generation(fidelity=fidelity, - best=self.inc_config, **kwargs) + traj, runtime, history = self.evolve_generation( + budget=budget, best=self.inc_config, **kwargs + ) self.traj.extend(traj) self.runtime.extend(runtime) self.history.extend(history) diff --git a/src/dehb/optimizers/dehb.py b/src/dehb/optimizers/dehb.py index 612b496..b36e269 100644 --- a/src/dehb/optimizers/dehb.py +++ b/src/dehb/optimizers/dehb.py @@ -12,7 +12,6 @@ from .de import DE, AsyncDE from ..utils import SHBracketManager -from ..utils import ConfigRepository logger.configure(handlers=[{"sink": sys.stdout, "level": "INFO"}]) @@ -25,12 +24,11 @@ class DEHBBase: def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, - crossover_prob=None, strategy=None, min_fidelity=None, - max_fidelity=None, eta=None, min_clip=None, max_clip=None, + crossover_prob=None, strategy=None, min_budget=None, + max_budget=None, eta=None, min_clip=None, max_clip=None, boundary_fix_type='random', max_age=np.inf, **kwargs): # Miscellaneous self._setup_logger(kwargs) - self.config_repository = ConfigRepository() # Benchmark related variables self.cs = cs @@ -62,11 +60,11 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, } # Hyperband related variables - self.min_fidelity = min_fidelity - self.max_fidelity = max_fidelity - if self.max_fidelity <= self.min_fidelity: - self.logger.error("Only (Max Fidelity > Min Fidelity) is supported for DEHB.") - if self.max_fidelity == self.min_fidelity: + self.min_budget = min_budget + self.max_budget = max_budget + if self.max_budget <= self.min_budget: + self.logger.error("Only (Max Budget > Min Budget) is supported for DEHB.") + if self.max_budget == self.min_budget: self.logger.error( "If you have a fixed fidelity, " \ "you can instead run DE. For more information checkout: " \ @@ -76,14 +74,14 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, self.min_clip = min_clip self.max_clip = max_clip - # Precomputing fidelity spacing and number of configurations for HB iterations + # Precomputing budget spacing and number of configurations for HB iterations self.max_SH_iter = None - self.fidelities = None - if self.min_fidelity is not None and \ - self.max_fidelity is not None and \ + self.budgets = None + if self.min_budget is not None and \ + self.max_budget is not None and \ self.eta is not None: - self.max_SH_iter = -int(np.log(self.min_fidelity / self.max_fidelity) / np.log(self.eta)) + 1 - self.fidelities = self.max_fidelity * np.power(self.eta, + self.max_SH_iter = -int(np.log(self.min_budget / self.max_budget) / np.log(self.eta)) + 1 + self.budgets = self.max_budget * np.power(self.eta, -np.linspace(start=self.max_SH_iter - 1, stop=0, num=self.max_SH_iter)) @@ -126,7 +124,7 @@ def init_population(self): def get_next_iteration(self, iteration): '''Computes the Successive Halving spacing - Given the iteration index, computes the fidelity spacing to be used and + Given the iteration index, computes the budget spacing to be used and the number of configurations to be used for the SH iterations. Parameters @@ -139,12 +137,12 @@ def get_next_iteration(self, iteration): Returns ------- ns : array - fidelities : array + budgets : array ''' # number of 'SH runs' s = self.max_SH_iter - 1 - (iteration % self.max_SH_iter) - # fidelity spacing for this iteration - fidelities = self.fidelities[(-s-1):] + # budget spacing for this iteration + budgets = self.budgets[(-s-1):] # number of configurations in that bracket n0 = int(np.floor((self.max_SH_iter)/(s+1)) * self.eta**s) ns = [max(int(n0*(self.eta**(-i))), 1) for i in range(s+1)] @@ -153,7 +151,7 @@ def get_next_iteration(self, iteration): elif self.min_clip is not None: ns = np.clip(ns, a_min=self.min_clip, a_max=np.max(ns)) - return ns, fidelities + return ns, budgets def get_incumbents(self): """ Returns a tuple of the (incumbent configuration, incumbent score/fitness). """ @@ -170,13 +168,13 @@ def run(self): class DEHB(DEHBBase): def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=0.5, - crossover_prob=0.5, strategy='rand1_bin', min_fidelity=None, - max_fidelity=None, eta=3, min_clip=None, max_clip=None, configspace=True, + crossover_prob=0.5, strategy='rand1_bin', min_budget=None, + max_budget=None, eta=3, min_clip=None, max_clip=None, configspace=True, boundary_fix_type='random', max_age=np.inf, n_workers=None, client=None, async_strategy="immediate", **kwargs): super().__init__(cs=cs, f=f, dimensions=dimensions, mutation_factor=mutation_factor, - crossover_prob=crossover_prob, strategy=strategy, min_fidelity=min_fidelity, - max_fidelity=max_fidelity, eta=eta, min_clip=min_clip, max_clip=max_clip, + crossover_prob=crossover_prob, strategy=strategy, min_budget=min_budget, + max_budget=max_budget, eta=eta, min_clip=min_clip, max_clip=max_clip, configspace=configspace, boundary_fix_type=boundary_fix_type, max_age=max_age, **kwargs) self.de_params.update({"async_strategy": async_strategy}) @@ -238,21 +236,19 @@ def _f_objective(self, job_info): # reprioritising a CUDA device order specific to this worker process os.environ.update({"CUDA_VISIBLE_DEVICES": job_info["gpu_devices"]}) - config, config_id = job_info["config"], job_info["config_id"] - fidelity, parent_id = job_info["fidelity"], job_info["parent_id"] - bracket_id = job_info["bracket_id"] + config, budget, parent_id = job_info['config'], job_info['budget'], job_info['parent_id'] + bracket_id = job_info['bracket_id'] kwargs = job_info["kwargs"] - res = self.de[fidelity].f_objective(config, fidelity, **kwargs) - info = res["info"] if "info" in res else {} + res = self.de[budget].f_objective(config, budget, **kwargs) + info = res["info"] if "info" in res else dict() run_info = { - "fitness": res["fitness"], - "cost": res["cost"], - "config": config, - "config_id": config_id, - "fidelity": fidelity, - "parent_id": parent_id, - "bracket_id": bracket_id, - "info": info, + 'fitness': res["fitness"], + 'cost': res["cost"], + 'config': config, + 'budget': budget, + 'parent_id': parent_id, + 'bracket_id': bracket_id, + 'info': info } if "gpu_devices" in job_info: @@ -299,13 +295,13 @@ def distribute_gpus(self): def vector_to_configspace(self, config): assert hasattr(self, "de") - assert len(self.fidelities) > 0 - return self.de[self.fidelities[0]].vector_to_configspace(config) + assert len(self.budgets) > 0 + return self.de[self.budgets[0]].vector_to_configspace(config) def configspace_to_vector(self, config): assert hasattr(self, "de") - assert len(self.fidelities) > 0 - return self.de[self.fidelities[0]].configspace_to_vector(config) + assert len(self.budgets) > 0 + return self.de[self.budgets[0]].configspace_to_vector(config) def reset(self): super().reset() @@ -357,7 +353,7 @@ def _update_incumbents(self, config, score, info): self.inc_info = info def _get_pop_sizes(self): - """Determines maximum pop size for each fidelity + """Determines maximum pop size for each budget """ self._max_pop_size = {} for i in range(self.max_SH_iter): @@ -368,30 +364,27 @@ def _get_pop_sizes(self): ) if r_j in self._max_pop_size.keys() else n[j] def _init_subpop(self): - """ List of DE objects corresponding to the fidelities + """ List of DE objects corresponding to the budgets (fidelities) """ self.de = {} - for i, f in enumerate(self._max_pop_size.keys()): - self.de[f] = AsyncDE(**self.de_params, pop_size=self._max_pop_size[f], - config_repository=self.config_repository) - self.de[f].population = self.de[f].init_population(pop_size=self._max_pop_size[f]) - self.de[f].population_ids = self.config_repository.announce_population(self.de[f].population, f) - self.de[f].fitness = np.array([np.inf] * self._max_pop_size[f]) + for i, b in enumerate(self._max_pop_size.keys()): + self.de[b] = AsyncDE(**self.de_params, budget=b, pop_size=self._max_pop_size[b]) + self.de[b].population = self.de[b].init_population(pop_size=self._max_pop_size[b]) + self.de[b].fitness = np.array([np.inf] * self._max_pop_size[b]) # adding attributes to DEHB objects to allow communication across subpopulations - self.de[f].parent_counter = 0 - self.de[f].promotion_pop = None - self.de[f].promotion_pop_ids = None - self.de[f].promotion_fitness = None + self.de[b].parent_counter = 0 + self.de[b].promotion_pop = None + self.de[b].promotion_fitness = None - def _concat_pops(self, exclude_fidelity=None): + def _concat_pops(self, exclude_budget=None): """ Concatenates all subpopulations """ - fidelities = list(self.fidelities) - if exclude_fidelity is not None: - fidelities.remove(exclude_fidelity) + budgets = list(self.budgets) + if exclude_budget is not None: + budgets.remove(exclude_budget) pop = [] - for fidelity in fidelities: - pop.extend(self.de[fidelity].population.tolist()) + for budget in budgets: + pop.extend(self.de[budget].population.tolist()) return np.array(pop) def _start_new_bracket(self): @@ -399,9 +392,9 @@ def _start_new_bracket(self): """ # start new bracket self.iteration_counter += 1 # iteration counter gives the bracket count or bracket ID - n_configs, fidelities = self.get_next_iteration(self.iteration_counter) + n_configs, budgets = self.get_next_iteration(self.iteration_counter) bracket = SHBracketManager( - n_configs=n_configs, fidelities=fidelities, bracket_id=self.iteration_counter + n_configs=n_configs, budgets=budgets, bracket_id=self.iteration_counter ) self.active_brackets.append(bracket) return bracket @@ -427,122 +420,109 @@ def is_worker_available(self, verbose=False): return False return True - def _get_promotion_candidate(self, low_fidelity, high_fidelity, n_configs): - """ Manages the population to be promoted from the lower to the higher fidelity. + def _get_promotion_candidate(self, low_budget, high_budget, n_configs): + """ Manages the population to be promoted from the lower to the higher budget. This is triggered or in action only during the first full HB bracket, which is equivalent to the number of brackets <= max_SH_iter. """ # finding the individuals that have been evaluated (fitness < np.inf) - evaluated_configs = np.where(self.de[low_fidelity].fitness != np.inf)[0] - promotion_candidate_pop = self.de[low_fidelity].population[evaluated_configs] - promotion_candidate_pop_ids = self.de[low_fidelity].population_ids[evaluated_configs] - promotion_candidate_fitness = self.de[low_fidelity].fitness[evaluated_configs] + evaluated_configs = np.where(self.de[low_budget].fitness != np.inf)[0] + promotion_candidate_pop = self.de[low_budget].population[evaluated_configs] + promotion_candidate_fitness = self.de[low_budget].fitness[evaluated_configs] # ordering the evaluated individuals based on their fitness values pop_idx = np.argsort(promotion_candidate_fitness) # creating population for promotion if none promoted yet or nothing to promote - if self.de[high_fidelity].promotion_pop is None or \ - len(self.de[high_fidelity].promotion_pop) == 0: - self.de[high_fidelity].promotion_pop = np.empty((0, self.dimensions)) - self.de[high_fidelity].promotion_pop_ids = np.array([], dtype=np.int64) - self.de[high_fidelity].promotion_fitness = np.array([]) - - # iterating over the evaluated individuals from the lower fidelity and including them - # in the promotion population for the higher fidelity only if it's not in the population + if self.de[high_budget].promotion_pop is None or \ + len(self.de[high_budget].promotion_pop) == 0: + self.de[high_budget].promotion_pop = np.empty((0, self.dimensions)) + self.de[high_budget].promotion_fitness = np.array([]) + + # iterating over the evaluated individuals from the lower budget and including them + # in the promotion population for the higher budget only if it's not in the population # this is done to ensure diversity of population and avoid redundant evaluations for idx in pop_idx: individual = promotion_candidate_pop[idx] - individual_id = promotion_candidate_pop_ids[idx] - # checks if the candidate individual already exists in the high fidelity population - if np.any(np.all(individual == self.de[high_fidelity].population, axis=1)): + # checks if the candidate individual already exists in the high budget population + if np.any(np.all(individual == self.de[high_budget].population, axis=1)): # skipping already present individual to allow diversity and reduce redundancy continue - self.de[high_fidelity].promotion_pop = np.append( - self.de[high_fidelity].promotion_pop, [individual], axis=0 - ) - self.de[high_fidelity].promotion_pop_ids = np.append( - self.de[high_fidelity].promotion_pop_ids, individual_id + self.de[high_budget].promotion_pop = np.append( + self.de[high_budget].promotion_pop, [individual], axis=0 ) - self.de[high_fidelity].promotion_fitness = np.append( - self.de[high_fidelity].promotion_pop, promotion_candidate_fitness[pop_idx] + self.de[high_budget].promotion_fitness = np.append( + self.de[high_budget].promotion_pop, promotion_candidate_fitness[pop_idx] ) # retaining only n_configs - self.de[high_fidelity].promotion_pop = self.de[high_fidelity].promotion_pop[:n_configs] - self.de[high_fidelity].promotion_pop_ids = self.de[high_fidelity].promotion_pop_ids[:n_configs] - self.de[high_fidelity].promotion_fitness = \ - self.de[high_fidelity].promotion_fitness[:n_configs] - - if len(self.de[high_fidelity].promotion_pop) > 0: - config = self.de[high_fidelity].promotion_pop[0] - config_id = self.de[high_fidelity].promotion_pop_ids[0] + self.de[high_budget].promotion_pop = self.de[high_budget].promotion_pop[:n_configs] + self.de[high_budget].promotion_fitness = \ + self.de[high_budget].promotion_fitness[:n_configs] + + if len(self.de[high_budget].promotion_pop) > 0: + config = self.de[high_budget].promotion_pop[0] # removing selected configuration from population - self.de[high_fidelity].promotion_pop = self.de[high_fidelity].promotion_pop[1:] - self.de[high_fidelity].promotion_pop_ids = self.de[high_fidelity].promotion_pop_ids[1:] - self.de[high_fidelity].promotion_fitness = self.de[high_fidelity].promotion_fitness[1:] + self.de[high_budget].promotion_pop = self.de[high_budget].promotion_pop[1:] + self.de[high_budget].promotion_fitness = self.de[high_budget].promotion_fitness[1:] else: - # in case of an edge failure case where all high fidelity individuals are same - # just choose the best performing individual from the lower fidelity (again) - config = self.de[low_fidelity].population[pop_idx[0]] - config_id = self.de[low_fidelity].population_ids[pop_idx[0]] - return config, config_id + # in case of an edge failure case where all high budget individuals are same + # just choose the best performing individual from the lower budget (again) + config = self.de[low_budget].population[pop_idx[0]] + return config - def _get_next_parent_for_subpop(self, fidelity): + def _get_next_parent_for_subpop(self, budget): """ Maintains a looping counter over a subpopulation, to iteratively select a parent """ - parent_id = self.de[fidelity].parent_counter - self.de[fidelity].parent_counter += 1 - self.de[fidelity].parent_counter = self.de[fidelity].parent_counter % self._max_pop_size[fidelity] + parent_id = self.de[budget].parent_counter + self.de[budget].parent_counter += 1 + self.de[budget].parent_counter = self.de[budget].parent_counter % self._max_pop_size[budget] return parent_id - def _acquire_config(self, bracket, fidelity): - """ Generates/chooses a configuration based on the fidelity and iteration number + def _acquire_config(self, bracket, budget): + """ Generates/chooses a configuration based on the budget and iteration number """ # select a parent/target - parent_id = self._get_next_parent_for_subpop(fidelity) - target = self.de[fidelity].population[parent_id] - # identify lower fidelity to transfer information from - lower_fidelity, num_configs = bracket.get_lower_fidelity_promotions(fidelity) + parent_id = self._get_next_parent_for_subpop(budget) + target = self.de[budget].population[parent_id] + # identify lower budget/fidelity to transfer information from + lower_budget, num_configs = bracket.get_lower_budget_promotions(budget) if self.iteration_counter < self.max_SH_iter: # promotions occur only in the first set of SH brackets under Hyperband - # for the first rung/fidelity in the current bracket, no promotion is possible and + # for the first rung/budget in the current bracket, no promotion is possible and # evolution can begin straight away - # for the subsequent rungs, individuals will be promoted from the lower_fidelity - if fidelity != bracket.fidelities[0]: - # TODO: check if generalizes to all fidelity spacings - config, config_id = self._get_promotion_candidate(lower_fidelity, fidelity, num_configs) - return config, config_id, parent_id + # for the subsequent rungs, individuals will be promoted from the lower_budget + if budget != bracket.budgets[0]: + # TODO: check if generalizes to all budget spacings + config = self._get_promotion_candidate(lower_budget, budget, num_configs) + return config, parent_id # DE evolution occurs when either all individuals in the subpopulation have been evaluated # at least once, i.e., has fitness < np.inf, which can happen if # iteration_counter <= max_SH_iter but certainly never when iteration_counter > max_SH_iter # a single DE evolution --- (mutation + crossover) occurs here - mutation_pop_idx = np.argsort(self.de[lower_fidelity].fitness)[:num_configs] - mutation_pop = self.de[lower_fidelity].population[mutation_pop_idx] - # generate mutants from previous fidelity subpopulation or global population - if len(mutation_pop) < self.de[fidelity]._min_pop_size: - filler = self.de[fidelity]._min_pop_size - len(mutation_pop) + 1 - new_pop = self.de[fidelity]._init_mutant_population( + mutation_pop_idx = np.argsort(self.de[lower_budget].fitness)[:num_configs] + mutation_pop = self.de[lower_budget].population[mutation_pop_idx] + # generate mutants from previous budget subpopulation or global population + if len(mutation_pop) < self.de[budget]._min_pop_size: + filler = self.de[budget]._min_pop_size - len(mutation_pop) + 1 + new_pop = self.de[budget]._init_mutant_population( pop_size=filler, population=self._concat_pops(), target=target, best=self.inc_config ) mutation_pop = np.concatenate((mutation_pop, new_pop)) # generate mutant from among individuals in mutation_pop - mutant = self.de[fidelity].mutation( + mutant = self.de[budget].mutation( current=target, best=self.inc_config, alt_pop=mutation_pop ) # perform crossover with selected parent - config = self.de[fidelity].crossover(target=target, mutant=mutant) - config = self.de[fidelity].boundary_check(config) - - # announce new config - config_id = self.config_repository.announce_config(config, fidelity) - return config, config_id, parent_id + config = self.de[budget].crossover(target=target, mutant=mutant) + config = self.de[budget].boundary_check(config) + return config, parent_id def _get_next_job(self): - """ Loads a configuration and fidelity to be evaluated next by a free worker + """ Loads a configuration and budget to be evaluated next by a free worker """ bracket = None if len(self.active_brackets) == 0 or \ @@ -561,14 +541,13 @@ def _get_next_job(self): if bracket is None: # start new bracket when existing list has all waiting brackets bracket = self._start_new_bracket() - # fidelity that the SH bracket allots - fidelity = bracket.get_next_job_fidelity() - config, config_id, parent_id = self._acquire_config(bracket, fidelity) - # notifies the Bracket Manager that a single config is to run for the fidelity chosen + # budget that the SH bracket allots + budget = bracket.get_next_job_budget() + config, parent_id = self._acquire_config(bracket, budget) + # notifies the Bracket Manager that a single config is to run for the budget chosen job_info = { "config": config, - "config_id": config_id, - "fidelity": fidelity, + "budget": budget, "parent_id": parent_id, "bracket_id": bracket.bracket_id } @@ -591,7 +570,7 @@ def _get_gpu_id_with_low_load(self): return gpu_ids def submit_job(self, job_info, **kwargs): - """ Asks a free worker to run the objective function on config and fidelity + """ Asks a free worker to run the objective function on config and budget """ job_info["kwargs"] = self.shared_data if self.shared_data is not None else kwargs # submit to to Dask client @@ -610,7 +589,7 @@ def submit_job(self, job_info, **kwargs): for bracket in self.active_brackets: if bracket.bracket_id == job_info['bracket_id']: # registering is IMPORTANT for Bracket Manager to perform SH - bracket.register_job(job_info['fidelity']) + bracket.register_job(job_info['budget']) break def _fetch_results_from_workers(self): @@ -639,32 +618,29 @@ def _fetch_results_from_workers(self): # update bracket information fitness, cost = run_info["fitness"], run_info["cost"] info = run_info["info"] if "info" in run_info else dict() - fidelity, parent_id = run_info["fidelity"], run_info["parent_id"] - config, config_id = run_info["config"], run_info["config_id"] + budget, parent_id = run_info["budget"], run_info["parent_id"] + config = run_info["config"] bracket_id = run_info["bracket_id"] for bracket in self.active_brackets: if bracket.bracket_id == bracket_id: # bracket job complete - bracket.complete_job(fidelity) # IMPORTANT to perform synchronous SH - - self.config_repository.tell_result(config_id, fidelity, fitness, cost, info) + bracket.complete_job(budget) # IMPORTANT to perform synchronous SH # carry out DE selection - if fitness <= self.de[fidelity].fitness[parent_id]: - self.de[fidelity].population[parent_id] = config - self.de[fidelity].population_ids[parent_id] = config_id - self.de[fidelity].fitness[parent_id] = fitness + if fitness <= self.de[budget].fitness[parent_id]: + self.de[budget].population[parent_id] = config + self.de[budget].fitness[parent_id] = fitness # updating incumbents - if self.de[fidelity].fitness[parent_id] < self.inc_score: + if self.de[budget].fitness[parent_id] < self.inc_score: self._update_incumbents( - config=self.de[fidelity].population[parent_id], - score=self.de[fidelity].fitness[parent_id], + config=self.de[budget].population[parent_id], + score=self.de[budget].fitness[parent_id], info=info ) # book-keeping self._update_trackers( traj=self.inc_score, runtime=cost, history=( - config.tolist(), float(fitness), float(cost), float(fidelity), info + config.tolist(), float(fitness), float(cost), float(budget), info ) ) # remove processed future @@ -751,7 +727,7 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus """ Main interface to run optimization by DEHB This function waits on workers and if a worker is free, asks for a configuration and a - fidelity to evaluate on and submits it to the worker. In each loop, it checks if a job + budget to evaluate on and submits it to the worker. In each loop, it checks if a job is complete, fetches the results, carries the necessary processing of it asynchronously to the worker computations. @@ -787,7 +763,7 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus break if self.is_worker_available(): job_info = self._get_next_job() - if brackets is not None and job_info["bracket_id"] >= brackets: + if brackets is not None and job_info['bracket_id'] >= brackets: # ignore submission and only collect results # when brackets are chosen as run budget, an extra bracket is created # since iteration_counter is incremented in _get_next_job() and then checked @@ -804,12 +780,11 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus # submits job_info to a worker for execution self.submit_job(job_info, **kwargs) if verbose: - fidelity = job_info["fidelity"] - config_id = job_info["config_id"] + budget = job_info['budget'] self._verbosity_runtime(fevals, brackets, total_cost) self.logger.info( - "Evaluating configuration {} with fidelity {} under " - "bracket ID {}".format(config_id, fidelity, job_info["bracket_id"]) + "Evaluating a configuration with budget {} under " + "bracket ID {}".format(budget, job_info['bracket_id']) ) self.logger.info( "Best score seen/Incumbent score: {}".format(self.inc_score) diff --git a/src/dehb/utils/__init__.py b/src/dehb/utils/__init__.py index 65bbe00..dd9d4f0 100644 --- a/src/dehb/utils/__init__.py +++ b/src/dehb/utils/__init__.py @@ -1,2 +1 @@ -from .bracket_manager import SHBracketManager -from .config_repository import ConfigRepository \ No newline at end of file +from .bracket_manager import SHBracketManager \ No newline at end of file diff --git a/src/dehb/utils/bracket_manager.py b/src/dehb/utils/bracket_manager.py index 2642223..6f2079e 100644 --- a/src/dehb/utils/bracket_manager.py +++ b/src/dehb/utils/bracket_manager.py @@ -4,93 +4,93 @@ class SHBracketManager(object): """ Synchronous Successive Halving utilities """ - def __init__(self, n_configs, fidelities, bracket_id=None): - assert len(n_configs) == len(fidelities) + def __init__(self, n_configs, budgets, bracket_id=None): + assert len(n_configs) == len(budgets) self.n_configs = n_configs - self.fidelities = fidelities + self.budgets = budgets self.bracket_id = bracket_id self.sh_bracket = {} self._sh_bracket = {} self._config_map = {} - for i, fidelity in enumerate(fidelities): + for i, budget in enumerate(budgets): # sh_bracket keeps track of jobs/configs that are still to be scheduled/allocatted # _sh_bracket keeps track of jobs/configs that have been run and results retrieved for # (sh_bracket[i] + _sh_bracket[i]) == n_configs[i] is when no jobs have been scheduled - # or all jobs for that fidelity/rung are over + # or all jobs for that budget/rung are over # (sh_bracket[i] + _sh_bracket[i]) < n_configs[i] indicates a job has been scheduled # and is queued/running and the bracket needs to be paused till results are retrieved - self.sh_bracket[fidelity] = n_configs[i] # each scheduled job does -= 1 - self._sh_bracket[fidelity] = 0 # each retrieved job does +=1 - self.n_rungs = len(fidelities) + self.sh_bracket[budget] = n_configs[i] # each scheduled job does -= 1 + self._sh_bracket[budget] = 0 # each retrieved job does +=1 + self.n_rungs = len(budgets) self.current_rung = 0 - def get_fidelity(self, rung=None): - """ Returns the exact fidelity that rung is pointing to. + def get_budget(self, rung=None): + """ Returns the exact budget that rung is pointing to. - Returns current rung's fidelity if no rung is passed. + Returns current rung's budget if no rung is passed. """ if rung is not None: - return self.fidelities[rung] - return self.fidelities[self.current_rung] + return self.budgets[rung] + return self.budgets[self.current_rung] - def get_lower_fidelity_promotions(self, fidelity): - """ Returns the immediate lower fidelity and the number of configs to be promoted from there + def get_lower_budget_promotions(self, budget): + """ Returns the immediate lower budget and the number of configs to be promoted from there """ - assert fidelity in self.fidelities - rung = np.where(fidelity == self.fidelities)[0][0] + assert budget in self.budgets + rung = np.where(budget == self.budgets)[0][0] prev_rung = np.clip(rung - 1, a_min=0, a_max=self.n_rungs-1) - lower_fidelity = self.fidelities[prev_rung] + lower_budget = self.budgets[prev_rung] num_promote_configs = self.n_configs[rung] - return lower_fidelity, num_promote_configs + return lower_budget, num_promote_configs - def get_next_job_fidelity(self): - """ Returns the fidelity that will be selected if current_rung is incremented by 1 + def get_next_job_budget(self): + """ Returns the budget that will be selected if current_rung is incremented by 1 """ - if self.sh_bracket[self.get_fidelity()] > 0: + if self.sh_bracket[self.get_budget()] > 0: # the current rung still has unallocated jobs (>0) - return self.get_fidelity() + return self.get_budget() else: # the current rung has no more jobs to allocate, increment it rung = (self.current_rung + 1) % self.n_rungs - if self.sh_bracket[self.get_fidelity(rung)] > 0: + if self.sh_bracket[self.get_budget(rung)] > 0: # the incremented rung has unallocated jobs (>0) - return self.get_fidelity(rung) + return self.get_budget(rung) else: # all jobs for this bracket has been allocated/bracket is complete - # no more fidelities to evaluate and can return None + # no more budgets to evaluate and can return None pass return None - def register_job(self, fidelity): - """ Registers the allocation of a configuration for the fidelity and updates current rung + def register_job(self, budget): + """ Registers the allocation of a configuration for the budget and updates current rung This function must be called when scheduling a job in order to allow the bracket manager - to continue job and fidelity allocation without waiting for jobs to finish and return + to continue job and budget allocation without waiting for jobs to finish and return results necessarily. This feature can be leveraged to run brackets asynchronously. """ - assert fidelity in self.fidelities - assert self.sh_bracket[fidelity] > 0 - self.sh_bracket[fidelity] -= 1 + assert budget in self.budgets + assert self.sh_bracket[budget] > 0 + self.sh_bracket[budget] -= 1 if not self._is_rung_pending(self.current_rung): # increment current rung if no jobs left in the rung self.current_rung = (self.current_rung + 1) % self.n_rungs - def complete_job(self, fidelity): - """ Notifies the bracket that a job for a fidelity has been completed + def complete_job(self, budget): + """ Notifies the bracket that a job for a budget has been completed - This function must be called when a config for a fidelity has finished evaluation to inform + This function must be called when a config for a budget has finished evaluation to inform the Bracket Manager that no job needs to be waited for and the next rung can begin for the synchronous Successive Halving case. """ - assert fidelity in self.fidelities - _max_configs = self.n_configs[list(self.fidelities).index(fidelity)] - assert self._sh_bracket[fidelity] < _max_configs - self._sh_bracket[fidelity] += 1 + assert budget in self.budgets + _max_configs = self.n_configs[list(self.budgets).index(budget)] + assert self._sh_bracket[budget] < _max_configs + self._sh_bracket[budget] += 1 def _is_rung_waiting(self, rung): """ Returns True if at least one job is still pending/running and waits for results """ - job_count = self._sh_bracket[self.fidelities[rung]] + self.sh_bracket[self.fidelities[rung]] + job_count = self._sh_bracket[self.budgets[rung]] + self.sh_bracket[self.budgets[rung]] if job_count < self.n_configs[rung]: return True return False @@ -98,7 +98,7 @@ def _is_rung_waiting(self, rung): def _is_rung_pending(self, rung): """ Returns True if at least one job pending to be allocatted in the rung """ - if self.sh_bracket[self.fidelities[rung]] > 0: + if self.sh_bracket[self.budgets[rung]] > 0: return True return False @@ -116,33 +116,33 @@ def is_bracket_done(self): return ~self.is_pending() and ~self.is_waiting() def is_pending(self): - """ Returns True if any of the rungs/fidelities have still a configuration to submit + """ Returns True if any of the rungs/budgets have still a configuration to submit """ - return np.any([self._is_rung_pending(i) > 0 for i, _ in enumerate(self.fidelities)]) + return np.any([self._is_rung_pending(i) > 0 for i, _ in enumerate(self.budgets)]) def is_waiting(self): - """ Returns True if any of the rungs/fidelities have a configuration pending/running + """ Returns True if any of the rungs/budgets have a configuration pending/running """ - return np.any([self._is_rung_waiting(i) > 0 for i, _ in enumerate(self.fidelities)]) + return np.any([self._is_rung_waiting(i) > 0 for i, _ in enumerate(self.budgets)]) def __repr__(self): - cell_width = 10 + cell_width = 9 cell = "{{:^{}}}".format(cell_width) - fidelity_cell = "{{:^{}.2f}}".format(cell_width) + budget_cell = "{{:^{}.2f}}".format(cell_width) header = "|{}|{}|{}|{}|".format( - cell.format("fidelity"), + cell.format("budget"), cell.format("pending"), cell.format("waiting"), cell.format("done") ) _hline = "-" * len(header) table = [header, _hline] - for i, fidelity in enumerate(self.fidelities): - pending = self.sh_bracket[fidelity] - done = self._sh_bracket[fidelity] + for i, budget in enumerate(self.budgets): + pending = self.sh_bracket[budget] + done = self._sh_bracket[budget] waiting = np.abs(self.n_configs[i] - pending - done) entry = "|{}|{}|{}|{}|".format( - fidelity_cell.format(fidelity), + budget_cell.format(budget), cell.format(pending), cell.format(waiting), cell.format(done) diff --git a/src/dehb/utils/config_repository.py b/src/dehb/utils/config_repository.py deleted file mode 100644 index 126b28f..0000000 --- a/src/dehb/utils/config_repository.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -import numpy as np - - -@dataclass -class ConfigItem: - """Data class to store information regarding a specific configuration. - - The results for this configuration are stored in the `results` dict, using the fidelity it has - been evaluated on as keys. - """ - config_id: int - config: np.ndarray - results: dict[float, ResultItem] - -@dataclass -class ResultItem: - """Data class storing the result information of a specific configuration + fidelity.""" - score: float - cost: float - info: dict[Any, Any] - -class ConfigRepository: - """Bookkeeps all configurations used throughout the course of the optimization. - - Keeps track of the configurations and their results on the different fidelitites. - A new configuration is announced via `announce_config`. After evaluating the configuration - on the specified fidelity, use `tell_result` to log the achieved performance, cost etc. - - The configurations are stored in a list of `ConfigItem`. - """ - def __init__(self) -> None: - """Initializes the class by calling `self.reset`.""" - self.configs : list[ConfigItem] - self.reset() - - def reset(self) -> None: - """Resets the config repository, clearing all collected configurations and results.""" - self.configs = [] - - def announce_config(self, config: np.ndarray, fidelity=None) -> int: - """Announces a new configuration with the respective fidelity it should be evaluated on. - - The configuration is then added to the list of so far seen configurations and the ID of the - configuration is returned. - - Args: - config (np.ndarray): New configuration - fidelity (float, optional): Fidelity on which `config` is evaluated or None. - Defaults to None. - - Returns: - int: ID of configuration - """ - config_id = len(self.configs) - fidelity = float(fidelity or 0) - result_dict = { - fidelity: ResultItem(np.inf, -1, {}), - } - config_item = ConfigItem(config_id, config, result_dict) - self.configs.append(config_item) - return config_id - - def announce_population(self, population: np.ndarray, fidelity=None) -> np.ndarray: - """Announce population, retrieving ids for the population. - - Args: - population (np.ndarray): Population to announce - fidelity (float, optional): Fidelity on which pop is evaluated or None. - Defaults to None. - - Returns: - np.ndarray: population ids - """ - population_ids = [] - for indiv in population: - conf_id = self.announce_config(indiv, float(fidelity or 0)) - population_ids.append(conf_id) - return np.array(population_ids) - - def announce_fidelity(self, config_id: int, fidelity: float) -> bool: - """Announce the evaluation of a new fidelity for a given config. - - This function may only be used if the config already exists in the repository. - - Args: - config_id (int): ID of Configuration - fidelity (float): Fidelity on which the config will be evaluated - - Returns: - bool: Success/Failure of operation - """ - if config_id >= len(self.configs) or config_id < 0: - # TODO: Error message - return False - - config_item = self.configs[config_id] - result_item = { - fidelity: ResultItem(np.inf, -1, {}), - } - config_item.results[fidelity] = result_item - return True - - def tell_result(self, config_id: int, fidelity: float, score: float, cost: float, info: dict): - """Logs the achieved performance, cost etc. of a specific configuration-fidelity pair. - - Args: - config_id (int): ID of evaluated configuration - fidelity (float): Fidelity on which configuration has been evaluated. - score (float): Achieved score, given by objective function - cost (float): Cost, given by objective function - info (dict): Run info, given by objective function - """ - config_item = self.configs[config_id] - - # If configuration has been promoted, there is no fidelity information yet - if fidelity not in config_item.results: - config_item.results[fidelity] = ResultItem(score, cost, info) - else: - # ResultItem already given for specified fidelity --> update entries - config_item.results[fidelity].score = score - config_item.results[fidelity].cost = cost - config_item.results[fidelity].info = info \ No newline at end of file diff --git a/tests/test_config_repository.py b/tests/test_config_repository.py deleted file mode 100644 index f63e870..0000000 --- a/tests/test_config_repository.py +++ /dev/null @@ -1,34 +0,0 @@ -import typing - -import numpy as np -from src.dehb.utils import ConfigRepository - - -class TestConfigAnnouncing(): - """Class that bundles all tests for announcing configurations to the repository.""" - def test_single_config(self): - """Tests announcing single config.""" - repo = ConfigRepository() - config = np.array([0.5]) - - config_id = repo.announce_config(config, 2) - - assert len(repo.configs) == 1 - assert config_id == 0 - assert repo.configs[config_id].config == config - - def test_population(self): - """Tests announcing a whole population.""" - repo = ConfigRepository() - pop = [] - for i in range(10): - config = np.array([i / 10]) - pop.append(config) - pop = np.array(pop) - - config_ids = repo.announce_population(pop) - - assert len(repo.configs) == 10 - - for conf_id in config_ids: - assert repo.configs[conf_id].config == pop[conf_id] \ No newline at end of file diff --git a/tests/test_de.py b/tests/test_de.py index 50099b2..f64457b 100644 --- a/tests/test_de.py +++ b/tests/test_de.py @@ -15,7 +15,7 @@ def create_toy_DEBase(configspace: ConfigSpace.ConfigurationSpace): """ dim = len(configspace.get_hyperparameters()) return DEBase(f=lambda: 1, cs=configspace, dimensions=dim, pop_size=10, max_age=5, - mutation_factor=0.5, crossover_prob=0.5, strategy="rand1_bin", fidelity=1) + mutation_factor=0.5, crossover_prob=0.5, strategy="rand1_bin", budget=1) class TestConversion(): """Class that bundles all ConfigSpace/vector conversion tests. diff --git a/tests/test_dehb.py b/tests/test_dehb.py index 277e959..8a96e03 100644 --- a/tests/test_dehb.py +++ b/tests/test_dehb.py @@ -22,15 +22,15 @@ def create_toy_searchspace(): ConfigSpace.UniformFloatHyperparameter("x0", lower=3, upper=10, log=False)) return cs -def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_fidelity: float, - max_fidelity: float, eta: int, +def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_budget: float, + max_budget: float, eta: int, objective_function: typing.Callable): """Creates a DEHB instance. Args: configspace (ConfigurationSpace): Searchspace to use - min_fidelity (float): Minimum fidelity for DEHB - max_fidelity (float): Maximum fidelity for DEHB + min_budget (float): Minimum budget for DEHB + max_budget (float): Maximum budget for DEHB eta (int): Eta parameter of DEHB objective_function (Callable): Function to optimize @@ -39,16 +39,16 @@ def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_fideli """ dim = len(configspace.get_hyperparameters()) return DEHB(f=objective_function, cs=configspace, dimensions=dim, - min_fidelity=min_fidelity, - max_fidelity=max_fidelity, eta=eta, n_workers=1) + min_budget=min_budget, + max_budget=max_budget, eta=eta, n_workers=1) -def objective_function(x: ConfigSpace.Configuration, fidelity: float, **kwargs): +def objective_function(x: ConfigSpace.Configuration, budget: float, **kwargs): """Toy objective function. Args: x (ConfigSpace.Configuration): Configuration to evaluate - fidelity (float): fidelity to evaluate x on + budget (float): Budget to evaluate x on Returns: dict: Result dictionary @@ -70,7 +70,7 @@ class TestBudgetExhaustion(): def test_runtime_exhaustion(self): """Test for runtime budget exhaustion.""" cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, objective_function=objective_function) dehb.start = time.time() - 10 @@ -80,7 +80,7 @@ def test_runtime_exhaustion(self): def test_fevals_exhaustion(self): """Test for function evaluations budget exhaustion.""" cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, objective_function=objective_function) dehb.traj.append("Just needed for the test") @@ -90,7 +90,7 @@ def test_fevals_exhaustion(self): def test_brackets_exhaustion(self): """Test for bracket budget exhaustion.""" cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, objective_function=objective_function) dehb.iteration_counter = 5 @@ -99,52 +99,16 @@ def test_brackets_exhaustion(self): class TestInitialization: """Class that bundles all tests regarding the initialization of DEHB.""" - def test_higher_min_fidelity(self): - """Test that verifies, that DEHB breaks if min_fidelity > max_fidelity.""" + def test_higher_min_budget(self): + """Test that verifies, that DEHB breaks if min_budget > max_budget.""" cs = create_toy_searchspace() with pytest.raises(AssertionError): - create_toy_optimizer(configspace=cs, min_fidelity=28, max_fidelity=27, eta=3, + create_toy_optimizer(configspace=cs, min_budget=28, max_budget=27, eta=3, objective_function=objective_function) - def test_equal_min_max_fidelity(self): - """Test that verifies, that DEHB breaks if min_fidelity == max_fidelity.""" + def test_equal_min_max_budget(self): + """Test that verifies, that DEHB breaks if min_budget == max_budget.""" cs = create_toy_searchspace() with pytest.raises(AssertionError): - create_toy_optimizer(configspace=cs, min_fidelity=27, max_fidelity=27, eta=3, + create_toy_optimizer(configspace=cs, min_budget=27, max_budget=27, eta=3, objective_function=objective_function) - -class TestConfigID: - """Class that bundles all tests regarding config ID functionality.""" - def test_initialization(self): - """Verifies, that the initial population is properly tracked by the config repository.""" - cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, - objective_function=objective_function) - # calculate how many configurations have been sampled for the initial populations - num_configs = 0 - for de_inst in dehb.de.values(): - num_configs += len(de_inst.population) - - # config repository should be exactly this long - assert len(dehb.config_repository.configs) == num_configs - - def test_single_bracket(self): - """Verifies, that the population is continously tracked over the run of a single bracket.""" - cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, - objective_function=objective_function) - # calculate how many configurations have been sampled for the initial populations - num_initial_configs = 0 - for de_inst in dehb.de.values(): - num_initial_configs += len(de_inst.population) - - # run for a single bracket - dehb.run(brackets=1, verbose=True) - - # for the first bracket, we only mutate on the lowest fidelity and then promote the best - # configs to the next fidelity. Please note, that this is only the case for the first - # DEHB bracket! - # Note: The final + 1 is due to the inner workings of DEHB. If the run budget is exhausted, - # we keep evolving new configurations without evaluating them, since we are only waiting to - # to fetch all results started ahead of the budget exhaustion. - assert len(dehb.config_repository.configs) == num_initial_configs + 9 + 1 \ No newline at end of file diff --git a/utils/README.md b/utils/README.md index 5297e17..65f4058 100644 --- a/utils/README.md +++ b/utils/README.md @@ -36,8 +36,8 @@ For example, running a DEHB optimization by specifiying `scheduler_file` makes t connect to the Dask cluster runnning. ```bash python examples/03_pytorch_mnist_hpo.py \ - --min_fidelity 1 \ - --max_fidelity 9 \ + --min_budget 1 \ + --max_budget 9 \ --runtime 200 \ --seed 123 \ --scheduler_file scheduler/scheduler_gpu.json \ From c97acaee61a41551a81c47f920de21adf4061ae1 Mon Sep 17 00:00:00 2001 From: Janis Fix <56302972+Bronzila@users.noreply.github.com> Date: Tue, 6 Feb 2024 12:51:34 +0100 Subject: [PATCH 5/9] Add arXiv badge to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 604fa6d..c91f036 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![Coverage Status](https://coveralls.io/repos/github/automl/DEHB/badge.svg)](https://coveralls.io/github/automl/DEHB) [![PyPI](https://img.shields.io/pypi/v/dehb)](https://pypi.org/project/dehb/) [![Static Badge](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20-blue)](https://pypi.org/project/dehb/) +[![arXiv](https://img.shields.io/badge/arXiv-2105.09821-b31b1b.svg)](https://arxiv.org/abs/2105.09821) ### Installation ```bash # from pypi From 4ac852ef883db340404e6e617115f33c422709d8 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Sat, 17 Feb 2024 15:45:35 +0100 Subject: [PATCH 6/9] Revert "Revert "Merge branch 'development' into master"" This reverts commit 5f71db9a3c661a766f51abce80659ec87f0656a6. --- README.md | 18 +- examples/00_interfacing_DEHB.ipynb | 90 +++--- ...1_Optimizing_RandomForest_using_DEHB.ipynb | 129 ++++---- .../02_using DEHB_without_ConfigSpace.ipynb | 42 ++- examples/03_pytorch_mnist_hpo.py | 30 +- src/dehb/optimizers/de.py | 142 ++++++--- src/dehb/optimizers/dehb.py | 291 ++++++++++-------- src/dehb/utils/__init__.py | 3 +- src/dehb/utils/bracket_manager.py | 104 +++---- src/dehb/utils/config_repository.py | 127 ++++++++ tests/test_config_repository.py | 34 ++ tests/test_de.py | 2 +- tests/test_dehb.py | 70 ++++- utils/README.md | 4 +- 14 files changed, 678 insertions(+), 408 deletions(-) create mode 100644 src/dehb/utils/config_repository.py create mode 100644 tests/test_config_repository.py diff --git a/README.md b/README.md index c91f036..8d07545 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ pip install -e DEHB # -e stands for editable, lets you modify the code and reru To run PyTorch example: (*note additional requirements*) ```bash python examples/03_pytorch_mnist_hpo.py \ - --min_budget 1 \ - --max_budget 3 \ + --min_fidelity 1 \ + --max_fidelity 3 \ --runtime 60 \ --verbose ``` @@ -63,8 +63,8 @@ to it by that DEHB run. To run the PyTorch MNIST example on a single node using 2 workers: ```bash python examples/03_pytorch_mnist_hpo.py \ - --min_budget 1 \ - --max_budget 3 \ + --min_fidelity 1 \ + --max_fidelity 3 \ --runtime 60 \ --n_workers 2 \ --single_node_with_gpus \ @@ -97,8 +97,8 @@ bash utils/run_dask_setup.sh \ # Make sure to sleep to allow the workers to setup properly sleep 5 python examples/03_pytorch_mnist_hpo.py \ - --min_budget 1 \ - --max_budget 3 \ + --min_fidelity 1 \ + --max_fidelity 3 \ --runtime 60 \ --scheduler_file dask_dump/scheduler.json \ --verbose @@ -112,9 +112,9 @@ and were found to be *generally* useful across all cases tested. However, the parameters are still available for tuning to a specific problem. The Hyperband components: -* *min\_budget*: Needs to be specified for every DEHB instantiation and is used in determining -the budget spacing for the problem at hand. -* *max\_budget*: Needs to be specified for every DEHB instantiation. Represents the full-budget +* *min\_fidelity*: Needs to be specified for every DEHB instantiation and is used in determining +the fidelity spacing for the problem at hand. +* *max\_fidelity*: Needs to be specified for every DEHB instantiation. Represents the full-fidelity evaluation or the actual black-box setting. * *eta*: (default=3) Sets the aggressiveness of Hyperband's aggressive early stopping by retaining 1/eta configurations every round diff --git a/examples/00_interfacing_DEHB.ipynb b/examples/00_interfacing_DEHB.ipynb index 350087c..807b124 100644 --- a/examples/00_interfacing_DEHB.ipynb +++ b/examples/00_interfacing_DEHB.ipynb @@ -40,7 +40,7 @@ "\n", "DEHB also uses Hyperband along with DE, to allow for cheaper approximations of the actual evaluations of $x$. Let $f(x)$ be the validation error of training a multilayer perceptron (MLP) on the complete training set. Multi-fidelity algorithms such as Hyperband, allow for cheaper approximations along a possible *fidelity*. For the MLP, a subset of the dataset maybe a cheaper approximation to the full data set evaluation. Whereas the fidelity can be quantifies as the fraction of the dataset used to evaluate the configuration $x$, instead of the full dataset. Such approximations can allow sneak-peek into the black-box, potentially revealing certain landscape feature of *f(x)*, thus rendering it a *gray*-box and not completely opaque and black! \n", "\n", - "The $z$ parameter is the fidelity parameter to the black-box function. If $z \\in [budget_{min}, budget_{max}]$, then $f(x, budget_{max})$ would be equivalent to the black-box case of $f(x)$.\n", + "The $z$ parameter is the fidelity parameter to the black-box function. If $z \\in [fidelity_{min}, fidelity_{max}]$, then $f(x, fidelity_{max})$ would be equivalent to the black-box case of $f(x)$.\n", "\n", "![boxes](imgs/black-gray-box.png)" ] @@ -62,7 +62,7 @@ "source": [ "def target_function(\n", " x: Union[ConfigSpace.Configuration, List, np.array], \n", - " budget: Union[int, float] = None,\n", + " fidelity: Union[int, float] = None,\n", " **kwargs\n", ") -> Dict:\n", " \"\"\" Target/objective function to optimize\n", @@ -70,7 +70,7 @@ " Parameters\n", " ----------\n", " x : configuration that DEHB wants to evaluate\n", - " budget : parameter determining cheaper evaluations\n", + " fidelity : parameter determining cheaper evaluations\n", " \n", " Returns\n", " -------\n", @@ -83,7 +83,7 @@ " # remove the code snippet below\n", " start = time.time()\n", " y = np.random.uniform() # placeholder response of evaluation\n", - " time.sleep(budget) # simulates runtime (mostly proportional to fidelity)\n", + " time.sleep(fidelity) # simulates runtime (mostly proportional to fidelity)\n", " cost = time.time() - start\n", " \n", " # result dict passed to DE/DEHB as function evaluation output\n", @@ -171,8 +171,9 @@ { "data": { "text/plain": [ - "Configuration:\n", - " x0, Value: 3.716302229868112" + "Configuration(values={\n", + " 'x0': 8.107160631154175,\n", + "})" ] }, "execution_count": 5, @@ -198,7 +199,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Defining fidelity/budget range for the target function" + "### Defining fidelity range for the target function" ] }, { @@ -207,7 +208,7 @@ "metadata": {}, "outputs": [], "source": [ - "min_budget, max_budget = (0.1, 3) " + "min_fidelity, max_fidelity = (0.1, 3) " ] }, { @@ -244,8 +245,8 @@ " f=target_function,\n", " dimensions=dimensions,\n", " cs=cs,\n", - " min_budget=min_budget,\n", - " max_budget=max_budget,\n", + " min_fidelity=min_fidelity,\n", + " max_fidelity=max_fidelity,\n", " output_path=\"./temp\",\n", " n_workers=1 # set to >1 to utilize parallel workers\n", ")\n", @@ -281,9 +282,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Configuration:\n", - " x0, Value: 4.060258498267547\n", - "\n" + "Configuration(values={\n", + " 'x0': 4.152073449922892,\n", + "})\n" ] } ], @@ -308,14 +309,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "2021-10-22 14:45:56.117 | INFO | dehb.optimizers.dehb:reset:107 - \n", - "\n", - "RESET at 10/22/21 14:45:56 CEST\n", + "\u001b[32m2023-10-22 20:03:06.057\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", "\n", + "RESET at 10/22/23 20:03:06 CEST\n", "\n", - "(Configuration:\n", - " x0, Value: 3.724555206841792\n", - ", 0.0938589687572785)\n" + "\u001b[0m\n", + "(Configuration(values={\n", + " 'x0': 8.96840263375364,\n", + "}), 0.05819975786653586)\n" ] } ], @@ -343,14 +344,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "2021-10-22 14:45:58.567 | INFO | dehb.optimizers.dehb:reset:107 - \n", - "\n", - "RESET at 10/22/21 14:45:58 CEST\n", + "\u001b[32m2023-10-22 20:03:11.073\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", "\n", + "RESET at 10/22/23 20:03:11 CEST\n", "\n", - "(Configuration:\n", - " x0, Value: 4.341818535733585\n", - ", 3.653636256717441e-05)\n" + "\u001b[0m\n", + "(Configuration(values={\n", + " 'x0': 8.708444163420975,\n", + "}), 0.0710929937087792)\n" ] } ], @@ -381,9 +382,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "(Configuration:\n", - " x0, Value: 4.610766436763522\n", - ", 0.007774399252232556)\n" + "(Configuration(values={\n", + " 'x0': 8.454086817115218,\n", + "}), 0.016305791635409683)\n" ] } ], @@ -392,8 +393,8 @@ " f=target_function,\n", " dimensions=dimensions,\n", " cs=cs,\n", - " min_budget=min_budget,\n", - " max_budget=max_budget,\n", + " min_fidelity=min_fidelity,\n", + " max_fidelity=max_fidelity,\n", " output_path=\"./temp\",\n", " n_workers=2\n", ")\n", @@ -413,9 +414,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Configuration:\n", - " x0, Value: 4.610766436763522\n", - "\n" + "Configuration(values={\n", + " 'x0': 8.454086817115218,\n", + "})\n" ] } ], @@ -432,10 +433,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "0.007774399252232556 0.007774399252232556\n", - "Configuration:\n", - " x0, Value: 4.610766436763522\n", - "\n" + "0.016305791635409683 0.016305791635409683\n", + "Configuration(values={\n", + " 'x0': 8.454086817115218,\n", + "})\n" ] } ], @@ -454,7 +455,7 @@ "\n", "As detailed above, the problem definition needs to be input to DEHB as the following information:\n", "* the *target_function* (`f`) that is the primary black-box function to optimize\n", - "* the fidelity range of `min_budget` and `max_budget` that allows the cheaper, faster gray-box optimization of `f`\n", + "* the fidelity range of `min_fidelity` and `max_fidelity` that allows the cheaper, faster gray-box optimization of `f`\n", "* the search space or the input domain of the function `f`, that can be represented as a `ConfigSpace` object and passed to DEHB at initialization\n", "\n", "\n", @@ -465,13 +466,20 @@ "\n", "DEHB will terminate once its chosen runtime budget is exhausted, and report the incumbent found. DEHB, as an *anytime* algorithm, constantly writes to disk a lightweight `json` file with the best found configuration and its score seen till that point." ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "dask", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "dask" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -483,7 +491,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.9" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/examples/01_Optimizing_RandomForest_using_DEHB.ipynb b/examples/01_Optimizing_RandomForest_using_DEHB.ipynb index c35427b..e5bd359 100644 --- a/examples/01_Optimizing_RandomForest_using_DEHB.ipynb +++ b/examples/01_Optimizing_RandomForest_using_DEHB.ipynb @@ -37,7 +37,7 @@ "* `min_samples_split`\n", "* `max_features`\n", "* `min_samples_leaf`\n", - "while the `n_estimators` hyperparameter to the Random Forest is chosen to be a fidelity parameter instead. Lesser number of trees ($<10$) in the Random Forest may not allow adequate ensembling for the grouped prediction to be significantly better than the individual tree predictions. Whereas a large number of trees (~$100$) often give accurate predictions but is naturally slower to train and predict on account of more trees to train. Therefore, a smaller `n_estimators` can be used as a cheaper approximation of the actual budget of `n_estimators=100`." + "while the `n_estimators` hyperparameter to the Random Forest is chosen to be a fidelity parameter instead. Lesser number of trees ($<10$) in the Random Forest may not allow adequate ensembling for the grouped prediction to be significantly better than the individual tree predictions. Whereas a large number of trees (~$100$) often give accurate predictions but is naturally slower to train and predict on account of more trees to train. Therefore, a smaller `n_estimators` can be used as a cheaper approximation of the actual fidelity of `n_estimators=100`." ] }, { @@ -53,7 +53,7 @@ "metadata": {}, "outputs": [], "source": [ - "min_budget, max_budget = 2, 50" + "min_fidelity, max_fidelity = 2, 50" ] }, { @@ -147,7 +147,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now the primary black/gray-box interface to the Random Forest model needs to be built for DEHB to query. As given in the `00_interfacing_DEHB` notebook, this function will have a signature akin to: `target_function(config, budget)`, and return a two-element tuple of the `score` and `cost`. It must be noted that DEHB **minimizes** and therefore the `score` being returned by this `target_function` should account for it." + "Now the primary black/gray-box interface to the Random Forest model needs to be built for DEHB to query. As given in the `00_interfacing_DEHB` notebook, this function will have a signature akin to: `target_function(config, fidelity)`, and return a two-element tuple of the `score` and `cost`. It must be noted that DEHB **minimizes** and therefore the `score` being returned by this `target_function` should account for it." ] }, { @@ -273,23 +273,23 @@ "metadata": {}, "outputs": [], "source": [ - "def target_function(config, budget, **kwargs):\n", + "def target_function(config, fidelity, **kwargs):\n", " # Extracting support information\n", " seed = kwargs[\"seed\"]\n", " train_X = kwargs[\"train_X\"]\n", " train_y = kwargs[\"train_y\"]\n", " valid_X = kwargs[\"valid_X\"]\n", " valid_y = kwargs[\"valid_y\"]\n", - " max_budget = kwargs[\"max_budget\"]\n", + " max_fidelity = kwargs[\"max_fidelity\"]\n", " \n", - " if budget is None:\n", - " budget = max_budget\n", + " if fidelity is None:\n", + " fidelity = max_fidelity\n", " \n", " start = time.time()\n", " # Building model \n", " model = RandomForestClassifier(\n", " **config.get_dictionary(),\n", - " n_estimators=int(budget),\n", + " n_estimators=int(fidelity),\n", " bootstrap=True,\n", " random_state=seed,\n", " )\n", @@ -308,7 +308,7 @@ " \"cost\": cost,\n", " \"info\": {\n", " \"test_score\": test_accuracy,\n", - " \"budget\": budget\n", + " \"fidelity\": fidelity\n", " }\n", " }\n", " return result" @@ -340,8 +340,8 @@ " f=target_function, \n", " cs=cs, \n", " dimensions=dimensions, \n", - " min_budget=min_budget, \n", - " max_budget=max_budget,\n", + " min_fidelity=min_fidelity, \n", + " max_fidelity=max_fidelity,\n", " n_workers=1,\n", " output_path=\"./temp\"\n", ")" @@ -363,7 +363,7 @@ " train_y=train_y,\n", " valid_X=valid_X,\n", " valid_y=valid_y,\n", - " max_budget=dehb.max_budget\n", + " max_fidelity=dehb.max_fidelity\n", ")" ] }, @@ -376,17 +376,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "473 473 473\n", + "454 454 454\n", "\n", "Last evaluated configuration, \n", "Configuration(values={\n", - " 'max_depth': 7,\n", - " 'max_features': 0.669059250229961,\n", - " 'min_samples_leaf': 2,\n", - " 'min_samples_split': 3,\n", - "})\n", - "got a score of -1.0, was evaluated at a budget of 50.00 and took 0.048 seconds to run.\n", - "The additional info attached: {'test_score': 1.0, 'budget': 50.0}\n" + " 'max_depth': 6,\n", + " 'max_features': 0.6215565437234547,\n", + " 'min_samples_leaf': 8,\n", + " 'min_samples_split': 4,\n", + "})got a score of -1.0, was evaluated at a fidelity of 16.67 and took 0.016 seconds to run.\n", + "The additional info attached: {'test_score': 1.0, 'fidelity': 16.666666666666664}\n" ] } ], @@ -395,12 +394,12 @@ "\n", "# Last recorded function evaluation\n", "last_eval = history[-1]\n", - "config, score, cost, budget, _info = last_eval\n", + "config, score, cost, fidelity, _info = last_eval\n", "\n", "print(\"Last evaluated configuration, \")\n", "print(dehb.vector_to_configspace(config), end=\"\")\n", - "print(\"got a score of {}, was evaluated at a budget of {:.2f} and \"\n", - " \"took {:.3f} seconds to run.\".format(score, budget, cost))\n", + "print(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"\n", + " \"took {:.3f} seconds to run.\".format(score, fidelity, cost))\n", "print(\"The additional info attached: {}\".format(_info))" ] }, @@ -420,29 +419,29 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m2023-06-22 12:00:41.016\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m107\u001b[0m - \u001b[1m\n", + "\u001b[32m2023-10-22 20:04:30.731\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", "\n", - "RESET at 06/22/23 12:00:40 CEST\n", + "RESET at 10/22/23 20:04:30 CEST\n", "\n", "\u001b[0m\n", - "\u001b[32m2023-06-22 12:00:51.085\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m107\u001b[0m - \u001b[1m\n", + "\u001b[32m2023-10-22 20:04:41.051\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", "\n", - "RESET at 06/22/23 12:00:51 CEST\n", + "RESET at 10/22/23 20:04:41 CEST\n", "\n", "\u001b[0m\n", - "\u001b[32m2023-06-22 12:01:01.182\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m107\u001b[0m - \u001b[1m\n", + "\u001b[32m2023-10-22 20:04:51.128\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", "\n", - "RESET at 06/22/23 12:01:01 CEST\n", + "RESET at 10/22/23 20:04:51 CEST\n", "\n", "\u001b[0m\n", - "\u001b[32m2023-06-22 12:01:11.238\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m107\u001b[0m - \u001b[1m\n", + "\u001b[32m2023-10-22 20:05:01.200\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", "\n", - "RESET at 06/22/23 12:01:11 CEST\n", + "RESET at 10/22/23 20:05:01 CEST\n", "\n", "\u001b[0m\n", - "\u001b[32m2023-06-22 12:01:21.293\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m107\u001b[0m - \u001b[1m\n", + "\u001b[32m2023-10-22 20:05:11.273\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mdehb.optimizers.dehb\u001b[0m:\u001b[36mreset\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1m\n", "\n", - "RESET at 06/22/23 12:01:21 CEST\n", + "RESET at 10/22/23 20:05:11 CEST\n", "\n", "\u001b[0m\n" ] @@ -466,14 +465,14 @@ " train_y=train_y,\n", " valid_X=valid_X,\n", " valid_y=valid_y,\n", - " max_budget=dehb.max_budget\n", + " max_fidelity=dehb.max_fidelity\n", " )\n", " best_config = dehb.vector_to_configspace(dehb.inc_config)\n", " \n", " # Creating a model using the best configuration found\n", " model = RandomForestClassifier(\n", " **best_config.get_dictionary(),\n", - " n_estimators=int(max_budget),\n", + " n_estimators=int(max_fidelity),\n", " bootstrap=True,\n", " random_state=seed,\n", " )\n", @@ -516,44 +515,39 @@ "output_type": "stream", "text": [ "Configuration(values={\n", - " 'max_depth': 13,\n", - " 'max_features': 0.5412753369058052,\n", - " 'min_samples_leaf': 12,\n", - " 'min_samples_split': 14,\n", - "})\n", - " got an accuracy of 1.0 on the test set.\n", - "\n", - "Configuration(values={\n", - " 'max_depth': 6,\n", - " 'max_features': 0.6764411582074702,\n", + " 'max_depth': 7,\n", + " 'max_features': 0.7162350418245509,\n", " 'min_samples_leaf': 1,\n", - " 'min_samples_split': 27,\n", - "})\n", - " got an accuracy of 1.0 on the test set.\n", + " 'min_samples_split': 18,\n", + "}) got an accuracy of 1.0 on the test set.\n", "\n", "Configuration(values={\n", - " 'max_depth': 5,\n", - " 'max_features': 0.5862915814751853,\n", + " 'max_depth': 11,\n", + " 'max_features': 0.564056444856198,\n", " 'min_samples_leaf': 2,\n", - " 'min_samples_split': 22,\n", - "})\n", - " got an accuracy of 1.0 on the test set.\n", + " 'min_samples_split': 7,\n", + "}) got an accuracy of 1.0 on the test set.\n", "\n", "Configuration(values={\n", - " 'max_depth': 14,\n", - " 'max_features': 0.5346143393392929,\n", - " 'min_samples_leaf': 5,\n", - " 'min_samples_split': 9,\n", - "})\n", - " got an accuracy of 1.0 on the test set.\n", + " 'max_depth': 9,\n", + " 'max_features': 0.7477652209361112,\n", + " 'min_samples_leaf': 1,\n", + " 'min_samples_split': 7,\n", + "}) got an accuracy of 1.0 on the test set.\n", "\n", "Configuration(values={\n", - " 'max_depth': 4,\n", - " 'max_features': 0.5541455312635835,\n", + " 'max_depth': 9,\n", + " 'max_features': 0.6510861760309854,\n", " 'min_samples_leaf': 4,\n", - " 'min_samples_split': 10,\n", - "})\n", - " got an accuracy of 1.0 on the test set.\n", + " 'min_samples_split': 24,\n", + "}) got an accuracy of 1.0 on the test set.\n", + "\n", + "Configuration(values={\n", + " 'max_depth': 6,\n", + " 'max_features': 0.5989756936409275,\n", + " 'min_samples_leaf': 2,\n", + " 'min_samples_split': 4,\n", + "}) got an accuracy of 1.0 on the test set.\n", "\n" ] } @@ -563,6 +557,13 @@ " print(\"{} got an accuracy of {} on the test set.\".format(config, score))\n", " print()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/examples/02_using DEHB_without_ConfigSpace.ipynb b/examples/02_using DEHB_without_ConfigSpace.ipynb index 11264c6..79c987d 100644 --- a/examples/02_using DEHB_without_ConfigSpace.ipynb +++ b/examples/02_using DEHB_without_ConfigSpace.ipynb @@ -61,7 +61,7 @@ "dimensions = len(param_space)\n", "\n", "# Declaring the fidelity range\n", - "min_budget, max_budget = 2, 50\n", + "min_fidelity, max_fidelity = 2, 50\n", "\n", "\n", "def transform_space(param_space, configuration):\n", @@ -164,27 +164,27 @@ " return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset\n", "\n", "\n", - "def target_function(config, budget, **kwargs):\n", + "def target_function(config, fidelity, **kwargs):\n", " # Extracting support information\n", " seed = kwargs[\"seed\"]\n", " train_X = kwargs[\"train_X\"]\n", " train_y = kwargs[\"train_y\"]\n", " valid_X = kwargs[\"valid_X\"]\n", " valid_y = kwargs[\"valid_y\"]\n", - " max_budget = kwargs[\"max_budget\"]\n", + " max_fidelity = kwargs[\"max_fidelity\"]\n", " \n", " # Mapping [0, 1]-vector to Sklearn parameters\n", " param_space = kwargs[\"param_space\"]\n", " config = transform_space(param_space, config)\n", " \n", - " if budget is None:\n", - " budget = max_budget\n", + " if fidelity is None:\n", + " fidelity = max_fidelity\n", " \n", " start = time.time()\n", " # Building model \n", " model = RandomForestClassifier(\n", " **config,\n", - " n_estimators=int(budget),\n", + " n_estimators=int(fidelity),\n", " bootstrap=True,\n", " random_state=seed,\n", " )\n", @@ -203,7 +203,7 @@ " \"cost\": cost,\n", " \"info\": {\n", " \"test_score\": test_accuracy,\n", - " \"budget\": budget\n", + " \"fidelity\": fidelity\n", " }\n", " }\n", " return result\n", @@ -238,8 +238,8 @@ "dehb = DEHB(\n", " f=target_function, \n", " dimensions=dimensions, \n", - " min_budget=min_budget, \n", - " max_budget=max_budget,\n", + " min_fidelity=min_fidelity, \n", + " max_fidelity=max_fidelity,\n", " n_workers=1,\n", " output_path=\"./temp\"\n", ")" @@ -260,7 +260,7 @@ " train_y=train_y,\n", " valid_X=valid_X,\n", " valid_y=valid_y,\n", - " max_budget=dehb.max_budget,\n", + " max_fidelity=dehb.max_fidelity,\n", " param_space=param_space\n", ")" ] @@ -274,9 +274,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Incumbent score: -0.9685185185185186\n", + "Incumbent score: -0.9611111111111111\n", "Incumbent configuration:\n", - "{'max_depth': 10, 'min_samples_split': 3, 'max_features': 0.24012458257841524, 'min_samples_leaf': 2}\n" + "{'max_depth': 9, 'min_samples_split': 3, 'max_features': 0.3990411414400532, 'min_samples_leaf': 1}\n" ] } ], @@ -301,14 +301,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Test accuracy: 1.0\n" + "Test accuracy: 0.9944444444444445\n" ] } ], "source": [ "model = RandomForestClassifier(\n", " **transform_space(param_space, dehb.inc_config),\n", - " n_estimators=int(max_budget),\n", + " n_estimators=int(max_fidelity),\n", " bootstrap=True,\n", " random_state=seed,\n", ")\n", @@ -334,14 +334,12 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhk0lEQVR4nO3de5ycZX338c93ZzOb3U3YYBIQs0DABGggQSEgchJBFB4rqAUlqBy0Yn1AsEoVqaXI82qrPi0oh9oioghVVASNNpXGgKCikHDKAQRSTkkIJAFCQhKS7O6vf9z3JsOwh5mQe+7Zne/79corc59mfhPIfnNd131flyICMzOzSjXlXYCZmQ0tDg4zM6uKg8PMzKri4DAzs6o4OMzMrCrNeRewvYwbNy4mTpyYdxlmZkPKvffeuyoixldzzbAJjokTJzJv3ry8yzAzG1IkPVXtNe6qMjOzqjg4zMysKpkGh6TjJD0iabGkC/o4fqSk+yR1STqpZP9bJP1B0iJJ8yV9OMs6zcyscpkFh6QCcBVwPDAFmCFpStlpTwNnAD8o278eOC0i9gWOA74haUxWtZqZWeWyHBw/GFgcEY8DSLoROBF4qPeEiHgyPdZTemFEPFry+hlJK4DxwOoM6zUzswpk2VU1AVhSsr003VcVSQcDReB/+jh2lqR5kuatXLlymws1M7PK1fXguKRdgOuBMyOip/x4RFwdEdMjYvr48VXdhmxmZtsoy66qZcCuJdud6b6KSNoB+E/gbyPij9u5ti3Wb+ri337zmsYMI4sFzjh0Im3FYfOoi5nZdpHlT8W5wGRJe5AExinAqZVcKKkI3AJ8PyJuyq5E2LCpmytuX/yqfb1LlEzeaTTHTtk5y483MxtyMguOiOiSdA5wK1AAro2IRZIuAeZFxExJB5EExI7A+yR9Jb2T6kPAkcBYSWekb3lGRDywvescO6qFJ/7pva/a98Sqdbzzn3/Dyxs3b++PMzMb8jLth4mIWcCssn0XlbyeS9KFVX7dDcANWdY2kPZiAYCXN3bnVYKZWd2q68HxvLS3JHm6fmNXzpWYmdUfB0cfWkckLY51m9ziMDMr5+DoQ1OTaCsW3OIwM+uDg6MfbcVmtzjMzPrg4OjHqJYC69ziMDN7DQdHP9qKzazf5OAwMyvn4OhHe0uBdb4d18zsNRwc/XCLw8ysbw6OfrS3FDw4bmbWBwdHP9qLzR4cNzPrg4OjH+0tDg4zs744OPrRViywflM30TtVrpmZAQ6OfrW3NNPVE2zqfs36UWZmDc3B0Y+2dIZc35JrZvZqDo5+9M6Q63EOM7NXc3D0oz1dMna9b8k1M3sVB0c/2lp6p1Z3i8PMrJSDox9bWhwe4zAzexUHRz/aW3qXj3WLw8yslIOjH1vHOBwcZmalHBz92DrG4a4qM7NSDo5+bB3jcIvDzKyUg6MfrSPc4jAz64uDox9NTaK96OVjzczKOTgG0NbixZzMzMo5OAaQtDjcVWVmVsrBMQAvH2tm9loOjgG0t7jFYWZWzsExgPaWZs9VZWZWxsExAK87bmb2Wg6OAfQuH2tmZls5OAbQ3uIWh5lZOQfHANpbCqzb1E1E5F2KmVndcHAMoK3YTHdPsLGrJ+9SzMzqhoNjAO3FZL4qj3OYmW3l4BhAW0syQ67HOczMtso0OCQdJ+kRSYslXdDH8SMl3SepS9JJZcdOl/RY+uv0LOvsz9bFnNziMDPrlVlwSCoAVwHHA1OAGZKmlJ32NHAG8IOya98A/D3wNuBg4O8l7ZhVrf3x8rFmZq+VZYvjYGBxRDweEZuAG4ETS0+IiCcjYj5QPvr8HmB2RLwQES8Cs4HjMqy1T+0tXj7WzKxclsExAVhSsr003bfdrpV0lqR5kuatXLlymwvtT1s6OO75qszMthrSg+MRcXVETI+I6ePHj9/u7791jMMtDjOzXlkGxzJg15LtznRf1tduN20tXj7WzKxclsExF5gsaQ9JReAUYGaF194KvFvSjumg+LvTfTU1yrfjmpm9RmbBERFdwDkkP/AfBn4cEYskXSLpBABJB0laCpwM/LukRem1LwD/jyR85gKXpPtqamRzAQnWOzjMzLZozvLNI2IWMKts30Ulr+eSdEP1de21wLVZ1jeYpibRNqLgriozsxIVBUfaXfQmYAPwZEQ0zORNbS1ePtbMrFS/wSGpAzgbmAEUgZXASGBnSX8E/jUibq9JlTka1dLs23HNzEoM1OK4Cfg+cERErC49IOlA4GOS9oyI72RYX+7aigUPjpuZleg3OCLi2AGO3Qvcm0lFdaa96HXHzcxKDXpXlaSbJb1X0pB+WHBbtbV4+Vgzs1KVhMG/AqcCj0n6qqS9M66prrQXvXysmVmpQYMjIn4dER8BDgCeBH4t6S5JZ0oakXWBeWtvKXhw3MysREXdT5LGkkx//pfA/cA3SYJkdmaV1Yk2j3GYmb3KoM9xSLoF2Bu4HnhfRCxPD/1I0rwsi6sH7ekYR0QgKe9yzMxyV8kDgJf397xGREzfzvXUnbZiM909wcauHkaOKORdjplZ7irpqpoiaUzvRjrx4P/NrqT60p6uyeE7q8zMEpUExydLHwBMV+T7ZGYV1Zl2z5BrZvYqlQRHQSWd++la4sXsSqovW4LDA+RmZkBlYxy/IhkI//d0+1Ppvobg5WPNzF6tkuD4IklYfDrdng1ck1lFdaa3xeEZcs3MEoMGRzqF+rfSXw2nd91xtzjMzBKVPMcxGfgnYArJtOoARMSeGdZVN9p71x334LiZGVDZ4Ph3SVobXcA7SaZavyHLoupJW9FdVWZmpSoJjtaImAMoIp6KiIuB92ZbVv3Y0uLwcxxmZkBlg+Mb0ynVH5N0DrAMGJVtWfWjdUQBCda7q8rMDKisxXEe0AacCxwIfBQ4Pcui6omkdDEntzjMzGCQFkf6sN+HI+J84GXgzJpUVWe8fKyZ2VYDtjgiohs4vEa11K32Frc4zMx6VTLGcb+kmcBPgHW9OyPi5syqqjNtxYLHOMzMUpUEx0jgeeDokn0BNExwtHsxJzOzLSp5crwhxzVKtbcUWPXyprzLMDOrC5U8Of5dkhbGq0TExzOpqA61tTSz7oX1eZdhZlYXKumq+mXJ65HAB4BnsimnPrUXC6z3XFVmZkBlXVU/Ld2W9EPgd5lVVIfaPMZhZrZFJQ8AlpsM7LS9C6lno1qaWb+pm4jX9NiZmTWcSsY41vLqMY5nSdboaBhtLQW6e4KNXT2MHFHIuxwzs1xV0lU1uhaF1LPeNTkuvGUBLc1JcHS0juCvj528ZdvMrFFU0uL4AHBbRLyUbo8BjoqIn2VbWv2Y2tnBhDGt/PaxVQBEBKte3kTnjq189JDdc67OzKy2NFi/vaQHIuItZfvuj4i3ZllYtaZPnx7z5s2ryWdFBH/xrbt4bs1Gbj//KIrN2zJUZGaWP0n3RsT0aq6p5CdeX+dUchvvsCWJc4+ZzLLVG/jpfUvzLsfMrKYqCY55ki6V9Ob016XAvVkXVu/esdd49u/s4KrbF7O5uyfvcszMaqaS4PgMsAn4EXAj8ApwdpZFDQWSOO9dk1n64gZuuX9Z3uWYmdXMoMEREesi4oKImB4RB0XEhRGxbrDrACQdJ+kRSYslXdDH8RZJP0qP3y1pYrp/hKTrJC2Q9LCkL1X9zWrgnXvvxH4TduCq2xfT5VaHmTWIQYND0uz0Tqre7R0l3VrBdQXgKuB4YAowQ9KUstM+AbwYEZOAy4CvpftPBloiYirJqoOf6g2VeiKJc4+ezFPPr2fmgw01C4uZNbBKuqrGRcTq3o2IeJHKnhw/GFgcEY9HxCaSbq4Ty845EbgufX0TcIwkkTxw2C6pGWgl6SpbU8Fn1tyxU3bmz3bZgStvW0x3j58sN7Phr5Lg6JG0W++GpN3pY7bcPkwAlpRsL0339XlORHQBLwFjSUJkHbAceBr454h4ofwDJJ0laZ6keStXrqygpO0vaXVM4vFV6/jlfLc6zGz4qyQ4/hb4naTrJd0A3AlkPeZwMNANvAnYA/i8pD3LT4qIq9Oxl+njx4/PuKT+vWffN7L3zqO5wq0OM2sAlQyO/wo4gK13VR0YEYOOcQDLgF1LtjvTfX2ek3ZLdZCsNngq8KuI2BwRK4DfA1U9oFJLTU3iM8dMYvGKl/mvhcvzLsfMLFOVPvLcDawgGWeYIunICq6ZC0yWtIekInAKMLPsnJnA6enrk0imNgmS7qmjASS1A4cAf6qw1lwcv98uTNppFFfMWUyPWx1mNoxVclfVX5J0T90KfCX9/eLBrkvHLM5Jz38Y+HFELJJ0iaQT0tO+A4yVtBj4HNB7y+5VwChJi0gC6LsRMb+aL1ZrhSbxmaMn8chza7l10bN5l2NmlplK5qpaABwE/DEi3iJpH+AfI+KDtSiwUrWcq6o/3T3BsZfeQbG5iVnnHkFTk3Ktx8xsMFnNVfVKRLySfkBLRPwJ2HtbChzuCk3inKMn8adn1/Lrh5/Luxwzs0xUEhxL0wcAfwbMlvRz4KksixrKTtj/Tew+to3Lb3vMKwaa2bBUyV1VH4iI1RFxMfB3JOMS78+4riGrudDE2e+cxMJla7j9kRV5l2Nmtt1VtZBERNwRETPTJ8GtHx946wR2fUMr35yz2K0OMxt2vAJRBkYUmjj7qEk8uGQ1dzyazxPtZmZZcXBk5IMHdDJhTCvfnOOxDjMbXqpayU/Sn0fEL7MqZjgpNjfx6aPezJd/tpDzfzKf0SPzXTRx5IgCZx42kZ13GJlrHWY29FX70+wSwMFRoZOnd/KTeUuY/VD+DwSu39TNLx58hus+fhCTdhqddzlmNoRVGxx+oq0KLc0Ffn7O4XmXAcDCZS9xxnfv4aR/+wPfOX06B+7+hrxLMrMhqtoxjk9lUoVlbr8JHdz86cMY0zqCU799N7Mf8gOKZrZtqr0d956sCrHs7Ta2jZ9++lD2eeNoPnX9PH5w99N5l2RmQ5DvqmowY0e18MOzDuEde43nwlsWcNnsR33Xl5lVxcHRgNqKzVx92nROPrCTb855jAtvWUBXd0/eZZnZELFN94hK2ied7NCGqBGFJr5+0jR23mEkV96+mJVrN3LFjANoLRbyLs3M6ty2Plzw38Bug55ldU0S579nb3buGMlFP1/Iqdf8kX94/1SKzZU1RCeMaXXQmDWgfoND0uX9HQLGZFKN5eJjh+zO+FEtnHvj/fyfy39b8XXv2Gs813384AwrM7N6NFCL40zg88DGPo7NyKYcy8tx+72RWecewUPL11R0/rW/e4JnX3ol46rMrB4NFBxzgYURcVf5AUkXZ1aR5WbSTqOYtNOois79zZ9WcM+TL2RckZnVo4GC4ySgz39SRsQe2ZRjQ0VrscCGTd15l2FmOeg3OCLC/5y0frUVC6x3cJg1pH5vn5H0C0nvkzSij2N7SrpE0sezLc/qVVuxmQ2bu+np8cODZo1moK6qTwKfA74h6QVgJTASmAj8D3BlRPw88wqtLrWlt+G+0tVNWzHfKePNrLYG6qp6FvgC8AVJE4FdgA3AoxGxvjblWb3qDY51Gx0cZo2mor/xEfEk8GSmldiQ0pqGhQfIzRqP56qybdLb4li/uSvnSsys1hwctk22BIdbHGYNZ9DgkHReJfussfSOa6zf6OAwazSVtDhO72PfGdu5DhtitrY43FVl1mgGmuRwBnAqsIekmSWHRgN+OLDB9c6Ku2GzWxxmjWagu6ruApYD44B/Kdm/FpifZVFW/zzGYda4BnqO4yngKeDtknYHJkfEryW1Aq0kAWINassYh4PDrOFUMjj+SeAm4N/TXZ3AzzKsyYaALS2OjR7jMGs0lQyOnw0cBqwBiIjHgJ2yLMrq34hCEyMKYr3HOMwaTiXBsTEiNvVuSGoGPLOd0TrCU6ubNaJKguMOSRcCrZKOBX4C/CLbsmwoaCs2+3ZcswZUSXB8kWRm3AXAp4BZwJezLMqGhraWAuvc4jBrOANOciipACyKiH2Ab1f75pKOA74JFIBrIuKrZcdbgO8DBwLPAx9OJ1RE0jSSAfkdgB7goIjwItd1pM2rAJo1pAFbHBHRDTwiabdq3zgNnauA44EpwAxJU8pO+wTwYkRMAi4DvpZe2wzcAPxVROwLHAVsrrYGy1bbCHdVmTWiSqZV3xFYJOkeYF3vzog4YZDrDgYWR8TjAJJuBE4EHio550Tg4vT1TcCVkgS8G5gfEQ+mn/V8BXVajbUWC6xev2nwE81sWKkkOP5uG997ArCkZHsp8Lb+zomILkkvAWOBvYCQdCswHrgxIr5e/gGSzgLOAthtt6obRfY6tRULPLPaXVVmjWbQ4IiIO2pRSJlm4HDgIGA9MEfSvRExp6y2q4GrAaZPn+5bhGssuavKwWHWaCp5cnytpDVlv5ZIukXSngNcugzYtWS7M93X5znpuEYHySD5UuDOiFiVLlM7Czig8q9ltdBWLHiMw6wBVXI77jeAvyHpVuoEzgd+ANwIXDvAdXOByZL2kFQETgFmlp0zk63Ttp8E3BYRAdwKTJXUlgbKO3j12IjVgSQ43OIwazSVjHGcEBH7l2xfLemBiPhi+mBgn9Ixi3NIQqAAXBsRiyRdAsyLiJnAd4DrJS0mmar9lPTaFyVdShI+AcyKiP/cpm9omWktFtjY1UN3T1BoUt7lmFmNVBIc6yV9iOSuJ0haBr3PUww4rhARs0i6mUr3XVTy+hXg5H6uvYHkllyrU20la3KMaqnkfyUzGw4q6ar6CPAxYAXwXPr6o+n06udkWJvVua3Lx3qcw6yRVHJX1ePA+/o5/LvtW44NJV7MyawxVXJX1V6S5khamG5Pk+S5qszBYdagKumq+jbwJdIpPyJiPukgtjW21rSrasNmd1WZNZJKgqMtIu4p2+efFLalxbFuo1scZo2kkuBYJenNpHdQSToJWJ5pVTYktI5wV5VZI6rkHsqzSab12EfSMuAJkjutrMG1t7iryqwRVXpX1bsktZO0UNaTjHE8lXFtVuc8OG7WmPrtqpK0g6QvSboyXTJ2Pcn0IIuBD9WqQKtfrb0PADo4zBrKQC2O64EXgT8AnwT+FhDwgYh4IPvSrN61jfDguFkjGig49oyIqQCSriEZEN/Ny7dar+ZCE8VCE+s9xmHWUAa6q2rLUq3pErJLHRpWrq3F646bNZqBWhz7S1qTvhbQmm4LiIjYIfPqrO61jfDU6maNpt/giIhCLQuxoam16BaHWaOp5AFAs361FZtZ51UAzRqKg8Nel1avAmjWcBwc9rq0u6vKrOE4OOx1aSs2s95dVWYNxcFhr4u7qswaj4PDXpc2B4dZw6lkdlyzfrUWC6x9ZTOf/P68vEtpSIdPGsfph07MuwxrMA4Oe12OnDye3y9exdIXN+RdSsNZufYV7nniBU57++5IyrscayAODntdDps0jl9+5oi8y2hIP7j7aS68ZQFLXtjAbmPb8i7HGojHOMyGqGmdHQDMX7Y630Ks4Tg4zIaovXYeTbHQxIKlL+VdijUYB4fZEFVsbuLPdhnNfAeH1ZiDw2wIm9Y5hoXLXqKnJ/IuxRqIg8NsCJva2cHajV088fy6vEuxBuLgMBvCegfIPc5hteTgMBvCJo0fxcgRTR7nsJpycJgNYc2FJvZ9UwcLfEuu1ZCDw2yImzqhg4XL1tDtAXKrEQeH2RA3rbODDZu7+Z+VL+ddijUIB4fZELflCXKPc1iNODjMhrg9xo2ivVhgwdLVeZdiDcLBYTbEFZrEvhM6mL/MLQ6rjUyDQ9Jxkh6RtFjSBX0cb5H0o/T43ZImlh3fTdLLks7Psk6zoW7ahA4eemYNm7t78i7FGkBmwSGpAFwFHA9MAWZImlJ22ieAFyNiEnAZ8LWy45cC/5VVjWbDxdTODjZ29fDYcx4gt+xl2eI4GFgcEY9HxCbgRuDEsnNOBK5LX98EHKN0RRpJ7weeABZlWKPZsDCtcwyAn+ewmsgyOCYAS0q2l6b7+jwnIrqAl4CxkkYBXwS+kmF9ZsPGxLFtjB7Z7DurrCbqdXD8YuCyiBiw3S3pLEnzJM1buXJlbSozq0OSmNbZwQIPkFsNZBkcy4BdS7Y70319niOpGegAngfeBnxd0pPAZ4ELJZ1T/gERcXVETI+I6ePHj9/uX8BsKJk6YQwPL1/Dxq7uvEuxYS7L4JgLTJa0h6QicAows+ycmcDp6euTgNsicURETIyIicA3gH+MiCszrNVsyJvW2cHm7uCRZ9fmXYoNc5kFRzpmcQ5wK/Aw8OOIWCTpEkknpKd9h2RMYzHwOeA1t+yaWWWmTvAT5FYbzVm+eUTMAmaV7buo5PUrwMmDvMfFmRRnNsx07tjKjm0jvDaHZa5eB8fNrEqSmNo5xk+QW+YcHGbDyLQJHTz63Fpe2ewBcsuOg8NsGJna2UF3T/DQ8jV5l2LDmIPDbBjxGuRWCw4Os2HkjTuMZNyoFt9ZZZlycJgNI1ufIF+ddyk2jDk4zIaZqRM6WLziZdZt7Mq7FBumHBxmw8z+u3bQE3iA3DLj4DAbZvbzE+SWMQeH2TCz0+iR7NIxkvleg9wy4uAwG4amTujwLbmWGQeH2TA0rbODx1etY80rm/MuxYYhB4fZMDQ1XUp2oeetsgxkOjuumeWjd4r1z974AB2tI3KuxrK2zy47cMWMt9bs8xwcZsPQG9qLnHfMZB5b4UWdGsGuO7bW9PMcHGbD1F8fu1feJdgw5TEOMzOrioPDzMyq4uAwM7OqODjMzKwqDg4zM6uKg8PMzKri4DAzs6o4OMzMrCqKiLxr2C4krQSe2sbLxwGrtmM5Q42/v7+/v3/j2jsiRldzwbB5cjwixm/rtZLmRcT07VnPUOLv7+/v79/Y37/aa9xVZWZmVXFwmJlZVRwciavzLiBn/v6Nzd+/sVX9/YfN4LiZmdWGWxxmZlYVB4eZmVWloYND0nGSHpG0WNIFeddTS5J2lXS7pIckLZJ0Xt415UFSQdL9kn6Zdy15kDRG0k2S/iTpYUlvz7umWpL01+n//wsl/VDSyLxrypKkayWtkLSwZN8bJM2W9Fj6+46DvU/DBoekAnAVcDwwBZghaUq+VdVUF/D5iJgCHAKc3WDfv9d5wMN5F5GjbwK/ioh9gP1poD8LSROAc4HpEbEfUABOybeqzH0POK5s3wXAnIiYDMxJtwfUsMEBHAwsjojHI2ITcCNwYs411UxELI+I+9LXa0l+YEzIt6raktQJvBe4Ju9a8iCpAzgS+A5ARGyKiNW5FlV7zUCrpGagDXgm53oyFRF3Ai+U7T4RuC59fR3w/sHep5GDYwKwpGR7KQ32g7OXpInAW4G7cy6l1r4BfAHoybmOvOwBrAS+m3bXXSOpPe+iaiUilgH/DDwNLAdeioj/zreqXOwcEcvT188COw92QSMHhwGSRgE/BT4bEWvyrqdWJP05sCIi7s27lhw1AwcA34qItwLrqKCbYrhI+/JPJAnQNwHtkj6ab1X5iuT5jEGf0Wjk4FgG7Fqy3ZnuaxiSRpCExn9ExM1511NjhwEnSHqSpJvyaEk35FtSzS0FlkZEb0vzJpIgaRTvAp6IiJURsRm4GTg055ry8JykXQDS31cMdkEjB8dcYLKkPSQVSQbFZuZcU81IEknf9sMRcWne9dRaRHwpIjojYiLJf/vbIqKh/rUZEc8CSyTtne46Bngox5Jq7WngEElt6d+HY2igmwNKzAROT1+fDvx8sAuGzey41YqILknnALeS3E1xbUQsyrmsWjoM+BiwQNID6b4LI2JWfiVZDj4D/Ef6j6fHgTNzrqdmIuJuSTcB95HcZXg/w3z6EUk/BI4CxklaCvw98FXgx5I+QbI0xYcGfR9POWJmZtVo5K4qMzPbBg4OMzOrioPDzMyq4uAwM7OqODjMzKwqDg7LnKTLJH22ZPtWSdeUbP+LpM8NcP33JJ2Uvv6NpOkDnHuGpCu3ocaJpTOGDnDOqSXb0yVdXu1nVVDLX0k6bXu/b5a29c/dhiYHh9XC70mfyJXUBIwD9i05fihwVw51VWsisCU4ImJeRJy7vT8kIv4tIr6/vd/XbHtxcFgt3AX0rvOwL7AQWCtpR0ktwJ8B90m6SNLcdG2Eq9OnefuVrqdyn6QHJc3p4/hESbdJmi9pjqTd0v07S7olve5BSYeWXbdnOunfQWVv+VXgCEkPpOs4HNW7joekiyVdJ+m3kp6S9EFJX5e0QNKv0uldkHSgpDsk3Zu2vHbpo+6LJZ2fvv6NpK9JukfSo5KO6OP8XSTdmda1sPccSe+W9If0z+gn6bxkSDpI0l3pd79H0mhJIyV9N633fknvTM89Q9LN6Xd4TNLXSz73zLSme0geKO3df3Jax4OS7hzov6ENTQ4Oy1xEPAN0pT+4DwX+QDIT79uB6cCCdGr7KyPioHRthFbgz/t7T0njgW8DfxER+wMn93HaFcB1ETEN+A+gt1vpcuCO9LoDgC0zBiiZfuOnwBkRMbfs/S4AfhsRb4mIy/r4vDcDRwMnADcAt0fEVGAD8N40PK4AToqIA4FrgX/o7zuWaI6Ig4HPkjzpW+5U4NaIeAvJmhoPSBoHfBl4V0QcAMwDPpc+If4j4Lz0+78rre9skjnupgIzgOu0dVGjtwAfBqYCH1ayCNguwFdIAuNwkjVtel0EvCd9/xMq+H42xDTslCNWc3eRhMahwKUkU9gfCrxE0pUF8E5JXyBZF+ENJD/Qf9HP+x0C3BkRTwBERPkaA5AE0wfT19cDvf9aPho4Lb2uG3hJyUyp40nm6flgRGzLnE3/FRGbJS0gmcbmV+n+BSTdXHsD+wGz08ZUgWQ678H0TkB5b/o+5eYC16bB9LOIeEDSO0h+mP8+/awiSWDvDSzvDcXeGZElHU4SakTEnyQ9BeyVvv+ciHgpPe8hYHeS7sbfRMTKdP+PSs7/PfA9ST8uqd2GEQeH1UrvOMdUkq6qJcDngTUk60GMBP6VZDW2JZIuBmq9jOdLJBPfHc62Tfa3ESAieiRtjq3z+fSQ/F0TsCgiql2edWP6ezd9/J2NiDslHUmyKNX3JF0KvAjMjogZpedKmlrlZ5d+fr81lNXzV5LeltZzr6QDI+L5bfhcq1PuqrJauYuk6+mFiOhOWwhjSFoFd7E1JFalffEnDfJ+fwSOlLQHJOsm9/OZvUuBfgT4bfp6DvDp9LqCkpXwADYBHwBOU8ndUyXWAqMHqWsgjwDjla7rLWmEpH0HuWZQknYHnouIb5OsZngAyZ/PYZImpee0S9orrWGX3vGbdHyjmeTP5iPpvr2A3dJz+3M38A5JY9OWzpauQklvjoi7I+IikoWidu3vTWxocovDamUBSffGD8r2jYqIVQCSvk3SGnmWpPulXxGxUtJZwM1K7tRaARxbdtpnSFozf0PyA6x35tfzgKuVzAbaTRIiy9P3XadkkafZkl6OiNKp9ucD3ZIeJFm7+f4qvj8RsUnJbcWXp2HVTLIK4eudlfko4G8kbQZeBk5L/3zOAH6o5AYEgC9HxKOSPgxcIamVZHzjXSStvW+l3WxdJGM8G9XP/QkRsTxtFf4BWA08UHL4/0uaTNLCmgM8+Dq/n9UZz45rZmZVcVeVmZlVxcFhZmZVcXCYmVlVHBxmZlYVB4eZmVXFwWFmZlVxcJiZWVX+FxfOQx1HEspKAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGwCAYAAABFFQqPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABKRklEQVR4nO3de1xUdf4/8NcMMDOoMKYoeAHxgimCoIAIGnhhQ9dMyl+hWSC2Vq6kRlliCqkpWmpY+k1zTe1ikttmpebGTuIVQ0FM8m4aeBkQU0bBuM35/WGMO8uoDAznDMzr+XjM4wFnPnPmPbNtvPqc9/l8ZIIgCCAiIiKyQXKpCyAiIiKSCoMQERER2SwGISIiIrJZDEJERERksxiEiIiIyGYxCBEREZHNYhAiIiIim2UvdQHWSK/X4/Lly3BycoJMJpO6HCIiIqoDQRBw8+ZNdOzYEXJ53eZ6GIRMuHz5Mtzd3aUug4iIiOqhoKAAnTt3rtNYBiETnJycANz5Ip2dnSWuhoiIiOpCp9PB3d3d8He8LhiETKi5HObs7MwgRERE1MSY09bCZmkiIiKyWQxCREREZLMYhIiIiMhmMQgRERGRzbKKILRq1Sp4enpCpVIhODgYWVlZ9xz7yy+/YOzYsfD09IRMJkNqamqtMSkpKQgKCoKTkxPat2+PqKgonDp1qhE/ARERETVFkgehtLQ0JCQkIDk5GTk5OfDz80NkZCSKiopMji8rK0O3bt2wePFiuLm5mRyze/duTJ06FQcPHkR6ejoqKyvx6KOPorS0tDE/ChERETUxMkEQBCkLCA4ORlBQEFauXAngzqrO7u7uePnllzFr1qz7vtbT0xMzZszAjBkz7jvu6tWraN++PXbv3o2wsLAH1qTT6aBWq1FSUsLb54mIiJqI+vz9lnRGqKKiAtnZ2YiIiDAck8vliIiIQGZmpsXep6SkBADQpk0bk8+Xl5dDp9MZPYiIiKj5kzQIFRcXo7q6Gq6urkbHXV1dodVqLfIeer0eM2bMwKBBg+Dj42NyTEpKCtRqteHB7TWIiIhsg+Q9Qo1t6tSpyMvLw+bNm+85JjExESUlJYZHQUGBiBUSERGRVCTdYsPFxQV2dnYoLCw0Ol5YWHjPRmhzxMfHY9u2bdizZ899N19TKpVQKpUNfj8iIiJqWiSdEVIoFAgICIBGozEc0+v10Gg0CAkJqfd5BUFAfHw8vv76a/z444/o2rWrJcolIiKiZkbyTVcTEhIQGxuLwMBADBgwAKmpqSgtLUVcXBwAICYmBp06dUJKSgqAOw3Wx48fN/x86dIl5ObmolWrVujRoweAO5fDNm3ahG+++QZOTk6GfiO1Wg1HR0cJPuUdZRVV+L20wuRzTkoHqFs4iFwRERGRbZP89nkAWLlyJd59911otVr4+/vj/fffR3BwMABgyJAh8PT0xIYNGwAAFy5cMDnDEx4ejoyMDAD33nV2/fr1mDhx4gPraazb5789ehnTvjhi8jl7uQyf/S0YA7u1tdj7ERER2ZL6/P2WfEYIuNPLEx8fb/K5mnBTw9PTEw/KblaQ7Uyyk8mgtK99NbKyWo8qvYC8SyUMQkRERCKyiiBkK0b17YBRfTvUOj5zy1Fsyb6IymrrDHBERETNVbO/fb4pcPhzlqiyWi9xJURERLaFQcgKOMjv9DQxCBEREYmLQcgKONjd+Z+hgkGIiIhIVAxCVsBwaayKPUJERERiYhCyAjUzQrw0RkREJC4GISugsGOPEBERkRQYhKwAe4SIiIikwSBkBWqCUBXXESIiIhIVg5AV4DpCRERE0mAQsgLsESIiIpIGg5AVuNsjxEtjREREYmIQsgKG2+erOCNEREQkJgYhK8B1hIiIiKTBIGQFFPbsESIiIpICg5AVYI8QERGRNBiErIC9nJfGiIiIpMAgZAV4aYyIiEgaDEJWgHeNERERSYNByAqwR4iIiEgaDEJWgLfPExERSYNByAooGISIiIgkwSBkBRz+bJbm7vNERETiYhCyAnd7hPQQBIYhIiIisTAIWYGaIAQAVXoGISIiIrEwCFkBxX8FIfYJERERiYdByAo42MkMP1dWcUaIiIhILAxCVsBOLoPszyxUwRkhIiIi0TAIWQGZTMa1hIiIiCTAIGQluJYQERGR+BiErIS9HTdeJSIiEpvkQWjVqlXw9PSESqVCcHAwsrKy7jn2l19+wdixY+Hp6QmZTIbU1NRaY/bs2YPRo0ejY8eOkMlk2Lp1a+MVb0GGtYTYLE1ERCQaSYNQWloaEhISkJycjJycHPj5+SEyMhJFRUUmx5eVlaFbt25YvHgx3NzcTI4pLS2Fn58fVq1a1ZilWxwvjREREYnPXso3X758OSZPnoy4uDgAwOrVq7F9+3Z8/PHHmDVrVq3xQUFBCAoKAgCTzwPAyJEjMXLkyMYrupE48NIYERGR6CSbEaqoqEB2djYiIiLuFiOXIyIiApmZmaLWUl5eDp1OZ/QQ239vs0FERETikCwIFRcXo7q6Gq6urkbHXV1dodVqRa0lJSUFarXa8HB3dxf1/YG7QYgbrxIREYlH8mZpa5CYmIiSkhLDo6CgQPQaHOzZI0RERCQ2yXqEXFxcYGdnh8LCQqPjhYWF92yEbixKpRJKpVLU9/xfCvYIERERiU6yGSGFQoGAgABoNBrDMb1eD41Gg5CQEKnKkszdHiFeGiMiIhKLpHeNJSQkIDY2FoGBgRgwYABSU1NRWlpquIssJiYGnTp1QkpKCoA7DdbHjx83/Hzp0iXk5uaiVatW6NGjBwDg1q1bOHv2rOE9zp8/j9zcXLRp0wYeHh4if8K6M2yxUcUZISIiIrFIGoSio6Nx9epVJCUlQavVwt/fHzt37jQ0UOfn50MuvztpdfnyZfTr18/w+9KlS7F06VKEh4cjIyMDAHD48GEMHTrUMCYhIQEAEBsbiw0bNjT+h6on7jVGREQkPkmDEADEx8cjPj7e5HM14aaGp6cnBOH+l46GDBnywDHWSGHPHiEiIiKx8a4xK8EeISIiIvExCFkJXhojIiISH4OQlTBsscFmaSIiItEwCFkJzggRERGJj0HISrBHiIiISHwMQlaCM0JERETiYxCyEtxig4iISHwMQlbi7owQL40RERGJhUHISnD3eSIiIvExCFkJ9ggRERGJj0HISrBHiIiISHwMQlbCcPt8FXuEiIiIxMIgZCV4aYyIiEh8DEJWgs3SRERE4mMQshLsESIiIhIfg5CV4BYbRERE4mMQshL2NT1C3H2eiIhINAxCVsKBl8aIiIhExyBkJRS8a4yIiEh0DEJWgnuNERERiY9ByErcbZbmjBAREZFYGISshML+To9QFYMQERGRaBiErAQvjREREYmPQchK8NIYERGR+BiErMR/7zUmCJwVIiIiEgODkJWouX1eEIBqPYMQERGRGBiErITDn83SAPuEiIiIxMIgZCVqLo0B7BMiIiISC4OQlbCX//eMEIMQERGRGBiErIRMJuN+Y0RERCJjELIihjvHqtgjREREJAarCEKrVq2Cp6cnVCoVgoODkZWVdc+xv/zyC8aOHQtPT0/IZDKkpqY2+JzWgmsJERERiUvyIJSWloaEhAQkJycjJycHfn5+iIyMRFFRkcnxZWVl6NatGxYvXgw3NzeLnNNaOHAHeiIiIlFJHoSWL1+OyZMnIy4uDt7e3li9ejVatGiBjz/+2OT4oKAgvPvuuxg3bhyUSqVFzmktFOwRIiIiEpWkQaiiogLZ2dmIiIgwHJPL5YiIiEBmZqZo5ywvL4dOpzN6SMHBnjNCREREYpI0CBUXF6O6uhqurq5Gx11dXaHVakU7Z0pKCtRqteHh7u5er/duKG68SkREJC7JL41Zg8TERJSUlBgeBQUFktTBHiEiIiJx2Uv55i4uLrCzs0NhYaHR8cLCwns2QjfGOZVK5T37jcTEHiEiIiJxSTojpFAoEBAQAI1GYzim1+uh0WgQEhJiNecUi+H2ea4jREREJApJZ4QAICEhAbGxsQgMDMSAAQOQmpqK0tJSxMXFAQBiYmLQqVMnpKSkALjTDH38+HHDz5cuXUJubi5atWqFHj161Omc1oqXxoiIiMQleRCKjo7G1atXkZSUBK1WC39/f+zcudPQ7Jyfnw+5/O7E1eXLl9GvXz/D70uXLsXSpUsRHh6OjIyMOp3TWvGuMSIiInHJBEHgdZj/odPpoFarUVJSAmdnZ9He928bD+E/J4qwZKwvooM8RHtfIiKi5qA+f79515gVubvFBrMpERGRGBiErIi9YdNVXhojIiISA4OQFXHg7fNERESiYhCyIgreNUZERCQqBiErwh4hIiIicTEIWRGuI0RERCQuBiEr4mD/Z48Qm6WJiIhEwSBkRWp6hKr0vDRGREQkBgYhK3K3R4gzQkRERGJgELIiDlxHiIiISFQMQlaE6wgRERGJi0HIiigMm66yR4iIiEgM9dp9Pj8/H7/99hvKysrQrl079OnTB0ql0tK12Rz2CBEREYmrzkHowoUL+PDDD7F582ZcvHgR/71pvUKhwCOPPIIXXngBY8eOhVzOiab64DpCRERE4qpTYpk2bRr8/Pxw/vx5vP322zh+/DhKSkpQUVEBrVaLHTt2YPDgwUhKSkLfvn1x6NChxq67WWKPEBERkbjqNCPUsmVL/Prrr2jbtm2t59q3b49hw4Zh2LBhSE5Oxs6dO1FQUICgoCCLF9vc3b1rjD1CREREYqhTEEpJSanzCUeMGFHvYmwde4SIiIjEZXYzT3JyMn777bfGqMXm8dIYERGRuMwOQt988w26d++O4cOHY9OmTSgvL2+MumySgs3SREREojI7COXm5uLQoUPo06cPpk+fDjc3N0yZMoUN0hbgwHWEiIiIRFWv+9z79euH999/H5cvX8a6detw8eJFDBo0CH379sWKFStQUlJi6TptgqFHiFtsEBERiaJBC/4IgoDKykpUVFRAEAQ89NBDWLlyJdzd3ZGWlmapGm1GTY9QlZ5BiIiISAz1CkLZ2dmIj49Hhw4d8Morr6Bfv344ceIEdu/ejTNnzmDhwoWYNm2apWtt9u72CPHSGBERkRjMDkK+vr4YOHAgzp8/j3Xr1qGgoACLFy9Gjx49DGPGjx+Pq1evWrRQW8Dd54mIiMRl9l5jTz/9NCZNmoROnTrdc4yLiwv0vLxjtppmaa4jREREJA6zg9DcuXMbow4C1xEiIiISm9mXxsaOHYslS5bUOv7OO+/gqaeeskhRtqqmR0gvANV69gkRERE1NrOD0J49e/DXv/611vGRI0diz549FinKVtX0CAGcFSIiIhKD2UHo1q1bUCgUtY47ODhAp9NZpChb9d9BiH1CREREja9ed42ZWiNo8+bN8Pb2tkhRtqqmRwjgnWNERERiqFez9JNPPolz585h2LBhAACNRoMvvvgCW7ZssXiBtkQmk8FeLkOVXuBaQkRERCIwOwiNHj0aW7duxaJFi/DPf/4Tjo6O6Nu3L/7zn/8gPDy8MWq0KQ52clTpq9kjREREJIJ6rSw9atQo7N+/H6WlpSguLsaPP/7YoBC0atUqeHp6QqVSITg4GFlZWfcdv2XLFvTq1QsqlQq+vr7YsWOH0fOFhYWYOHEiOnbsiBYtWmDEiBE4c+ZMvesTU83lMfYIERERNb4G7TVmCWlpaUhISEBycjJycnLg5+eHyMhIFBUVmRx/4MABjB8/Hs8//zyOHDmCqKgoREVFIS8vD8Cd/c+ioqLw66+/4ptvvsGRI0fQpUsXREREoLS0VMyPVi8Kww70DEJERESNTSYIglnNKNXV1Xjvvffw5ZdfIj8/HxUVFUbP//7772YVEBwcjKCgIKxcuRIAoNfr4e7ujpdffhmzZs2qNT46OhqlpaXYtm2b4djAgQPh7++P1atX4/Tp03j44YeRl5eHPn36GM7p5uaGRYsW4W9/+1utc5aXl6O8vNzwu06ng7u7O0pKSuDs7GzW52mokBQNrpT8gW0vD4ZPJ7Wo701ERNSU6XQ6qNVqs/5+mz0jNG/ePCxfvhzR0dEoKSlBQkICnnzyScjlcrz11ltmnauiogLZ2dmIiIi4W5BcjoiICGRmZpp8TWZmptF4AIiMjDSMrwk0KpXK6JxKpRL79u0zec6UlBSo1WrDw93d3azPYUk1t9Dz0hgREVHjMzsIff7551i7di1effVV2NvbY/z48fjHP/6BpKQkHDx40KxzFRcXo7q6Gq6urkbHXV1dodVqTb5Gq9Xed3yvXr3g4eGBxMREXL9+HRUVFViyZAkuXryIK1eumDxnYmIiSkpKDI+CggKzPoclGbbZ4O3zREREjc7sIKTVauHr6wsAaNWqFUpKSgAAjz32GLZv327Z6urBwcEB//rXv3D69Gm0adMGLVq0wK5duzBy5EjI5aY/rlKphLOzs9FDKoYd6Hn7PBERUaMzOwh17tzZMLPSvXt3/PDDDwCAQ4cOQalUmnUuFxcX2NnZobCw0Oh4YWEh3NzcTL7Gzc3tgeMDAgKQm5uLGzdu4MqVK9i5cyeuXbuGbt26mVWfFNgsTUREJB6zg9ATTzwBjUYDAHj55Zcxd+5ceHl5ISYmBpMmTTLrXAqFAgEBAYbzAXcamzUaDUJCQky+JiQkxGg8AKSnp5scr1ar0a5dO5w5cwaHDx/GmDFjzKpPCuwRIiIiEo/ZCyouXrzY8HN0dDS6dOmCAwcOwMvLC6NHjza7gISEBMTGxiIwMBADBgxAamoqSktLERcXBwCIiYlBp06dkJKSAgCYPn06wsPDsWzZMowaNQqbN2/G4cOH8dFHHxnOuWXLFrRr1w4eHh44duwYpk+fjqioKDz66KNm1yc2Q48QgxAREVGjMysIVVZW4sUXX8TcuXPRtWtXAHduXR84cGC9C4iOjsbVq1eRlJQErVYLf39/7Ny509AQnZ+fb9TbExoaik2bNmHOnDmYPXs2vLy8sHXrVvj4+BjGXLlyBQkJCSgsLESHDh0QExODuXPn1rtGMd3tEWIQIiIiamxmryOkVquRm5trCELNUX3WIbCU5zccguZkEd4Z2xdPB0l3Gz8REVFTI8o6QlFRUdi6dau5L6M6sucWG0RERKIxu0fIy8sL8+fPx/79+xEQEICWLVsaPT9t2jSLFWeLeGmMiIhIPGYHoXXr1qF169bIzs5Gdna20XMymYxBqIEUDEJERESiMTsInT9/vjHqoD9xQUUiIiLxSL77PBlzsP+zR4hbbBARETU6s2eEHrRo4scff1zvYujujFCVnkGIiIiosZkdhK5fv270e2VlJfLy8nDjxg0MGzbMYoXZKgUvjREREYnG7CD09ddf1zqm1+sxZcoUdO/e3SJF2TLDFhu8NEZERNToLNIjJJfLkZCQgPfee88Sp7NpvH2eiIhIPBZrlj537hyqqqosdTqbVdMszSBERETU+My+NJaQkGD0uyAIuHLlCrZv347Y2FiLFWar2CNEREQkHrOD0JEjR4x+l8vlaNeuHZYtW/bAO8rowQw9QpwRIiIianRmB6Fdu3Y1Rh30J0OPEJuliYiIGp3ZPULnz5/HmTNnah0/c+YMLly4YImabJqDHXuEiIiIxGJ2EJo4cSIOHDhQ6/hPP/2EiRMnWqImm8YtNoiIiMRjdhA6cuQIBg0aVOv4wIEDkZuba4mabBp7hIiIiMRjdhCSyWS4efNmreMlJSWorq62SFG2jJfGiIiIxGN2EAoLC0NKSopR6KmurkZKSgoGDx5s0eJskYM9F1QkIiISi9l3jS1ZsgRhYWF4+OGH8cgjjwAA9u7dC51Ohx9//NHiBdoawzpCVewRIiIiamxmByFvb2/8/PPPWLlyJY4ePQpHR0fExMQgPj4ebdq0aYwabUpNj9ClG7cRvynHcNy9TQu89ujDsJPLpCqNiIio2TE7CAFAx44dsWjRIkvXQgDaOSkBALfKq7Dt5ytGzwV5PoRhvVylKIuIiKhZMjsIrV+/Hq1atcJTTz1ldHzLli0oKyvjNhsN1NWlJdbHBeFCcanh2H9OFGL/2WvYc7qYQYiIiMiCzA5CKSkpWLNmTa3j7du3xwsvvMAgZAFDH24PPHz39w5qxz+D0FXpiiIiImqGzL5rLD8/H127dq11vEuXLsjPz7dIUWQstEdb2Mll+LW4FAW/l0ldDhERUbNhdhBq3749fv7551rHjx49irZt21qkKDLmrHJAP/fWAIC9Z4qlLYaIiKgZMTsIjR8/HtOmTcOuXbtQXV2N6upq/Pjjj5g+fTrGjRvXGDUSgLCe7QCAl8eIiIgsyOweoQULFuDChQsYPnw47O3vvFyv1yMmJoZ3kjWiR7xcsDz9NPafK0ZVtR72dmZnWCIiIvofZgchhUKBtLQ0LFiwwLCOkK+vL7p06dIY9dGf+nZujdYtHHCjrBJHL95AQBeu2URERNRQ9VpHCAB69uyJnj17WrIWug87uQyDerhg+89XsPt0MYMQERGRBdQrCF28eBHffvst8vPzUVFRYfTc8uXLLVIY1RbmdScI7T1zFQl/YQglIiJqKLODkEajweOPP45u3brh5MmT8PHxwYULFyAIAvr3798YNdKfahqmjxbcQElZJdQtHCSuiIiIqGkzu+M2MTERr732Go4dOwaVSoWvvvoKBQUFCA8Pr7XadF2tWrUKnp6eUKlUCA4ORlZW1n3Hb9myBb169YJKpYKvry927Nhh9PytW7cQHx+Pzp07w9HREd7e3li9enW9arMmHdSO8GrfCnoB2HeWt9ETERE1lNlB6MSJE4iJiQEA2Nvb4/bt22jVqhXmz5+PJUuWmF1AWloaEhISkJycjJycHPj5+SEyMhJFRUUmxx84cADjx4/H888/jyNHjiAqKgpRUVHIy8szjElISMDOnTvx2Wef4cSJE5gxYwbi4+Px7bffml2ftXnE686s0N4zvI2eiIioocwOQi1btjT0BXXo0AHnzp0zPFdcbP4sxfLlyzF58mTExcUZZm5atGiBjz/+2OT4FStWYMSIEZg5cyZ69+6NBQsWoH///li5cqVhzIEDBxAbG4shQ4bA09MTL7zwAvz8/O4501ReXg6dTmf0sFZhPV0A3FlPSBAEiashIiJq2swOQgMHDsS+ffsAAH/961/x6quvYuHChZg0aRIGDhxo1rkqKiqQnZ2NiIiIuwXJ5YiIiEBmZqbJ12RmZhqNB4DIyEij8aGhofj2229x6dIlCIKAXbt24fTp03j00UdNnjMlJQVqtdrwcHd3N+tziCm4a1so7OW4XPIHzl0tffALiIiI6J7MDkLLly9HcHAwAGDevHkYPnw40tLS4OnpiXXr1pl1ruLiYlRXV8PV1XhHdVdXV2i1WpOv0Wq1Dxz/wQcfwNvbG507d4ZCocCIESOwatUqhIWFmTxnYmIiSkpKDI+CggKzPoeYHBV2GOB559Z5rjJNRETUMGbfNdatWzfDzy1btrTKJuQPPvgABw8exLfffosuXbpgz549mDp1Kjp27FhrNgkAlEollEqlBJXWT1hPF+w7W4w9Z65i0uDaG+ASERFR3dR7QUVLcHFxgZ2dHQoLC42OFxYWws3NzeRr3Nzc7jv+9u3bmD17Nr7++muMGjUKANC3b1/k5uZi6dKlJoNQU3OnYfokDv56DeVV1VDa20ldEhERUZMk6YZVCoUCAQEB0Gg0hmN6vR4ajQYhISEmXxMSEmI0HgDS09MN4ysrK1FZWQm53Pij2dnZQa/XW/gTSKOXmxPaOynxR6Uehy9cl7ocIiKiJkvynTsTEhKwdu1abNy4ESdOnMCUKVNQWlqKuLg4AEBMTAwSExMN46dPn46dO3di2bJlOHnyJN566y0cPnwY8fHxAABnZ2eEh4dj5syZyMjIwPnz57FhwwZ88skneOKJJyT5jJYmk8kMt9GzT4iIiKj+JL00BgDR0dG4evUqkpKSoNVq4e/vj507dxoaovPz841md0JDQ7Fp0ybMmTMHs2fPhpeXF7Zu3QofHx/DmM2bNyMxMRETJkzA77//ji5dumDhwoV46aWXRP98jSWspwu+yrmIPWeKkfjg4URERGSCTGjAYjT79+9HYGBgk2o0rgudTge1Wo2SkhI4OztLXY5J126VI3DhfyAIQNabw9HeSSV1SURERJKqz9/vBl0aGzlyJC5dutSQU1A9tW2lhE9HNQBg72lut0FERFQfDQpCXNlYWo943VllmtttEBER1Y/kzdJUfzW70e89Uwy9nqGUiIjIXA0KQmvWrKm1yjOJp7/HQ2ipsMO10gocv2K9+6MRERFZqwYFoWeeeQYtW7a0VC1kJoW9HCHd2wIA9vDyGBERkdl4aayJq7k8xvWEiIiIzMcg1MSF/bmwYvZv11FaXiVxNURERE0Lg1AT16VtC7i3cURltYCDv16TuhwiIqImhUGoiZPJZIZZIV4eIyIiMo/FgtD169fxySefWOp0ZIaafcf2nuHCikREROawWBDKz883bJRK4grt0RZ2chl+LS5Fwe9lUpdDRETUZNR501Wd7v7r1Ny8ebPBxVD9OKsc0N+jNQ5duI49Z65iQnAXqUsiIiJqEuochFq3bg2ZTHbP5wVBuO/z1Lge8WqHQxeuY+/pYgYhIiKiOqpzEHJycsKbb76J4OBgk8+fOXMGL774osUKI/OE9WyH5emnsf9cMaqq9bC3Yx88ERHRg9Q5CPXv3x8AEB4ebvL51q1bcxNWCfl2UqN1CwfcKKtEbsENBHq2kbokIiIiq1fnaYNnnnkGKpXqns+7ubkhOTnZIkWR+ezkMgzqcWc3+j28e4yIiKhOZAKncWrR6XRQq9UoKSmBs7Oz1OXU2ZeHCvD6Vz+jU2tHDO3VTpT3bNdKhSlDukNhz0txREQkrfr8/a7zpTGyfmE920EuAy7duI3PDuaL9r5OKntMGtxVtPcjIiKylDoFoc2bN2PcuHF1OmFBQQHy8/MxaNCgBhVG5nNTq/CP2EAcLSgR5f1+u1aKrbmX8dGeXzFhoAeU9naivC8REZGl1CkIffjhh5g3bx7i4uIwevRo9O7d2+j5kpIS7N+/H5999hnS09Oxbt26RimWHmxYL1cM6+UqynuVV1Uj89dr0Or+wL9yLmH8AA9R3peIiMhS6tTYsXv3bixZsgTp6enw8fGBs7MzvLy84Ovri86dO6Nt27aYNGkSPDw8kJeXh8cff7yx6yYroLS3w+RHugEAPsw4h6pqvcQVERERmcfsZuni4mLs27cPv/32G27fvg0XFxf069cP/fr1g1zePBpmm2qztBTKKqoweMku/F5agfei/fBEv85Sl0RERDZKlGZpFxcXREVFmfsyaqZaKOzx/OCuePffp/B/u85hjF8nyOVcYZyIiJqG5jGFQ5J6LqQLnFT2OFN0Cz8c10pdDhERUZ0xCFGDOascEBviCQBYuessVxgnIqImg0GILGLS4K5wdLBD3iUddp++KnU5REREdcIgRBbRpqUCE4Lv3D6/atdZiashIiKqG7OD0Pz581FWVlbr+O3btzF//nyLFEVN0+SwblDYyXHownX89Os1qcshIiJ6ILOD0Lx583Dr1q1ax8vKyjBv3jyLFEVNk6uzCk8F3rl9fiVnhYiIqAkwOwgJggCZrPbt0UePHkWbNm0sUhQ1XS+Fd4edXIa9Z4pxtOCG1OUQERHdV52D0EMPPYQ2bdpAJpOhZ8+eaNOmjeGhVqvxl7/8BU8//XRj1kpNgHubFhjj3xEAZ4WIiMj61TkIpaamYvny5RAEAfPmzcN7771neKxevRr79u3DqlWr6lXEqlWr4OnpCZVKheDgYGRlZd13/JYtW9CrVy+oVCr4+vpix44dRs/LZDKTj3fffbde9ZF5/j6kB2QyIP14IU5qdVKXQ0REdE9mb7Gxe/duDBo0CPb2Zi9KbVJaWhpiYmKwevVqBAcHIzU1FVu2bMGpU6fQvn37WuMPHDiAsLAwpKSk4LHHHsOmTZuwZMkS5OTkwMfHBwCg1Rov6vf999/j+eefx9mzZ9GtW7cH1sQtNhpu6uc52H7sChzsZLCvw9YrchkwZUh3xA/zEqE6IiJqjurz99vsIAQA586dw/r163Hu3DmsWLEC7du3x/fffw8PDw/06dPHrHMFBwcjKCgIK1euBADo9Xq4u7vj5ZdfxqxZs2qNj46ORmlpKbZt22Y4NnDgQPj7+2P16tUm3yMqKgo3b96ERqOpU00MQg13SnsTj6/ch/Kqum/E2tO1FX54JbwRqyIiouZMlL3Gdu/ejZEjR2LQoEHYs2cPFi5ciPbt2+Po0aNYt24d/vnPf9b5XBUVFcjOzkZiYqLhmFwuR0REBDIzM02+JjMzEwkJCUbHIiMjsXXrVpPjCwsLsX37dmzcuPGedZSXl6O8vNzwu07HyzkN9bCbEw7NiUBJWeUDxx67VIK/f56DCjNCExERkSWYfdfYrFmz8PbbbyM9PR0KhcJwfNiwYTh48KBZ5youLkZ1dTVcXV2Njru6uta6vFVDq9WaNX7jxo1wcnLCk08+ec86UlJSoFarDQ93d3ezPgeZ5qxygHubFg98dGrtCAAMQkREJDqzg9CxY8fwxBNP1Drevn17FBcXW6QoS/r4448xYcIEqFSqe45JTExESUmJ4VFQUCBihaSwv/OPYUU19ygjIiJxmX1prHXr1rhy5Qq6du1qdPzIkSPo1KmTWedycXGBnZ0dCgsLjY4XFhbCzc3N5Gvc3NzqPH7v3r04deoU0tLS7luHUqmEUqk0q3ayHEMQqqqWuBIiIrI1Zs8IjRs3Dm+88Qa0Wi1kMhn0ej3279+P1157DTExMWadS6FQICAgwKiJWa/XQ6PRICQkxORrQkJCajU9p6enmxy/bt06BAQEwM/Pz6y6SFwKu5oZIV4aIyIicZkdhBYtWoRevXrB3d0dt27dgre3N8LCwhAaGoo5c+aYXUBCQgLWrl2LjRs34sSJE5gyZQpKS0sRFxcHAIiJiTFqpp4+fTp27tyJZcuW4eTJk3jrrbdw+PBhxMfHG51Xp9Nhy5Yt+Nvf/mZ2TSQupWFGiEGIiIjEZdalMUEQoNVq8f777yMpKQnHjh3DrVu30K9fP3h51W/9l+joaFy9ehVJSUnQarXw9/fHzp07DQ3R+fn5kP/XOjShoaHYtGkT5syZg9mzZ8PLywtbt241rCFUY/PmzRAEAePHj69XXSSemktjegGoqtbD3s7sfE5ERFQvZq0jpNfroVKp8Msvv9Q7+DQFXEdIXKXlVeiT/G8AwPH5kWihsMxinUREZFvq8/fbrP/0lsvl8PLywrVr1+pVIJEpNTNCAFBZxTvHiIhIPGZfg1i8eDFmzpyJvLy8xqiHbJC9XAaZ7M7P5dW8c4yIiMRj9jWImJgYlJWVwc/PDwqFAo6OjkbP//777xYrjmyDTCaDwk6O8io9G6aJiEhUZgeh1NTURiiDbJ3CnkGIiIjEZ3YQio2NbYw6yMYp7eW4Ca4lRERE4jI7CN1rQ1KZTAalUmm0/xhRXTnYcS0hIiISX7222JDVdLaa0LlzZ0ycOBHJyclG6/8Q3U/NnWOVnBEiIiIRmR2ENmzYgDfffBMTJ07EgAEDAABZWVnYuHEj5syZg6tXr2Lp0qVQKpWYPXu2xQum5qlmm41yzggREZGIzA5CGzduxLJly/D0008bjo0ePRq+vr5Ys2YNNBoNPDw8sHDhQgYhqjMFt9kgIiIJmH3t6sCBA+jXr1+t4/369UNmZiYAYPDgwcjPz294dWQzGISIiEgKZgchd3d3rFu3rtbxdevWwd3dHQBw7do1PPTQQw2vjmyGA3egJyIiCZh9aWzp0qV46qmn8P333yMoKAgAcPjwYZw8eRL//Oc/AQCHDh1CdHS0ZSulZo070BMRkRTMDkKPP/44Tp48iTVr1uD06dMAgJEjR2Lr1q3w9PQEAEyZMsWiRVLzV9MszbvGiIhITPXa5rtr165YvHixpWshG8YeISIikkK9FvrZu3cvnn32WYSGhuLSpUsAgE8//RT79u2zaHFkO2qCEG+fJyIiMZkdhL766itERkbC0dEROTk5KC8vBwCUlJRg0aJFFi+QbIOCzdJERCQBs4PQ22+/jdWrV2Pt2rVwcHAwHB80aBBycnIsWhzZDgdeGiMiIgmYHYROnTqFsLCwWsfVajVu3LhhiZrIBim41xgREUnA7CDk5uaGs2fP1jq+b98+dOvWzSJFke1Rcq8xIiKSgNlBaPLkyZg+fTp++uknyGQyXL58GZ9//jlee+013jZP9ca7xoiISApm3z4/a9Ys6PV6DB8+HGVlZQgLC4NSqcRrr72Gl19+uTFqJBvAZmkiIpKC2UFIJpPhzTffxMyZM3H27FncunUL3t7eaNWqFW7fvg1HR8fGqJOaOd4+T0REUqjXOkIAoFAo4O3tjQEDBsDBwQHLly9H165dLVkb2RAHNksTEZEE6hyEysvLkZiYiMDAQISGhmLr1q0AgPXr16Nr165477338MorrzRWndTMsUeIiIikUOdLY0lJSVizZg0iIiJw4MABPPXUU4iLi8PBgwexfPlyPPXUU7Czs2vMWqkZU/CuMSIikkCdg9CWLVvwySef4PHHH0deXh769u2LqqoqHD16FDKZrDFrJBtg2H2eQYiIiERU50tjFy9eREBAAADAx8cHSqUSr7zyCkMQWQQXVCQiIinUOQhVV1dDoVAYfre3t0erVq0apSiyPewRIiIiKdT50pggCJg4cSKUSiUA4I8//sBLL72Eli1bGo3717/+ZdkKySbU3DXG2+eJiEhMdQ5CsbGxRr8/++yzFi+GbJeCPUJERCSBOgeh9evXN2YdZON4aYyIiKRQ7wUVLWnVqlXw9PSESqVCcHAwsrKy7jt+y5Yt6NWrF1QqFXx9fbFjx45aY06cOIHHH38carUaLVu2RFBQEPLz8xvrI1AD1TRL8/Z5IiISk+RBKC0tDQkJCUhOTkZOTg78/PwQGRmJoqIik+MPHDiA8ePH4/nnn8eRI0cQFRWFqKgo5OXlGcacO3cOgwcPRq9evZCRkYGff/4Zc+fOhUqlEutjkZmUnBEiIiIJyARBEKQsIDg4GEFBQVi5ciUAQK/Xw93dHS+//DJmzZpVa3x0dDRKS0uxbds2w7GBAwfC398fq1evBgCMGzcODg4O+PTTT+tVk06ng1qtRklJCZydnet1DjLPheJSDFmagZYKO/wyf4TU5RARURNUn7/fks4IVVRUIDs7GxEREYZjcrkcERERyMzMNPmazMxMo/EAEBkZaRiv1+uxfft29OzZE5GRkWjfvj2Cg4MNW4KYUl5eDp1OZ/QgcbFZmoiIpCBpECouLkZ1dTVcXV2Njru6ukKr1Zp8jVarve/4oqIi3Lp1C4sXL8aIESPwww8/4IknnsCTTz6J3bt3mzxnSkoK1Gq14eHu7m6BT0fmuLvFhgC9XtJJSiIisiGS9whZml5/Z0ZhzJgxeOWVV+Dv749Zs2bhscceM1w6+1+JiYkoKSkxPAoKCsQsmXA3CAGcFSIiIvHU+fb5xuDi4gI7OzsUFhYaHS8sLISbm5vJ17i5ud13vIuLC+zt7eHt7W00pnfv3ti3b5/JcyqVSsNCkSSNmrvGgDt3jqkcuIEvERE1PklnhBQKBQICAqDRaAzH9Ho9NBoNQkJCTL4mJCTEaDwApKenG8YrFAoEBQXh1KlTRmNOnz6NLl26WPgTkKX8dxDinWNERCQWSWeEACAhIQGxsbEIDAzEgAEDkJqaitLSUsTFxQEAYmJi0KlTJ6SkpAAApk+fjvDwcCxbtgyjRo3C5s2bcfjwYXz00UeGc86cORPR0dEICwvD0KFDsXPnTnz33XfIyMiQ4iNSHcjlMtjLZajSC7w0RkREopE8CEVHR+Pq1atISkqCVquFv78/du7caWiIzs/Ph1x+d7YgNDQUmzZtwpw5czB79mx4eXlh69at8PHxMYx54oknsHr1aqSkpGDatGl4+OGH8dVXX2Hw4MGifz6qO4W9HFUV1ZwRIiIi0Ui+jpA14jpC0vCf/wNulFUi/ZUweLk6SV0OERE1MU1uHSGi/6bgDvRERCQyBiGyGnfXEmIQIiIicTAIkdXgDvRERCQ2BiGyGjWXxnjXGBERiYVBiKwGZ4SIiEhsDEJkNQwzQgxCREQkEgYhshrcgZ6IiMTGIERWg5fGiIhIbAxCZDUc2CxNREQiYxAiq8EZISIiEhuDEFkNJZuliYhIZAxCZDU4I0RERGJjECKrwbvGiIhIbAxCZDW4sjQREYmNQYishgMvjRERkcgYhMhqcGVpIiISG4MQWQ02SxMRkdgYhMhqKNksTUREImMQIqvBGSEiIhIbgxBZjZoeoUrOCBERkUgYhMhq1Ow1Vs4ZISIiEgmDEFkNXhojIiKxMQiR1eDK0kREJDYGIbIanBEiIiKxMQiR1eDu80REJDYGIbIaNVts8K4xIiISC4MQWQ1usUFERGJjECKrwWZpIiISG4MQWY2aIMR1hIiISCwMQmQ1eGmMiIjExiBEVuO/N10VBEHiaoiIyBZYRRBatWoVPD09oVKpEBwcjKysrPuO37JlC3r16gWVSgVfX1/s2LHD6PmJEydCJpMZPUaMGNGYH4EsoGaLDUEAqvUMQkRE1PgkD0JpaWlISEhAcnIycnJy4Ofnh8jISBQVFZkcf+DAAYwfPx7PP/88jhw5gqioKERFRSEvL89o3IgRI3DlyhXD44svvhDj41AD1PQIAWyYJiIicUgehJYvX47JkycjLi4O3t7eWL16NVq0aIGPP/7Y5PgVK1ZgxIgRmDlzJnr37o0FCxagf//+WLlypdE4pVIJNzc3w+Ohhx4S4+NQAxgFIfYJERGRCCQNQhUVFcjOzkZERIThmFwuR0REBDIzM02+JjMz02g8AERGRtYan5GRgfbt2+Phhx/GlClTcO3atXvWUV5eDp1OZ/Qg8dnLZZDJ7vzMIERERGKQNAgVFxejuroarq6uRsddXV2h1WpNvkar1T5w/IgRI/DJJ59Ao9FgyZIl2L17N0aOHInq6mqT50xJSYFarTY83N3dG/jJqD5kMpnhzjHeQk9ERGKwl7qAxjBu3DjDz76+vujbty+6d++OjIwMDB8+vNb4xMREJCQkGH7X6XQMQxJR2MtRXqVnjxAREYlC0hkhFxcX2NnZobCw0Oh4YWEh3NzcTL7Gzc3NrPEA0K1bN7i4uODs2bMmn1cqlXB2djZ6kDRqZoS43xgREYlB0iCkUCgQEBAAjUZjOKbX66HRaBASEmLyNSEhIUbjASA9Pf2e4wHg4sWLuHbtGjp06GCZwqnRGLbZ4KUxIiISgeR3jSUkJGDt2rXYuHEjTpw4gSlTpqC0tBRxcXEAgJiYGCQmJhrGT58+HTt37sSyZctw8uRJvPXWWzh8+DDi4+MBALdu3cLMmTNx8OBBXLhwARqNBmPGjEGPHj0QGRkpyWekumMQIiIiMUneIxQdHY2rV68iKSkJWq0W/v7+2Llzp6EhOj8/H3L53bwWGhqKTZs2Yc6cOZg9eza8vLywdetW+Pj4AADs7Ozw888/Y+PGjbhx4wY6duyIRx99FAsWLIBSqZTkM1LdcZsNIiISk0zgXga16HQ6qNVqlJSUsF9IZKPe34tfLuswf0wfBHTh2k9EZJpn25ZoqZT8v+XJytTn7zf/KSKrUrPfWNI3v0hcCRFZs06tHbHn9aGwk8ukLoWaOAYhsipPB7qjUFeOKj0vjRGRaYW6cly6cRs3yirQthVbHqhhGITIqowb4IFxAzykLoOIrFjft/4N3R9VuHG7kkGIGkzyu8aIiIjM0bqFAgBwo6xS4kqoOWAQIiKiJqV1CwcAQMntCokroeaAQYiIiJoUteOdIMQZIbIEBiEiImpSGITIkhiEiIioSbl7aYxBiBqOQYiIiJqU1o53mqUZhMgSGISIiKhJqZkRulHGZmlqOAYhIiJqUgw9QpwRIgtgECIioiaFzdJkSQxCRETUpNQsqKjjjBBZAIMQERE1KYYeIQYhsgAGISIialJaO95tltbrBYmroaaOQYiIiJoU5z+DkF4AblVUSVwNNXUMQkRE1KSoHOygcrjz56uEDdPUQAxCRETU5NQsqsg7x6ihGISIiKjJ4TYbZCkMQkRE1OTcXVSRq0tTwzAIERFRk8NFFclSGISIiKjJ4aUxshQGISIianJqVpfmxqvUUAxCRETU5NRcGuOMEDUUgxARETU5hm022CNEDcQgRERETc7du8YYhKhhGISIiKjJqVlQkStLU0MxCBERUZNzdwd6NktTwzAIERFRk8N1hMhSGISIiKjJqZkRKq/S44/KaomroaaMQYiIiJqcVkp72MllAHgLPTUMgxARETU5MpmMl8fIIqwiCK1atQqenp5QqVQIDg5GVlbWfcdv2bIFvXr1gkqlgq+vL3bs2HHPsS+99BJkMhlSU1MtXDUREUmptSEIsWGa6k/yIJSWloaEhAQkJycjJycHfn5+iIyMRFFRkcnxBw4cwPjx4/H888/jyJEjiIqKQlRUFPLy8mqN/frrr3Hw4EF07NixsT8GERGJTN2CawlRw0kehJYvX47JkycjLi4O3t7eWL16NVq0aIGPP/7Y5PgVK1ZgxIgRmDlzJnr37o0FCxagf//+WLlypdG4S5cu4eWXX8bnn38OBweH+9ZQXl4OnU5n9CAiIutWMyPEtYSoISQNQhUVFcjOzkZERIThmFwuR0REBDIzM02+JjMz02g8AERGRhqN1+v1eO655zBz5kz06dPngXWkpKRArVYbHu7u7vX8REREJBbuN0aWIGkQKi4uRnV1NVxdXY2Ou7q6QqvVmnyNVqt94PglS5bA3t4e06ZNq1MdiYmJKCkpMTwKCgrM/CRERCQ2ww70XFSRGsBe6gIsLTs7GytWrEBOTg5kMlmdXqNUKqFUKhu5MiIisiTeNUaWIGkQcnFxgZ2dHQoLC42OFxYWws3NzeRr3Nzc7jt+7969KCoqgoeHh+H56upqvPrqq0hNTcWFCxcs+yGIiEgSNYsqakv+wMXrZRJXQ+ZydLBD21bST0JIGoQUCgUCAgKg0WgQFRUF4E5/j0ajQXx8vMnXhISEQKPRYMaMGYZj6enpCAkJAQA899xzJnuInnvuOcTFxTXK5yAiIvHVBCHNySJoTpq+05is1+N+HfH++H5SlyH9pbGEhATExsYiMDAQAwYMQGpqKkpLSw2hJSYmBp06dUJKSgoAYPr06QgPD8eyZcswatQobN68GYcPH8ZHH30EAGjbti3atm1r9B4ODg5wc3PDww8/LO6HIyKiRjOwW1t4tGmBQt0fUpdC9WBvV7f2lcYmeRCKjo7G1atXkZSUBK1WC39/f+zcudPQEJ2fnw+5/G5Pd2hoKDZt2oQ5c+Zg9uzZ8PLywtatW+Hj4yPVRyAiIgl0UDtiz+tDpS6DmjiZIAiC1EVYG51OB7VajZKSEjg7O0tdDhEREdVBff5+S76gIhEREZFUGISIiIjIZjEIERERkc1iECIiIiKbxSBERERENotBiIiIiGwWgxARERHZLAYhIiIislkMQkRERGSzGISIiIjIZjEIERERkc1iECIiIiKbxSBERERENste6gKskSAIAO7sYktERERNQ83f7Zq/43XBIGTCzZs3AQDu7u4SV0JERETmunnzJtRqdZ3GygRzYpON0Ov1uHz5MpycnCCTySx2Xp1OB3d3dxQUFMDZ2dli56V743cuDX7v4uN3Lg1+7+K733cuCAJu3ryJjh07Qi6vW/cPZ4RMkMvl6Ny5c6Od39nZmf+HERm/c2nwexcfv3Np8HsX372+87rOBNVgszQRERHZLAYhIiIislkMQiJSKpVITk6GUqmUuhSbwe9cGvzexcfvXBr83sVn6e+czdJERERkszgjRERERDaLQYiIiIhsFoMQERER2SwGISIiIrJZDEIiWbVqFTw9PaFSqRAcHIysrCypS2rWUlJSEBQUBCcnJ7Rv3x5RUVE4deqU1GXZlMWLF0Mmk2HGjBlSl9LsXbp0Cc8++yzatm0LR0dH+Pr64vDhw1KX1WxVV1dj7ty56Nq1KxwdHdG9e3csWLDArP2t6MH27NmD0aNHo2PHjpDJZNi6davR84IgICkpCR06dICjoyMiIiJw5swZs9+HQUgEaWlpSEhIQHJyMnJycuDn54fIyEgUFRVJXVqztXv3bkydOhUHDx5Eeno6Kisr8eijj6K0tFTq0mzCoUOHsGbNGvTt21fqUpq969evY9CgQXBwcMD333+P48ePY9myZXjooYekLq3ZWrJkCT788EOsXLkSJ06cwJIlS/DOO+/ggw8+kLq0ZqW0tBR+fn5YtWqVyeffeecdvP/++1i9ejV++ukntGzZEpGRkfjjjz/MeyOBGt2AAQOEqVOnGn6vrq4WOnbsKKSkpEhYlW0pKioSAAi7d++WupRm7+bNm4KXl5eQnp4uhIeHC9OnT5e6pGbtjTfeEAYPHix1GTZl1KhRwqRJk4yOPfnkk8KECRMkqqj5AyB8/fXXht/1er3g5uYmvPvuu4ZjN27cEJRKpfDFF1+YdW7OCDWyiooKZGdnIyIiwnBMLpcjIiICmZmZElZmW0pKSgAAbdq0kbiS5m/q1KkYNWqU0T/z1Hi+/fZbBAYG4qmnnkL79u3Rr18/rF27VuqymrXQ0FBoNBqcPn0aAHD06FHs27cPI0eOlLgy23H+/HlotVqjf8+o1WoEBweb/beVm642suLiYlRXV8PV1dXouKurK06ePClRVbZFr9djxowZGDRoEHx8fKQup1nbvHkzcnJycOjQIalLsRm//vorPvzwQyQkJGD27Nk4dOgQpk2bBoVCgdjYWKnLa5ZmzZoFnU6HXr16wc7ODtXV1Vi4cCEmTJggdWk2Q6vVAoDJv601z9UVgxA1e1OnTkVeXh727dsndSnNWkFBAaZPn4709HSoVCqpy7EZer0egYGBWLRoEQCgX79+yMvLw+rVqxmEGsmXX36Jzz//HJs2bUKfPn2Qm5uLGTNmoGPHjvzOmyBeGmtkLi4usLOzQ2FhodHxwsJCuLm5SVSV7YiPj8e2bduwa9cudO7cWepymrXs7GwUFRWhf//+sLe3h729PXbv3o33338f9vb2qK6ulrrEZqlDhw7w9vY2Ota7d2/k5+dLVFHzN3PmTMyaNQvjxo2Dr68vnnvuObzyyitISUmRujSbUfP30xJ/WxmEGplCoUBAQAA0Go3hmF6vh0ajQUhIiISVNW+CICA+Ph5ff/01fvzxR3Tt2lXqkpq94cOH49ixY8jNzTU8AgMDMWHCBOTm5sLOzk7qEpulQYMG1Voa4vTp0+jSpYtEFTV/ZWVlkMuN/3za2dlBr9dLVJHt6dq1K9zc3Iz+tup0Ovz0009m/23lpTERJCQkIDY2FoGBgRgwYABSU1NRWlqKuLg4qUtrtqZOnYpNmzbhm2++gZOTk+GasVqthqOjo8TVNU9OTk61erBatmyJtm3bsjerEb3yyisIDQ3FokWL8PTTTyMrKwsfffQRPvroI6lLa7ZGjx6NhQsXwsPDA3369MGRI0ewfPlyTJo0SerSmpVbt27h7Nmzht/Pnz+P3NxctGnTBh4eHpgxYwbefvtteHl5oWvXrpg7dy46duyIqKgo897IQne20QN88MEHgoeHh6BQKIQBAwYIBw8elLqkZg2Aycf69eulLs2m8PZ5cXz33XeCj4+PoFQqhV69egkfffSR1CU1azqdTpg+fbrg4eEhqFQqoVu3bsKbb74plJeXS11as7Jr1y6T/x6PjY0VBOHOLfRz584VXF1dBaVSKQwfPlw4deqU2e8jEwQuhUlERES2iT1CREREZLMYhIiIiMhmMQgRERGRzWIQIiIiIpvFIEREREQ2i0GIiIiIbBaDEBEREdksBiEiIiKyWQxCRDYiIyMDMpkMN27cAABs2LABrVu3rvPrPT09kZqaarF6LHU+S9dVVxMnTjR/Kf9m4MKFC5DJZMjNzZW6FCKLYBAisjKrV6+Gk5MTqqqqDMdu3boFBwcHDBkyxGhsTbg5d+6cyFWK717B7dChQ3jhhRdEr2fFihXYsGGD6O9LRJbFIERkZYYOHYpbt27h8OHDhmN79+6Fm5sbfvrpJ/zxxx+G47t27YKHhwe6d+8uRalWoV27dmjRooXo76tWq82aUSMi68QgRGRlHn74YXTo0AEZGRmGYxkZGRgzZgy6du2KgwcPGh0fOnQoAODTTz9FYGAgnJyc4ObmhmeeeQZFRUVmvfd3332HoKAgqFQquLi44Iknnrjn2Pz8fIwZMwatWrWCs7Mznn76aRQWFtb7fP/4xz/QunVraDSaWs9lZGQgLi4OJSUlkMlkkMlkeOuttwDUvjQmk8mwZs0aPPbYY2jRogV69+6NzMxMnD17FkOGDEHLli0RGhpaaxbtm2++Qf/+/aFSqdCtWzfMmzfPaFbuf/3vpbEhQ4Zg2rRpeP3119GmTRu4ubkZaryXjIwMDBgwAC1btkTr1q0xaNAg/Pbbb3Wu6caNG3jxxRfh6uoKlUoFHx8fbNu2zfD8V199hT59+kCpVMLT0xPLli0zen9PT08sWrQIkyZNgpOTEzw8PGrtWp+VlYV+/fpBpVIhMDAQR44cMXr++vXrmDBhAtq1awdHR0d4eXlh/fr19/3cRNaEQYjICg0dOhS7du0y/L5r1y4MGTIE4eHhhuO3b9/GTz/9ZAhClZWVWLBgAY4ePYqtW7fiwoULmDhxYp3fc/v27XjiiSfw17/+FUeOHIFGo8GAAQNMjtXr9RgzZgx+//137N69G+np6fj1118RHR1dr/O98847mDVrFn744QcMHz681vOhoaFITU2Fs7Mzrly5gitXruC1116752dZsGABYmJikJubi169euGZZ57Biy++iMTERBw+fBiCICA+Pt4wfu/evYiJicH06dNx/PhxrFmzBhs2bMDChQvr+vUBADZu3IiWLVvip59+wjvvvIP58+cjPT3d5NiqqipERUUhPDwcP//8MzIzM/HCCy9AJpPVqSa9Xo+RI0di//79+Oyzz3D8+HEsXrwYdnZ2AIDs7Gw8/fTTGDduHI4dO4a33noLc+fOrXU5b9myZYaA8/e//x1TpkzBqVOnANy5JPvYY4/B29sb2dnZeOutt2p973PnzsXx48fx/fff48SJE/jwww/h4uJi1vdGJCmz96snoka3du1aoWXLlkJlZaWg0+kEe3t7oaioSNi0aZMQFhYmCIIgaDQaAYDw22+/mTzHoUOHBADCzZs3BUEQhF27dgkAhOvXrwuCIAjr168X1Gq1YXxISIgwYcKEe9bUpUsX4b333hMEQRB++OEHwc7OTsjPzzc8/8svvwgAhKysLLPO9/rrrwsdOnQQ8vLy7vud/G+9puoSBEEAIMyZM8fwe2ZmpgBAWLduneHYF198IahUKsPvw4cPFxYtWmR03k8//VTo0KHDPeuJjY0VxowZY/g9PDxcGDx4sNGYoKAg4Y033jD5+mvXrgkAhIyMDJPPP6imf//734JcLhdOnTpl8vXPPPOM8Je//MXo2MyZMwVvb2/D7126dBGeffZZw+96vV5o37698OGHHwqCIAhr1qwR2rZtK9y+fdsw5sMPPxQACEeOHBEEQRBGjx4txMXFmayBqCngjBCRFRoyZAhKS0tx6NAh7N27Fz179kS7du0QHh5u6BPKyMhAt27d4OHhAeDODMDo0aPh4eEBJycnhIeHA7hzCasucnNzTc7GmHLixAm4u7vD3d3dcMzb2xutW7fGiRMn6ny+ZcuWYe3atdi3bx/69OlTp/eui759+xp+dnV1BQD4+voaHfvjjz+g0+kAAEePHsX8+fPRqlUrw2Py5Mm4cuUKysrK6vW+ANChQ4d7Xp5s06YNJk6ciMjISIwePRorVqzAlStXDM8/qKbc3Fx07twZPXv2NHn+EydOYNCgQUbHBg0ahDNnzqC6utpkzTKZDG5uboaaT5w4gb59+0KlUhnGhISEGJ1zypQp2Lx5M/z9/fH666/jwIED9/uKiKwOgxCRFerRowc6d+6MXbt2YdeuXYZQ07FjR7i7u+PAgQPYtWsXhg0bBgAoLS1FZGQknJ2d8fnnn+PQoUP4+uuvAQAVFRV1ek9HR0eLfoa6nO+RRx5BdXU1vvzyS4u+t4ODg+HnmktNpo7p9XoAdy4BzZs3D7m5uYbHsWPHcObMGaMQYM771rxPzXuYsn79emRmZiI0NBRpaWno2bOnoQfsQTVZ6n8vc2v+XyNHjsRvv/2GV155BZcvX8bw4cPve9mSyNowCBFZqaFDhyIjIwMZGRlGt82HhYXh+++/R1ZWlqE/6OTJk7h27RoWL16MRx55BL169TK7Ubpv374mG5VN6d27NwoKClBQUGA4dvz4cdy4cQPe3t51Pt+AAQPw/fffY9GiRVi6dOl9xyoUCqOZDEvq378/Tp06hR49etR6yOWN+6/Jfv36ITExEQcOHICPjw82bdpUp5r69u2Lixcv4vTp0ybP27t3b+zfv9/o2P79+9GzZ09DH9GD9O7dGz///LPRnYr/3axfo127doiNjcVnn32G1NTUWg3XRNbMXuoCiMi0oUOHYurUqaisrDTMCAFAeHg44uPjUVFRYQhCHh4eUCgU+OCDD/DSSy8hLy8PCxYsMOv9kpOTMXz4cHTv3h3jxo1DVVUVduzYgTfeeKPW2IiICPj6+mLChAlITU1FVVUV/v73vyM8PByBgYFmnS80NBQ7duzAyJEjYW9vjxkzZpisz9PTE7du3YJGo4Gfnx9atGhhsdvmk5KS8Nhjj8HDwwP/7//9P8jlchw9ehR5eXl4++23LfIe/+v8+fP46KOP8Pjjj6Njx444deoUzpw5g5iYmDrVFB4ejrCwMIwdOxbLly9Hjx49cPLkSchkMowYMQKvvvoqgoKCsGDBAkRHRyMzMxMrV67E//3f/9W5xmeeeQZvvvkmJk+ejMTERFy4cKFWYE1KSkJAQAD69OmD8vJybNu2Db1797bod0XUmDgjRGSlhg4ditu3b6NHjx6GPhfgThC6efOm4TZ74M5/kW/YsAFbtmyBt7c3Fi9e/MAZlv81ZMgQbNmyBd9++y38/f0xbNgwZGVlmRwrk8nwzTff4KGHHkJYWBgiIiLQrVs3pKWl1et8gwcPxvbt2zFnzhx88MEHJseEhobipZdeQnR0NNq1a4d33nnHrM93P5GRkdi2bRt++OEHBAUFYeDAgXjvvffQpUsXi73H/2rRogVOnjyJsWPHomfPnnjhhRcwdepUvPjii3Wu6auvvkJQUBDGjx8Pb29vvP7664ZZs/79++PLL7/E5s2b4ePjg6SkJMyfP9+sOwlbtWqF7777DseOHUO/fv3w5ptvYsmSJUZjFAoFEhMT0bdvX4SFhcHOzg6bN29u+BdEJBKZIAiC1EUQERERSYEzQkRERGSzGISIiIjIZjEIERERkc1iECIiIiKbxSBERERENotBiIiIiGwWgxARERHZLAYhIiIislkMQkRERGSzGISIiIjIZjEIERERkc36/2BmNJKCJBmEAAAAAElFTkSuQmCC", "text/plain": [ - "
" + "
" ] }, - "metadata": { - "needs_background": "light" - }, + "metadata": {}, "output_type": "display_data" } ], @@ -356,9 +354,9 @@ ], "metadata": { "kernelspec": { - "display_name": "dask", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "dask" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -370,7 +368,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.9" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/examples/03_pytorch_mnist_hpo.py b/examples/03_pytorch_mnist_hpo.py index c7eef27..14fba45 100644 --- a/examples/03_pytorch_mnist_hpo.py +++ b/examples/03_pytorch_mnist_hpo.py @@ -6,7 +6,7 @@ this space can be passed to an object of class Model() which can instantiate a CNN architecture from it. The objective_function() is the target function that DEHB minimizes for this problem. This function instantiates an architecture, an optimizer, as defined by a configuration and performs the -training and evaluation (on the validation set) as per the budget passed. +training and evaluation (on the validation set) as per the fidelity passed. The argument `runtime` can be passed to DEHB as a wallclock budget for running the optimisation. This tutorial also briefly refers to the different methods of interfacing DEHB with the Dask @@ -167,7 +167,7 @@ def evaluate(model, device, data_loader, acc=False): return loss -def train_and_evaluate(config, max_budget, verbose=False, **kwargs): +def train_and_evaluate(config, max_fidelity, verbose=False, **kwargs): device = kwargs["device"] batch_size = config["batch_size"] train_set = kwargs["train_set"] @@ -176,7 +176,7 @@ def train_and_evaluate(config, max_budget, verbose=False, **kwargs): test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False) model = Model(config).to(device) optimizer = optim.Adadelta(model.parameters(), lr=config["lr"]) - for epoch in range(1, int(max_budget)+1): + for epoch in range(1, int(max_fidelity)+1): train(model, device, train_loader, optimizer) accuracy = evaluate(model, device, test_loader, acc=True) if verbose: @@ -184,7 +184,7 @@ def train_and_evaluate(config, max_budget, verbose=False, **kwargs): return accuracy -def objective_function(config, budget, **kwargs): +def objective_function(config, fidelity, **kwargs): """ The target function to minimize for HPO""" device = kwargs["device"] @@ -204,7 +204,7 @@ def objective_function(config, budget, **kwargs): optimizer = optim.Adadelta(model.parameters(), lr=config["lr"]) start = time.time() # measuring wallclock time - for epoch in range(1, int(budget)+1): + for epoch in range(1, int(fidelity)+1): train(model, device, train_loader, optimizer) loss = evaluate(model, device, valid_loader) cost = time.time() - start @@ -216,7 +216,7 @@ def objective_function(config, budget, **kwargs): res = { "fitness": loss, "cost": cost, - "info": {"test_loss": test_loss, "budget": budget} + "info": {"test_loss": test_loss, "fidelity": fidelity} } return res @@ -228,11 +228,11 @@ def input_arguments(): parser.add_argument('--seed', type=int, default=123, metavar='S', help='random seed (default: 123)') parser.add_argument('--refit_training', action='store_true', default=False, - help='Refit with incumbent configuration on full training data and budget') - parser.add_argument('--min_budget', type=float, default=None, - help='Minimum budget (epoch length)') - parser.add_argument('--max_budget', type=float, default=None, - help='Maximum budget (epoch length)') + help='Refit with incumbent configuration on full training data and fidelity') + parser.add_argument('--min_fidelity', type=float, default=None, + help='Minimum fidelity (epoch length)') + parser.add_argument('--max_fidelity', type=float, default=None, + help='Maximum fidelity (epoch length)') parser.add_argument('--eta', type=int, default=3, help='Parameter for Hyperband controlling early stopping aggressiveness') parser.add_argument('--output_path', type=str, default="./pytorch_mnist_dehb", @@ -250,7 +250,7 @@ def input_arguments(): parser.add_argument('--verbose', action="store_true", default=False, help='Decides verbosity of DEHB optimization') parser.add_argument('--runtime', type=float, default=300, - help='Total time in seconds as budget to run DEHB') + help='Total time in seconds as fidelity to run DEHB') args = parser.parse_args() return args @@ -300,8 +300,8 @@ def main(): # DEHB optimisation block # ########################### np.random.seed(args.seed) - dehb = DEHB(f=objective_function, cs=cs, dimensions=dimensions, min_budget=args.min_budget, - max_budget=args.max_budget, eta=args.eta, output_path=args.output_path, + dehb = DEHB(f=objective_function, cs=cs, dimensions=dimensions, min_fidelity=args.min_fidelity, + max_fidelity=args.max_fidelity, eta=args.eta, output_path=args.output_path, # if client is not None and of type Client, n_workers is ignored # if client is None, a Dask client with n_workers is set up client=client, n_workers=args.n_workers) @@ -325,7 +325,7 @@ def main(): root='./data', train=True, download=True, transform=transform ) incumbent = dehb.vector_to_configspace(dehb.inc_config) - acc = train_and_evaluate(incumbent, args.max_budget, verbose=True, + acc = train_and_evaluate(incumbent, args.max_fidelity, verbose=True, train_set=train_set, test_set=test_set, device=device) dehb.logger.info("Test accuracy of {:.3f} for the best found configuration: ".format(acc)) dehb.logger.info(incumbent) diff --git a/src/dehb/optimizers/de.py b/src/dehb/optimizers/de.py index d1227f5..d1c40a2 100644 --- a/src/dehb/optimizers/de.py +++ b/src/dehb/optimizers/de.py @@ -1,17 +1,20 @@ import os -import numpy as np +from typing import List + import ConfigSpace import ConfigSpace.util -from typing import List +import numpy as np from distributed import Client +from ..utils import ConfigRepository + class DEBase(): '''Base class for Differential Evolution ''' def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=None, - mutation_factor=None, crossover_prob=None, strategy=None, budget=None, - boundary_fix_type='random', **kwargs): + mutation_factor=None, crossover_prob=None, strategy=None, + boundary_fix_type='random', config_repository=None, **kwargs): # Benchmark related variables self.cs = cs self.f = f @@ -26,7 +29,6 @@ def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=None self.mutation_factor = mutation_factor self.crossover_prob = crossover_prob self.strategy = strategy - self.budget = budget self.fix_type = boundary_fix_type # Miscellaneous @@ -39,18 +41,28 @@ def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=None self.output_path = kwargs['output_path'] if 'output_path' in kwargs else './' os.makedirs(self.output_path, exist_ok=True) + if config_repository: + self.config_repository = config_repository + else: + self.config_repository = ConfigRepository() + # Global trackers - self.inc_score = np.inf - self.inc_config = None - self.population = None - self.fitness = None - self.age = None - self.history = [] + self.inc_score : float + self.inc_config : np.ndarray[float] + self.inc_id : int + self.population : np.ndarray[np.ndarray[float]] + self.population_ids :np.ndarray[int] + self.fitness : np.ndarray[float] + self.age : int + self.history : list[object] + self.reset() def reset(self): self.inc_score = np.inf self.inc_config = None + self.inc_id = -1 self.population = None + self.population_ids = None self.fitness = None self.age = None self.history = [] @@ -95,6 +107,7 @@ def init_population(self, pop_size: int) -> List: else: # if no ConfigSpace representation available, uniformly sample from [0, 1] population = np.random.uniform(low=0.0, high=1.0, size=(pop_size, self.dimensions)) + return np.array(population) def sample_population(self, size: int = 3, alt_pop: List = None) -> List: @@ -118,7 +131,7 @@ def sample_population(self, size: int = 3, alt_pop: List = None) -> List: selection = np.random.choice(np.arange(len(self.population)), size, replace=False) return self.population[selection] - def boundary_check(self, vector: np.array) -> np.array: + def boundary_check(self, vector: np.ndarray) -> np.ndarray: ''' Checks whether each of the dimensions of the input vector are within [0, 1]. If not, values of those dimensions are replaced with the type of fix selected. @@ -143,7 +156,7 @@ def boundary_check(self, vector: np.array) -> np.array: vector[violations] = np.clip(vector[violations], a_min=0, a_max=1) return vector - def vector_to_configspace(self, vector: np.array) -> ConfigSpace.Configuration: + def vector_to_configspace(self, vector: np.ndarray) -> ConfigSpace.Configuration: '''Converts numpy array to ConfigSpace object Works when self.cs is a ConfigSpace object and the input vector is in the domain [0, 1]. @@ -181,7 +194,7 @@ def vector_to_configspace(self, vector: np.array) -> ConfigSpace.Configuration: ) return new_config - def configspace_to_vector(self, config: ConfigSpace.Configuration) -> np.array: + def configspace_to_vector(self, config: ConfigSpace.Configuration) -> np.ndarray: '''Converts ConfigSpace object to numpy array scaled to [0,1] Works when self.cs is a ConfigSpace object and the input config is a ConfigSpace object. @@ -231,10 +244,11 @@ def run(self): class DE(DEBase): def __init__(self, cs=None, f=None, dimensions=None, pop_size=20, max_age=np.inf, mutation_factor=None, crossover_prob=None, strategy='rand1_bin', - budget=None, encoding=False, dim_map=None, **kwargs): + encoding=False, dim_map=None, config_repository=None, **kwargs): super().__init__(cs=cs, f=f, dimensions=dimensions, pop_size=pop_size, max_age=max_age, mutation_factor=mutation_factor, crossover_prob=crossover_prob, - strategy=strategy, budget=budget, **kwargs) + strategy=strategy, config_repository=config_repository, + **kwargs) if self.strategy is not None: self.mutation_strategy = self.strategy.split('_')[0] self.crossover_strategy = self.strategy.split('_')[1] @@ -285,7 +299,7 @@ def map_to_original(self, vector): new_vector[i] = np.max(np.array(vector)[self.dim_map[i]]) return new_vector - def f_objective(self, x, budget=None, **kwargs): + def f_objective(self, x, fidelity=None, **kwargs): if self.f is None: raise NotImplementedError("An objective function needs to be passed.") if self.encoding: @@ -296,18 +310,19 @@ def f_objective(self, x, budget=None, **kwargs): else: # can insert custom scaling/transform function here config = x.copy() - if budget is not None: # to be used when called by multi-fidelity based optimizers - res = self.f(config, budget=budget, **kwargs) + if fidelity is not None: # to be used when called by multi-fidelity based optimizers + res = self.f(config, fidelity=fidelity, **kwargs) else: res = self.f(config, **kwargs) assert "fitness" in res assert "cost" in res return res - def init_eval_pop(self, budget=None, eval=True, **kwargs): + def init_eval_pop(self, fidelity=None, eval=True, **kwargs): '''Creates new population of 'pop_size' and evaluates individuals. ''' self.population = self.init_population(self.pop_size) + self.population_ids = self.config_repository.announce_population(self.population, fidelity) self.fitness = np.array([np.inf for i in range(self.pop_size)]) self.age = np.array([self.max_age] * self.pop_size) @@ -320,25 +335,29 @@ def init_eval_pop(self, budget=None, eval=True, **kwargs): for i in range(self.pop_size): config = self.population[i] - res = self.f_objective(config, budget, **kwargs) + config_id = self.population_ids[i] + res = self.f_objective(config, fidelity, **kwargs) self.fitness[i], cost = res["fitness"], res["cost"] info = res["info"] if "info" in res else dict() if self.fitness[i] < self.inc_score: self.inc_score = self.fitness[i] self.inc_config = config + self.inc_id = config_id + self.config_repository.tell_result(config_id, float(fidelity or 0), res["fitness"], res["cost"], info) traj.append(self.inc_score) runtime.append(cost) - history.append((config.tolist(), float(self.fitness[i]), float(budget or 0), info)) + history.append((config.tolist(), float(self.fitness[i]), float(fidelity or 0), info)) return traj, runtime, history - def eval_pop(self, population=None, budget=None, **kwargs): + def eval_pop(self, population=None, population_ids=None, fidelity=None, **kwargs): '''Evaluates a population If population=None, the current population's fitness will be evaluated If population!=None, this population will be evaluated ''' pop = self.population if population is None else population + pop_ids = self.population_ids if population_ids is None else population_ids pop_size = self.pop_size if population is None else len(pop) traj = [] runtime = [] @@ -347,7 +366,7 @@ def eval_pop(self, population=None, budget=None, **kwargs): costs = [] ages = [] for i in range(pop_size): - res = self.f_objective(pop[i], budget, **kwargs) + res = self.f_objective(pop[i], fidelity, **kwargs) fitness, cost = res["fitness"], res["cost"] info = res["info"] if "info" in res else dict() if population is None: @@ -355,9 +374,11 @@ def eval_pop(self, population=None, budget=None, **kwargs): if fitness <= self.inc_score: self.inc_score = fitness self.inc_config = pop[i] + self.inc_id = pop_ids[i] + self.config_repository.tell_result(pop_ids[i], float(fidelity or 0), info) traj.append(self.inc_score) runtime.append(cost) - history.append((pop[i].tolist(), float(fitness), float(budget or 0), info)) + history.append((pop[i].tolist(), float(fitness), float(fidelity or 0), info)) fitnesses.append(fitness) costs.append(cost) ages.append(self.max_age) @@ -463,7 +484,7 @@ def crossover(self, target, mutant): offspring = self.crossover_exp(target, mutant) return offspring - def selection(self, trials, budget=None, **kwargs): + def selection(self, trials, trial_ids, fidelity=None, **kwargs): '''Carries out a parent-offspring competition given a set of trial population ''' traj = [] @@ -471,13 +492,16 @@ def selection(self, trials, budget=None, **kwargs): history = [] for i in range(len(trials)): # evaluation of the newly created individuals - res = self.f_objective(trials[i], budget, **kwargs) + res = self.f_objective(trials[i], fidelity, **kwargs) fitness, cost = res["fitness"], res["cost"] info = res["info"] if "info" in res else dict() + # log result to config repo + self.config_repository.tell_result(trial_ids[i], float(fidelity or 0), fitness, cost, info) # selection -- competition between parent[i] -- child[i] ## equality is important for landscape exploration if fitness <= self.fitness[i]: self.population[i] = trials[i] + self.population_ids[i] = trial_ids[i] self.fitness[i] = fitness # resetting age since new individual in the population self.age[i] = self.max_age @@ -488,23 +512,28 @@ def selection(self, trials, budget=None, **kwargs): if self.fitness[i] < self.inc_score: self.inc_score = self.fitness[i] self.inc_config = self.population[i] + self.inc_id = self.population[i] traj.append(self.inc_score) runtime.append(cost) - history.append((trials[i].tolist(), float(fitness), float(budget or 0), info)) + history.append((trials[i].tolist(), float(fitness), float(fidelity or 0), info)) return traj, runtime, history - def evolve_generation(self, budget=None, best=None, alt_pop=None, **kwargs): + def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs): '''Performs a complete DE evolution: mutation -> crossover -> selection ''' trials = [] + trial_ids = [] for j in range(self.pop_size): target = self.population[j] donor = self.mutation(current=target, best=best, alt_pop=alt_pop) trial = self.crossover(target, donor) trial = self.boundary_check(trial) + trial_id = self.config_repository.announce_config(trial, float(fidelity or 0)) trials.append(trial) + trial_ids.append(trial_id) trials = np.array(trials) - traj, runtime, history = self.selection(trials, budget, **kwargs) + trial_ids = np.array(trial_ids) + traj, runtime, history = self.selection(trials, trial_ids, fidelity, **kwargs) return traj, runtime, history def sample_mutants(self, size, population=None): @@ -525,20 +554,20 @@ def sample_mutants(self, size, population=None): return mutants - def run(self, generations=1, verbose=False, budget=None, reset=True, **kwargs): + def run(self, generations=1, verbose=False, fidelity=None, reset=True, **kwargs): # checking if a run exists if not hasattr(self, 'traj') or reset: self.reset() if verbose: print("Initializing and evaluating new population...") - self.traj, self.runtime, self.history = self.init_eval_pop(budget=budget, **kwargs) + self.traj, self.runtime, self.history = self.init_eval_pop(fidelity=fidelity, **kwargs) if verbose: print("Running evolutionary search...") for i in range(generations): if verbose: print("Generation {:<2}/{:<2} -- {:<0.7}".format(i+1, generations, self.inc_score)) - traj, runtime, history = self.evolve_generation(budget=budget, **kwargs) + traj, runtime, history = self.evolve_generation(fidelity=fidelity, **kwargs) self.traj.extend(traj) self.runtime.extend(runtime) self.history.extend(history) @@ -552,7 +581,7 @@ def run(self, generations=1, verbose=False, budget=None, reset=True, **kwargs): class AsyncDE(DE): def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=np.inf, mutation_factor=None, crossover_prob=None, strategy='rand1_bin', - budget=None, async_strategy='immediate', **kwargs): + async_strategy='immediate', config_repository=None, **kwargs): '''Extends DE to be Asynchronous with variations Parameters @@ -571,7 +600,8 @@ def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=np.i ''' super().__init__(cs=cs, f=f, dimensions=dimensions, pop_size=pop_size, max_age=max_age, mutation_factor=mutation_factor, crossover_prob=crossover_prob, - strategy=strategy, budget=budget, **kwargs) + strategy=strategy, config_repository=config_repository, + **kwargs) if self.strategy is not None: self.mutation_strategy = self.strategy.split('_')[0] self.crossover_strategy = self.strategy.split('_')[1] @@ -642,8 +672,9 @@ def _sample_population(self, size=3, alt_pop=None, target=None): selection = np.random.choice(np.arange(len(population)), size, replace=False) return population[selection] - def eval_pop(self, population=None, budget=None, **kwargs): + def eval_pop(self, population=None, population_ids=None, fidelity=None, **kwargs): pop = self.population if population is None else population + pop_ids = self.population_ids if population_ids is None else population_ids pop_size = self.pop_size if population is None else len(pop) traj = [] runtime = [] @@ -652,7 +683,7 @@ def eval_pop(self, population=None, budget=None, **kwargs): costs = [] ages = [] for i in range(pop_size): - res = self.f_objective(pop[i], budget, **kwargs) + res = self.f_objective(pop[i], fidelity, **kwargs) fitness, cost = res["fitness"], res["cost"] info = res["info"] if "info" in res else dict() if population is None: @@ -660,9 +691,11 @@ def eval_pop(self, population=None, budget=None, **kwargs): if fitness <= self.inc_score: self.inc_score = fitness self.inc_config = pop[i] + self.inc_id = pop_ids[i] + self.config_repository.tell_result(pop_ids[i], float(fidelity or 0), fitness, cost, info) traj.append(self.inc_score) runtime.append(cost) - history.append((pop[i].tolist(), float(fitness), float(budget or 0), info)) + history.append((pop[i].tolist(), float(fitness), float(fidelity or 0), info)) fitnesses.append(fitness) costs.append(cost) ages.append(self.max_age) @@ -723,40 +756,46 @@ def sample_mutants(self, size, population=None): return mutants - def evolve_generation(self, budget=None, best=None, alt_pop=None, **kwargs): + def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs): '''Performs a complete DE evolution, mutation -> crossover -> selection ''' traj = [] runtime = [] history = [] - if self.async_strategy == 'deferred': + if self.async_strategy == "deferred": trials = [] + trial_ids = [] for j in range(self.pop_size): target = self.population[j] donor = self.mutation(current=target, best=best, alt_pop=alt_pop) trial = self.crossover(target, donor) trial = self.boundary_check(trial) + trial_id = self.config_repository.announce_config(trial, float(fidelity or 0)) trials.append(trial) + trial_ids.append(trial_id) # selection takes place on a separate trial population only after # one iteration through the population has taken place trials = np.array(trials) - traj, runtime, history = self.selection(trials, budget, **kwargs) + traj, runtime, history = self.selection(trials, trial_ids, fidelity, **kwargs) return traj, runtime, history - elif self.async_strategy == 'immediate': + elif self.async_strategy == "immediate": for i in range(self.pop_size): target = self.population[i] donor = self.mutation(current=target, best=best, alt_pop=alt_pop) trial = self.crossover(target, donor) trial = self.boundary_check(trial) + trial_id = self.config_repository.announce_config(trial, float(fidelity or 0)) # evaluating a single trial population for the i-th individual de_traj, de_runtime, de_history, fitnesses, costs = \ - self.eval_pop(trial.reshape(1, self.dimensions), budget=budget, **kwargs) + self.eval_pop(trial.reshape(1, self.dimensions), + np.array([trial_id]), fidelity=fidelity, **kwargs) # one-vs-one selection ## can replace the i-the population despite not completing one iteration if fitnesses[0] <= self.fitness[i]: self.population[i] = trial + self.population_ids[i] = trial_id self.fitness[i] = fitnesses[0] traj.extend(de_traj) runtime.extend(de_runtime) @@ -766,7 +805,7 @@ def evolve_generation(self, budget=None, best=None, alt_pop=None, **kwargs): else: # async_strategy == 'random' or async_strategy == 'worst': for count in range(self.pop_size): # choosing target individual - if self.async_strategy == 'random': + if self.async_strategy == "random": i = np.random.choice(np.arange(self.pop_size)) else: # async_strategy == 'worst' i = np.argsort(-self.fitness)[0] @@ -774,9 +813,11 @@ def evolve_generation(self, budget=None, best=None, alt_pop=None, **kwargs): mutant = self.mutation(current=target, best=best, alt_pop=alt_pop) trial = self.crossover(target, mutant) trial = self.boundary_check(trial) + trial_id = self.config_repository.announce_config(trial, float(fidelity or 0)) # evaluating a single trial population for the i-th individual de_traj, de_runtime, de_history, fitnesses, costs = \ - self.eval_pop(trial.reshape(1, self.dimensions), budget=budget, **kwargs) + self.eval_pop(trial.reshape(1, self.dimensions), np.array([trial_id]), + fidelity=fidelity, **kwargs) # one-vs-one selection ## can replace the i-the population despite not completing one iteration if fitnesses[0] <= self.fitness[i]: @@ -788,22 +829,21 @@ def evolve_generation(self, budget=None, best=None, alt_pop=None, **kwargs): return traj, runtime, history - def run(self, generations=1, verbose=False, budget=None, reset=True, **kwargs): + def run(self, generations=1, verbose=False, fidelity=None, reset=True, **kwargs): # checking if a run exists - if not hasattr(self, 'traj') or reset: + if not hasattr(self, "traj") or reset: self.reset() if verbose: print("Initializing and evaluating new population...") - self.traj, self.runtime, self.history = self.init_eval_pop(budget=budget, **kwargs) + self.traj, self.runtime, self.history = self.init_eval_pop(fidelity=fidelity, **kwargs) if verbose: print("Running evolutionary search...") for i in range(generations): if verbose: print("Generation {:<2}/{:<2} -- {:<0.7}".format(i+1, generations, self.inc_score)) - traj, runtime, history = self.evolve_generation( - budget=budget, best=self.inc_config, **kwargs - ) + traj, runtime, history = self.evolve_generation(fidelity=fidelity, + best=self.inc_config, **kwargs) self.traj.extend(traj) self.runtime.extend(runtime) self.history.extend(history) diff --git a/src/dehb/optimizers/dehb.py b/src/dehb/optimizers/dehb.py index b36e269..612b496 100644 --- a/src/dehb/optimizers/dehb.py +++ b/src/dehb/optimizers/dehb.py @@ -12,6 +12,7 @@ from .de import DE, AsyncDE from ..utils import SHBracketManager +from ..utils import ConfigRepository logger.configure(handlers=[{"sink": sys.stdout, "level": "INFO"}]) @@ -24,11 +25,12 @@ class DEHBBase: def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, - crossover_prob=None, strategy=None, min_budget=None, - max_budget=None, eta=None, min_clip=None, max_clip=None, + crossover_prob=None, strategy=None, min_fidelity=None, + max_fidelity=None, eta=None, min_clip=None, max_clip=None, boundary_fix_type='random', max_age=np.inf, **kwargs): # Miscellaneous self._setup_logger(kwargs) + self.config_repository = ConfigRepository() # Benchmark related variables self.cs = cs @@ -60,11 +62,11 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, } # Hyperband related variables - self.min_budget = min_budget - self.max_budget = max_budget - if self.max_budget <= self.min_budget: - self.logger.error("Only (Max Budget > Min Budget) is supported for DEHB.") - if self.max_budget == self.min_budget: + self.min_fidelity = min_fidelity + self.max_fidelity = max_fidelity + if self.max_fidelity <= self.min_fidelity: + self.logger.error("Only (Max Fidelity > Min Fidelity) is supported for DEHB.") + if self.max_fidelity == self.min_fidelity: self.logger.error( "If you have a fixed fidelity, " \ "you can instead run DE. For more information checkout: " \ @@ -74,14 +76,14 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, self.min_clip = min_clip self.max_clip = max_clip - # Precomputing budget spacing and number of configurations for HB iterations + # Precomputing fidelity spacing and number of configurations for HB iterations self.max_SH_iter = None - self.budgets = None - if self.min_budget is not None and \ - self.max_budget is not None and \ + self.fidelities = None + if self.min_fidelity is not None and \ + self.max_fidelity is not None and \ self.eta is not None: - self.max_SH_iter = -int(np.log(self.min_budget / self.max_budget) / np.log(self.eta)) + 1 - self.budgets = self.max_budget * np.power(self.eta, + self.max_SH_iter = -int(np.log(self.min_fidelity / self.max_fidelity) / np.log(self.eta)) + 1 + self.fidelities = self.max_fidelity * np.power(self.eta, -np.linspace(start=self.max_SH_iter - 1, stop=0, num=self.max_SH_iter)) @@ -124,7 +126,7 @@ def init_population(self): def get_next_iteration(self, iteration): '''Computes the Successive Halving spacing - Given the iteration index, computes the budget spacing to be used and + Given the iteration index, computes the fidelity spacing to be used and the number of configurations to be used for the SH iterations. Parameters @@ -137,12 +139,12 @@ def get_next_iteration(self, iteration): Returns ------- ns : array - budgets : array + fidelities : array ''' # number of 'SH runs' s = self.max_SH_iter - 1 - (iteration % self.max_SH_iter) - # budget spacing for this iteration - budgets = self.budgets[(-s-1):] + # fidelity spacing for this iteration + fidelities = self.fidelities[(-s-1):] # number of configurations in that bracket n0 = int(np.floor((self.max_SH_iter)/(s+1)) * self.eta**s) ns = [max(int(n0*(self.eta**(-i))), 1) for i in range(s+1)] @@ -151,7 +153,7 @@ def get_next_iteration(self, iteration): elif self.min_clip is not None: ns = np.clip(ns, a_min=self.min_clip, a_max=np.max(ns)) - return ns, budgets + return ns, fidelities def get_incumbents(self): """ Returns a tuple of the (incumbent configuration, incumbent score/fitness). """ @@ -168,13 +170,13 @@ def run(self): class DEHB(DEHBBase): def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=0.5, - crossover_prob=0.5, strategy='rand1_bin', min_budget=None, - max_budget=None, eta=3, min_clip=None, max_clip=None, configspace=True, + crossover_prob=0.5, strategy='rand1_bin', min_fidelity=None, + max_fidelity=None, eta=3, min_clip=None, max_clip=None, configspace=True, boundary_fix_type='random', max_age=np.inf, n_workers=None, client=None, async_strategy="immediate", **kwargs): super().__init__(cs=cs, f=f, dimensions=dimensions, mutation_factor=mutation_factor, - crossover_prob=crossover_prob, strategy=strategy, min_budget=min_budget, - max_budget=max_budget, eta=eta, min_clip=min_clip, max_clip=max_clip, + crossover_prob=crossover_prob, strategy=strategy, min_fidelity=min_fidelity, + max_fidelity=max_fidelity, eta=eta, min_clip=min_clip, max_clip=max_clip, configspace=configspace, boundary_fix_type=boundary_fix_type, max_age=max_age, **kwargs) self.de_params.update({"async_strategy": async_strategy}) @@ -236,19 +238,21 @@ def _f_objective(self, job_info): # reprioritising a CUDA device order specific to this worker process os.environ.update({"CUDA_VISIBLE_DEVICES": job_info["gpu_devices"]}) - config, budget, parent_id = job_info['config'], job_info['budget'], job_info['parent_id'] - bracket_id = job_info['bracket_id'] + config, config_id = job_info["config"], job_info["config_id"] + fidelity, parent_id = job_info["fidelity"], job_info["parent_id"] + bracket_id = job_info["bracket_id"] kwargs = job_info["kwargs"] - res = self.de[budget].f_objective(config, budget, **kwargs) - info = res["info"] if "info" in res else dict() + res = self.de[fidelity].f_objective(config, fidelity, **kwargs) + info = res["info"] if "info" in res else {} run_info = { - 'fitness': res["fitness"], - 'cost': res["cost"], - 'config': config, - 'budget': budget, - 'parent_id': parent_id, - 'bracket_id': bracket_id, - 'info': info + "fitness": res["fitness"], + "cost": res["cost"], + "config": config, + "config_id": config_id, + "fidelity": fidelity, + "parent_id": parent_id, + "bracket_id": bracket_id, + "info": info, } if "gpu_devices" in job_info: @@ -295,13 +299,13 @@ def distribute_gpus(self): def vector_to_configspace(self, config): assert hasattr(self, "de") - assert len(self.budgets) > 0 - return self.de[self.budgets[0]].vector_to_configspace(config) + assert len(self.fidelities) > 0 + return self.de[self.fidelities[0]].vector_to_configspace(config) def configspace_to_vector(self, config): assert hasattr(self, "de") - assert len(self.budgets) > 0 - return self.de[self.budgets[0]].configspace_to_vector(config) + assert len(self.fidelities) > 0 + return self.de[self.fidelities[0]].configspace_to_vector(config) def reset(self): super().reset() @@ -353,7 +357,7 @@ def _update_incumbents(self, config, score, info): self.inc_info = info def _get_pop_sizes(self): - """Determines maximum pop size for each budget + """Determines maximum pop size for each fidelity """ self._max_pop_size = {} for i in range(self.max_SH_iter): @@ -364,27 +368,30 @@ def _get_pop_sizes(self): ) if r_j in self._max_pop_size.keys() else n[j] def _init_subpop(self): - """ List of DE objects corresponding to the budgets (fidelities) + """ List of DE objects corresponding to the fidelities """ self.de = {} - for i, b in enumerate(self._max_pop_size.keys()): - self.de[b] = AsyncDE(**self.de_params, budget=b, pop_size=self._max_pop_size[b]) - self.de[b].population = self.de[b].init_population(pop_size=self._max_pop_size[b]) - self.de[b].fitness = np.array([np.inf] * self._max_pop_size[b]) + for i, f in enumerate(self._max_pop_size.keys()): + self.de[f] = AsyncDE(**self.de_params, pop_size=self._max_pop_size[f], + config_repository=self.config_repository) + self.de[f].population = self.de[f].init_population(pop_size=self._max_pop_size[f]) + self.de[f].population_ids = self.config_repository.announce_population(self.de[f].population, f) + self.de[f].fitness = np.array([np.inf] * self._max_pop_size[f]) # adding attributes to DEHB objects to allow communication across subpopulations - self.de[b].parent_counter = 0 - self.de[b].promotion_pop = None - self.de[b].promotion_fitness = None + self.de[f].parent_counter = 0 + self.de[f].promotion_pop = None + self.de[f].promotion_pop_ids = None + self.de[f].promotion_fitness = None - def _concat_pops(self, exclude_budget=None): + def _concat_pops(self, exclude_fidelity=None): """ Concatenates all subpopulations """ - budgets = list(self.budgets) - if exclude_budget is not None: - budgets.remove(exclude_budget) + fidelities = list(self.fidelities) + if exclude_fidelity is not None: + fidelities.remove(exclude_fidelity) pop = [] - for budget in budgets: - pop.extend(self.de[budget].population.tolist()) + for fidelity in fidelities: + pop.extend(self.de[fidelity].population.tolist()) return np.array(pop) def _start_new_bracket(self): @@ -392,9 +399,9 @@ def _start_new_bracket(self): """ # start new bracket self.iteration_counter += 1 # iteration counter gives the bracket count or bracket ID - n_configs, budgets = self.get_next_iteration(self.iteration_counter) + n_configs, fidelities = self.get_next_iteration(self.iteration_counter) bracket = SHBracketManager( - n_configs=n_configs, budgets=budgets, bracket_id=self.iteration_counter + n_configs=n_configs, fidelities=fidelities, bracket_id=self.iteration_counter ) self.active_brackets.append(bracket) return bracket @@ -420,109 +427,122 @@ def is_worker_available(self, verbose=False): return False return True - def _get_promotion_candidate(self, low_budget, high_budget, n_configs): - """ Manages the population to be promoted from the lower to the higher budget. + def _get_promotion_candidate(self, low_fidelity, high_fidelity, n_configs): + """ Manages the population to be promoted from the lower to the higher fidelity. This is triggered or in action only during the first full HB bracket, which is equivalent to the number of brackets <= max_SH_iter. """ # finding the individuals that have been evaluated (fitness < np.inf) - evaluated_configs = np.where(self.de[low_budget].fitness != np.inf)[0] - promotion_candidate_pop = self.de[low_budget].population[evaluated_configs] - promotion_candidate_fitness = self.de[low_budget].fitness[evaluated_configs] + evaluated_configs = np.where(self.de[low_fidelity].fitness != np.inf)[0] + promotion_candidate_pop = self.de[low_fidelity].population[evaluated_configs] + promotion_candidate_pop_ids = self.de[low_fidelity].population_ids[evaluated_configs] + promotion_candidate_fitness = self.de[low_fidelity].fitness[evaluated_configs] # ordering the evaluated individuals based on their fitness values pop_idx = np.argsort(promotion_candidate_fitness) # creating population for promotion if none promoted yet or nothing to promote - if self.de[high_budget].promotion_pop is None or \ - len(self.de[high_budget].promotion_pop) == 0: - self.de[high_budget].promotion_pop = np.empty((0, self.dimensions)) - self.de[high_budget].promotion_fitness = np.array([]) - - # iterating over the evaluated individuals from the lower budget and including them - # in the promotion population for the higher budget only if it's not in the population + if self.de[high_fidelity].promotion_pop is None or \ + len(self.de[high_fidelity].promotion_pop) == 0: + self.de[high_fidelity].promotion_pop = np.empty((0, self.dimensions)) + self.de[high_fidelity].promotion_pop_ids = np.array([], dtype=np.int64) + self.de[high_fidelity].promotion_fitness = np.array([]) + + # iterating over the evaluated individuals from the lower fidelity and including them + # in the promotion population for the higher fidelity only if it's not in the population # this is done to ensure diversity of population and avoid redundant evaluations for idx in pop_idx: individual = promotion_candidate_pop[idx] - # checks if the candidate individual already exists in the high budget population - if np.any(np.all(individual == self.de[high_budget].population, axis=1)): + individual_id = promotion_candidate_pop_ids[idx] + # checks if the candidate individual already exists in the high fidelity population + if np.any(np.all(individual == self.de[high_fidelity].population, axis=1)): # skipping already present individual to allow diversity and reduce redundancy continue - self.de[high_budget].promotion_pop = np.append( - self.de[high_budget].promotion_pop, [individual], axis=0 + self.de[high_fidelity].promotion_pop = np.append( + self.de[high_fidelity].promotion_pop, [individual], axis=0 + ) + self.de[high_fidelity].promotion_pop_ids = np.append( + self.de[high_fidelity].promotion_pop_ids, individual_id ) - self.de[high_budget].promotion_fitness = np.append( - self.de[high_budget].promotion_pop, promotion_candidate_fitness[pop_idx] + self.de[high_fidelity].promotion_fitness = np.append( + self.de[high_fidelity].promotion_pop, promotion_candidate_fitness[pop_idx] ) # retaining only n_configs - self.de[high_budget].promotion_pop = self.de[high_budget].promotion_pop[:n_configs] - self.de[high_budget].promotion_fitness = \ - self.de[high_budget].promotion_fitness[:n_configs] - - if len(self.de[high_budget].promotion_pop) > 0: - config = self.de[high_budget].promotion_pop[0] + self.de[high_fidelity].promotion_pop = self.de[high_fidelity].promotion_pop[:n_configs] + self.de[high_fidelity].promotion_pop_ids = self.de[high_fidelity].promotion_pop_ids[:n_configs] + self.de[high_fidelity].promotion_fitness = \ + self.de[high_fidelity].promotion_fitness[:n_configs] + + if len(self.de[high_fidelity].promotion_pop) > 0: + config = self.de[high_fidelity].promotion_pop[0] + config_id = self.de[high_fidelity].promotion_pop_ids[0] # removing selected configuration from population - self.de[high_budget].promotion_pop = self.de[high_budget].promotion_pop[1:] - self.de[high_budget].promotion_fitness = self.de[high_budget].promotion_fitness[1:] + self.de[high_fidelity].promotion_pop = self.de[high_fidelity].promotion_pop[1:] + self.de[high_fidelity].promotion_pop_ids = self.de[high_fidelity].promotion_pop_ids[1:] + self.de[high_fidelity].promotion_fitness = self.de[high_fidelity].promotion_fitness[1:] else: - # in case of an edge failure case where all high budget individuals are same - # just choose the best performing individual from the lower budget (again) - config = self.de[low_budget].population[pop_idx[0]] - return config + # in case of an edge failure case where all high fidelity individuals are same + # just choose the best performing individual from the lower fidelity (again) + config = self.de[low_fidelity].population[pop_idx[0]] + config_id = self.de[low_fidelity].population_ids[pop_idx[0]] + return config, config_id - def _get_next_parent_for_subpop(self, budget): + def _get_next_parent_for_subpop(self, fidelity): """ Maintains a looping counter over a subpopulation, to iteratively select a parent """ - parent_id = self.de[budget].parent_counter - self.de[budget].parent_counter += 1 - self.de[budget].parent_counter = self.de[budget].parent_counter % self._max_pop_size[budget] + parent_id = self.de[fidelity].parent_counter + self.de[fidelity].parent_counter += 1 + self.de[fidelity].parent_counter = self.de[fidelity].parent_counter % self._max_pop_size[fidelity] return parent_id - def _acquire_config(self, bracket, budget): - """ Generates/chooses a configuration based on the budget and iteration number + def _acquire_config(self, bracket, fidelity): + """ Generates/chooses a configuration based on the fidelity and iteration number """ # select a parent/target - parent_id = self._get_next_parent_for_subpop(budget) - target = self.de[budget].population[parent_id] - # identify lower budget/fidelity to transfer information from - lower_budget, num_configs = bracket.get_lower_budget_promotions(budget) + parent_id = self._get_next_parent_for_subpop(fidelity) + target = self.de[fidelity].population[parent_id] + # identify lower fidelity to transfer information from + lower_fidelity, num_configs = bracket.get_lower_fidelity_promotions(fidelity) if self.iteration_counter < self.max_SH_iter: # promotions occur only in the first set of SH brackets under Hyperband - # for the first rung/budget in the current bracket, no promotion is possible and + # for the first rung/fidelity in the current bracket, no promotion is possible and # evolution can begin straight away - # for the subsequent rungs, individuals will be promoted from the lower_budget - if budget != bracket.budgets[0]: - # TODO: check if generalizes to all budget spacings - config = self._get_promotion_candidate(lower_budget, budget, num_configs) - return config, parent_id + # for the subsequent rungs, individuals will be promoted from the lower_fidelity + if fidelity != bracket.fidelities[0]: + # TODO: check if generalizes to all fidelity spacings + config, config_id = self._get_promotion_candidate(lower_fidelity, fidelity, num_configs) + return config, config_id, parent_id # DE evolution occurs when either all individuals in the subpopulation have been evaluated # at least once, i.e., has fitness < np.inf, which can happen if # iteration_counter <= max_SH_iter but certainly never when iteration_counter > max_SH_iter # a single DE evolution --- (mutation + crossover) occurs here - mutation_pop_idx = np.argsort(self.de[lower_budget].fitness)[:num_configs] - mutation_pop = self.de[lower_budget].population[mutation_pop_idx] - # generate mutants from previous budget subpopulation or global population - if len(mutation_pop) < self.de[budget]._min_pop_size: - filler = self.de[budget]._min_pop_size - len(mutation_pop) + 1 - new_pop = self.de[budget]._init_mutant_population( + mutation_pop_idx = np.argsort(self.de[lower_fidelity].fitness)[:num_configs] + mutation_pop = self.de[lower_fidelity].population[mutation_pop_idx] + # generate mutants from previous fidelity subpopulation or global population + if len(mutation_pop) < self.de[fidelity]._min_pop_size: + filler = self.de[fidelity]._min_pop_size - len(mutation_pop) + 1 + new_pop = self.de[fidelity]._init_mutant_population( pop_size=filler, population=self._concat_pops(), target=target, best=self.inc_config ) mutation_pop = np.concatenate((mutation_pop, new_pop)) # generate mutant from among individuals in mutation_pop - mutant = self.de[budget].mutation( + mutant = self.de[fidelity].mutation( current=target, best=self.inc_config, alt_pop=mutation_pop ) # perform crossover with selected parent - config = self.de[budget].crossover(target=target, mutant=mutant) - config = self.de[budget].boundary_check(config) - return config, parent_id + config = self.de[fidelity].crossover(target=target, mutant=mutant) + config = self.de[fidelity].boundary_check(config) + + # announce new config + config_id = self.config_repository.announce_config(config, fidelity) + return config, config_id, parent_id def _get_next_job(self): - """ Loads a configuration and budget to be evaluated next by a free worker + """ Loads a configuration and fidelity to be evaluated next by a free worker """ bracket = None if len(self.active_brackets) == 0 or \ @@ -541,13 +561,14 @@ def _get_next_job(self): if bracket is None: # start new bracket when existing list has all waiting brackets bracket = self._start_new_bracket() - # budget that the SH bracket allots - budget = bracket.get_next_job_budget() - config, parent_id = self._acquire_config(bracket, budget) - # notifies the Bracket Manager that a single config is to run for the budget chosen + # fidelity that the SH bracket allots + fidelity = bracket.get_next_job_fidelity() + config, config_id, parent_id = self._acquire_config(bracket, fidelity) + # notifies the Bracket Manager that a single config is to run for the fidelity chosen job_info = { "config": config, - "budget": budget, + "config_id": config_id, + "fidelity": fidelity, "parent_id": parent_id, "bracket_id": bracket.bracket_id } @@ -570,7 +591,7 @@ def _get_gpu_id_with_low_load(self): return gpu_ids def submit_job(self, job_info, **kwargs): - """ Asks a free worker to run the objective function on config and budget + """ Asks a free worker to run the objective function on config and fidelity """ job_info["kwargs"] = self.shared_data if self.shared_data is not None else kwargs # submit to to Dask client @@ -589,7 +610,7 @@ def submit_job(self, job_info, **kwargs): for bracket in self.active_brackets: if bracket.bracket_id == job_info['bracket_id']: # registering is IMPORTANT for Bracket Manager to perform SH - bracket.register_job(job_info['budget']) + bracket.register_job(job_info['fidelity']) break def _fetch_results_from_workers(self): @@ -618,29 +639,32 @@ def _fetch_results_from_workers(self): # update bracket information fitness, cost = run_info["fitness"], run_info["cost"] info = run_info["info"] if "info" in run_info else dict() - budget, parent_id = run_info["budget"], run_info["parent_id"] - config = run_info["config"] + fidelity, parent_id = run_info["fidelity"], run_info["parent_id"] + config, config_id = run_info["config"], run_info["config_id"] bracket_id = run_info["bracket_id"] for bracket in self.active_brackets: if bracket.bracket_id == bracket_id: # bracket job complete - bracket.complete_job(budget) # IMPORTANT to perform synchronous SH + bracket.complete_job(fidelity) # IMPORTANT to perform synchronous SH + + self.config_repository.tell_result(config_id, fidelity, fitness, cost, info) # carry out DE selection - if fitness <= self.de[budget].fitness[parent_id]: - self.de[budget].population[parent_id] = config - self.de[budget].fitness[parent_id] = fitness + if fitness <= self.de[fidelity].fitness[parent_id]: + self.de[fidelity].population[parent_id] = config + self.de[fidelity].population_ids[parent_id] = config_id + self.de[fidelity].fitness[parent_id] = fitness # updating incumbents - if self.de[budget].fitness[parent_id] < self.inc_score: + if self.de[fidelity].fitness[parent_id] < self.inc_score: self._update_incumbents( - config=self.de[budget].population[parent_id], - score=self.de[budget].fitness[parent_id], + config=self.de[fidelity].population[parent_id], + score=self.de[fidelity].fitness[parent_id], info=info ) # book-keeping self._update_trackers( traj=self.inc_score, runtime=cost, history=( - config.tolist(), float(fitness), float(cost), float(budget), info + config.tolist(), float(fitness), float(cost), float(fidelity), info ) ) # remove processed future @@ -727,7 +751,7 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus """ Main interface to run optimization by DEHB This function waits on workers and if a worker is free, asks for a configuration and a - budget to evaluate on and submits it to the worker. In each loop, it checks if a job + fidelity to evaluate on and submits it to the worker. In each loop, it checks if a job is complete, fetches the results, carries the necessary processing of it asynchronously to the worker computations. @@ -763,7 +787,7 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus break if self.is_worker_available(): job_info = self._get_next_job() - if brackets is not None and job_info['bracket_id'] >= brackets: + if brackets is not None and job_info["bracket_id"] >= brackets: # ignore submission and only collect results # when brackets are chosen as run budget, an extra bracket is created # since iteration_counter is incremented in _get_next_job() and then checked @@ -780,11 +804,12 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus # submits job_info to a worker for execution self.submit_job(job_info, **kwargs) if verbose: - budget = job_info['budget'] + fidelity = job_info["fidelity"] + config_id = job_info["config_id"] self._verbosity_runtime(fevals, brackets, total_cost) self.logger.info( - "Evaluating a configuration with budget {} under " - "bracket ID {}".format(budget, job_info['bracket_id']) + "Evaluating configuration {} with fidelity {} under " + "bracket ID {}".format(config_id, fidelity, job_info["bracket_id"]) ) self.logger.info( "Best score seen/Incumbent score: {}".format(self.inc_score) diff --git a/src/dehb/utils/__init__.py b/src/dehb/utils/__init__.py index dd9d4f0..65bbe00 100644 --- a/src/dehb/utils/__init__.py +++ b/src/dehb/utils/__init__.py @@ -1 +1,2 @@ -from .bracket_manager import SHBracketManager \ No newline at end of file +from .bracket_manager import SHBracketManager +from .config_repository import ConfigRepository \ No newline at end of file diff --git a/src/dehb/utils/bracket_manager.py b/src/dehb/utils/bracket_manager.py index 6f2079e..2642223 100644 --- a/src/dehb/utils/bracket_manager.py +++ b/src/dehb/utils/bracket_manager.py @@ -4,93 +4,93 @@ class SHBracketManager(object): """ Synchronous Successive Halving utilities """ - def __init__(self, n_configs, budgets, bracket_id=None): - assert len(n_configs) == len(budgets) + def __init__(self, n_configs, fidelities, bracket_id=None): + assert len(n_configs) == len(fidelities) self.n_configs = n_configs - self.budgets = budgets + self.fidelities = fidelities self.bracket_id = bracket_id self.sh_bracket = {} self._sh_bracket = {} self._config_map = {} - for i, budget in enumerate(budgets): + for i, fidelity in enumerate(fidelities): # sh_bracket keeps track of jobs/configs that are still to be scheduled/allocatted # _sh_bracket keeps track of jobs/configs that have been run and results retrieved for # (sh_bracket[i] + _sh_bracket[i]) == n_configs[i] is when no jobs have been scheduled - # or all jobs for that budget/rung are over + # or all jobs for that fidelity/rung are over # (sh_bracket[i] + _sh_bracket[i]) < n_configs[i] indicates a job has been scheduled # and is queued/running and the bracket needs to be paused till results are retrieved - self.sh_bracket[budget] = n_configs[i] # each scheduled job does -= 1 - self._sh_bracket[budget] = 0 # each retrieved job does +=1 - self.n_rungs = len(budgets) + self.sh_bracket[fidelity] = n_configs[i] # each scheduled job does -= 1 + self._sh_bracket[fidelity] = 0 # each retrieved job does +=1 + self.n_rungs = len(fidelities) self.current_rung = 0 - def get_budget(self, rung=None): - """ Returns the exact budget that rung is pointing to. + def get_fidelity(self, rung=None): + """ Returns the exact fidelity that rung is pointing to. - Returns current rung's budget if no rung is passed. + Returns current rung's fidelity if no rung is passed. """ if rung is not None: - return self.budgets[rung] - return self.budgets[self.current_rung] + return self.fidelities[rung] + return self.fidelities[self.current_rung] - def get_lower_budget_promotions(self, budget): - """ Returns the immediate lower budget and the number of configs to be promoted from there + def get_lower_fidelity_promotions(self, fidelity): + """ Returns the immediate lower fidelity and the number of configs to be promoted from there """ - assert budget in self.budgets - rung = np.where(budget == self.budgets)[0][0] + assert fidelity in self.fidelities + rung = np.where(fidelity == self.fidelities)[0][0] prev_rung = np.clip(rung - 1, a_min=0, a_max=self.n_rungs-1) - lower_budget = self.budgets[prev_rung] + lower_fidelity = self.fidelities[prev_rung] num_promote_configs = self.n_configs[rung] - return lower_budget, num_promote_configs + return lower_fidelity, num_promote_configs - def get_next_job_budget(self): - """ Returns the budget that will be selected if current_rung is incremented by 1 + def get_next_job_fidelity(self): + """ Returns the fidelity that will be selected if current_rung is incremented by 1 """ - if self.sh_bracket[self.get_budget()] > 0: + if self.sh_bracket[self.get_fidelity()] > 0: # the current rung still has unallocated jobs (>0) - return self.get_budget() + return self.get_fidelity() else: # the current rung has no more jobs to allocate, increment it rung = (self.current_rung + 1) % self.n_rungs - if self.sh_bracket[self.get_budget(rung)] > 0: + if self.sh_bracket[self.get_fidelity(rung)] > 0: # the incremented rung has unallocated jobs (>0) - return self.get_budget(rung) + return self.get_fidelity(rung) else: # all jobs for this bracket has been allocated/bracket is complete - # no more budgets to evaluate and can return None + # no more fidelities to evaluate and can return None pass return None - def register_job(self, budget): - """ Registers the allocation of a configuration for the budget and updates current rung + def register_job(self, fidelity): + """ Registers the allocation of a configuration for the fidelity and updates current rung This function must be called when scheduling a job in order to allow the bracket manager - to continue job and budget allocation without waiting for jobs to finish and return + to continue job and fidelity allocation without waiting for jobs to finish and return results necessarily. This feature can be leveraged to run brackets asynchronously. """ - assert budget in self.budgets - assert self.sh_bracket[budget] > 0 - self.sh_bracket[budget] -= 1 + assert fidelity in self.fidelities + assert self.sh_bracket[fidelity] > 0 + self.sh_bracket[fidelity] -= 1 if not self._is_rung_pending(self.current_rung): # increment current rung if no jobs left in the rung self.current_rung = (self.current_rung + 1) % self.n_rungs - def complete_job(self, budget): - """ Notifies the bracket that a job for a budget has been completed + def complete_job(self, fidelity): + """ Notifies the bracket that a job for a fidelity has been completed - This function must be called when a config for a budget has finished evaluation to inform + This function must be called when a config for a fidelity has finished evaluation to inform the Bracket Manager that no job needs to be waited for and the next rung can begin for the synchronous Successive Halving case. """ - assert budget in self.budgets - _max_configs = self.n_configs[list(self.budgets).index(budget)] - assert self._sh_bracket[budget] < _max_configs - self._sh_bracket[budget] += 1 + assert fidelity in self.fidelities + _max_configs = self.n_configs[list(self.fidelities).index(fidelity)] + assert self._sh_bracket[fidelity] < _max_configs + self._sh_bracket[fidelity] += 1 def _is_rung_waiting(self, rung): """ Returns True if at least one job is still pending/running and waits for results """ - job_count = self._sh_bracket[self.budgets[rung]] + self.sh_bracket[self.budgets[rung]] + job_count = self._sh_bracket[self.fidelities[rung]] + self.sh_bracket[self.fidelities[rung]] if job_count < self.n_configs[rung]: return True return False @@ -98,7 +98,7 @@ def _is_rung_waiting(self, rung): def _is_rung_pending(self, rung): """ Returns True if at least one job pending to be allocatted in the rung """ - if self.sh_bracket[self.budgets[rung]] > 0: + if self.sh_bracket[self.fidelities[rung]] > 0: return True return False @@ -116,33 +116,33 @@ def is_bracket_done(self): return ~self.is_pending() and ~self.is_waiting() def is_pending(self): - """ Returns True if any of the rungs/budgets have still a configuration to submit + """ Returns True if any of the rungs/fidelities have still a configuration to submit """ - return np.any([self._is_rung_pending(i) > 0 for i, _ in enumerate(self.budgets)]) + return np.any([self._is_rung_pending(i) > 0 for i, _ in enumerate(self.fidelities)]) def is_waiting(self): - """ Returns True if any of the rungs/budgets have a configuration pending/running + """ Returns True if any of the rungs/fidelities have a configuration pending/running """ - return np.any([self._is_rung_waiting(i) > 0 for i, _ in enumerate(self.budgets)]) + return np.any([self._is_rung_waiting(i) > 0 for i, _ in enumerate(self.fidelities)]) def __repr__(self): - cell_width = 9 + cell_width = 10 cell = "{{:^{}}}".format(cell_width) - budget_cell = "{{:^{}.2f}}".format(cell_width) + fidelity_cell = "{{:^{}.2f}}".format(cell_width) header = "|{}|{}|{}|{}|".format( - cell.format("budget"), + cell.format("fidelity"), cell.format("pending"), cell.format("waiting"), cell.format("done") ) _hline = "-" * len(header) table = [header, _hline] - for i, budget in enumerate(self.budgets): - pending = self.sh_bracket[budget] - done = self._sh_bracket[budget] + for i, fidelity in enumerate(self.fidelities): + pending = self.sh_bracket[fidelity] + done = self._sh_bracket[fidelity] waiting = np.abs(self.n_configs[i] - pending - done) entry = "|{}|{}|{}|{}|".format( - budget_cell.format(budget), + fidelity_cell.format(fidelity), cell.format(pending), cell.format(waiting), cell.format(done) diff --git a/src/dehb/utils/config_repository.py b/src/dehb/utils/config_repository.py new file mode 100644 index 0000000..126b28f --- /dev/null +++ b/src/dehb/utils/config_repository.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import numpy as np + + +@dataclass +class ConfigItem: + """Data class to store information regarding a specific configuration. + + The results for this configuration are stored in the `results` dict, using the fidelity it has + been evaluated on as keys. + """ + config_id: int + config: np.ndarray + results: dict[float, ResultItem] + +@dataclass +class ResultItem: + """Data class storing the result information of a specific configuration + fidelity.""" + score: float + cost: float + info: dict[Any, Any] + +class ConfigRepository: + """Bookkeeps all configurations used throughout the course of the optimization. + + Keeps track of the configurations and their results on the different fidelitites. + A new configuration is announced via `announce_config`. After evaluating the configuration + on the specified fidelity, use `tell_result` to log the achieved performance, cost etc. + + The configurations are stored in a list of `ConfigItem`. + """ + def __init__(self) -> None: + """Initializes the class by calling `self.reset`.""" + self.configs : list[ConfigItem] + self.reset() + + def reset(self) -> None: + """Resets the config repository, clearing all collected configurations and results.""" + self.configs = [] + + def announce_config(self, config: np.ndarray, fidelity=None) -> int: + """Announces a new configuration with the respective fidelity it should be evaluated on. + + The configuration is then added to the list of so far seen configurations and the ID of the + configuration is returned. + + Args: + config (np.ndarray): New configuration + fidelity (float, optional): Fidelity on which `config` is evaluated or None. + Defaults to None. + + Returns: + int: ID of configuration + """ + config_id = len(self.configs) + fidelity = float(fidelity or 0) + result_dict = { + fidelity: ResultItem(np.inf, -1, {}), + } + config_item = ConfigItem(config_id, config, result_dict) + self.configs.append(config_item) + return config_id + + def announce_population(self, population: np.ndarray, fidelity=None) -> np.ndarray: + """Announce population, retrieving ids for the population. + + Args: + population (np.ndarray): Population to announce + fidelity (float, optional): Fidelity on which pop is evaluated or None. + Defaults to None. + + Returns: + np.ndarray: population ids + """ + population_ids = [] + for indiv in population: + conf_id = self.announce_config(indiv, float(fidelity or 0)) + population_ids.append(conf_id) + return np.array(population_ids) + + def announce_fidelity(self, config_id: int, fidelity: float) -> bool: + """Announce the evaluation of a new fidelity for a given config. + + This function may only be used if the config already exists in the repository. + + Args: + config_id (int): ID of Configuration + fidelity (float): Fidelity on which the config will be evaluated + + Returns: + bool: Success/Failure of operation + """ + if config_id >= len(self.configs) or config_id < 0: + # TODO: Error message + return False + + config_item = self.configs[config_id] + result_item = { + fidelity: ResultItem(np.inf, -1, {}), + } + config_item.results[fidelity] = result_item + return True + + def tell_result(self, config_id: int, fidelity: float, score: float, cost: float, info: dict): + """Logs the achieved performance, cost etc. of a specific configuration-fidelity pair. + + Args: + config_id (int): ID of evaluated configuration + fidelity (float): Fidelity on which configuration has been evaluated. + score (float): Achieved score, given by objective function + cost (float): Cost, given by objective function + info (dict): Run info, given by objective function + """ + config_item = self.configs[config_id] + + # If configuration has been promoted, there is no fidelity information yet + if fidelity not in config_item.results: + config_item.results[fidelity] = ResultItem(score, cost, info) + else: + # ResultItem already given for specified fidelity --> update entries + config_item.results[fidelity].score = score + config_item.results[fidelity].cost = cost + config_item.results[fidelity].info = info \ No newline at end of file diff --git a/tests/test_config_repository.py b/tests/test_config_repository.py new file mode 100644 index 0000000..f63e870 --- /dev/null +++ b/tests/test_config_repository.py @@ -0,0 +1,34 @@ +import typing + +import numpy as np +from src.dehb.utils import ConfigRepository + + +class TestConfigAnnouncing(): + """Class that bundles all tests for announcing configurations to the repository.""" + def test_single_config(self): + """Tests announcing single config.""" + repo = ConfigRepository() + config = np.array([0.5]) + + config_id = repo.announce_config(config, 2) + + assert len(repo.configs) == 1 + assert config_id == 0 + assert repo.configs[config_id].config == config + + def test_population(self): + """Tests announcing a whole population.""" + repo = ConfigRepository() + pop = [] + for i in range(10): + config = np.array([i / 10]) + pop.append(config) + pop = np.array(pop) + + config_ids = repo.announce_population(pop) + + assert len(repo.configs) == 10 + + for conf_id in config_ids: + assert repo.configs[conf_id].config == pop[conf_id] \ No newline at end of file diff --git a/tests/test_de.py b/tests/test_de.py index f64457b..50099b2 100644 --- a/tests/test_de.py +++ b/tests/test_de.py @@ -15,7 +15,7 @@ def create_toy_DEBase(configspace: ConfigSpace.ConfigurationSpace): """ dim = len(configspace.get_hyperparameters()) return DEBase(f=lambda: 1, cs=configspace, dimensions=dim, pop_size=10, max_age=5, - mutation_factor=0.5, crossover_prob=0.5, strategy="rand1_bin", budget=1) + mutation_factor=0.5, crossover_prob=0.5, strategy="rand1_bin", fidelity=1) class TestConversion(): """Class that bundles all ConfigSpace/vector conversion tests. diff --git a/tests/test_dehb.py b/tests/test_dehb.py index 8a96e03..277e959 100644 --- a/tests/test_dehb.py +++ b/tests/test_dehb.py @@ -22,15 +22,15 @@ def create_toy_searchspace(): ConfigSpace.UniformFloatHyperparameter("x0", lower=3, upper=10, log=False)) return cs -def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_budget: float, - max_budget: float, eta: int, +def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_fidelity: float, + max_fidelity: float, eta: int, objective_function: typing.Callable): """Creates a DEHB instance. Args: configspace (ConfigurationSpace): Searchspace to use - min_budget (float): Minimum budget for DEHB - max_budget (float): Maximum budget for DEHB + min_fidelity (float): Minimum fidelity for DEHB + max_fidelity (float): Maximum fidelity for DEHB eta (int): Eta parameter of DEHB objective_function (Callable): Function to optimize @@ -39,16 +39,16 @@ def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_budget """ dim = len(configspace.get_hyperparameters()) return DEHB(f=objective_function, cs=configspace, dimensions=dim, - min_budget=min_budget, - max_budget=max_budget, eta=eta, n_workers=1) + min_fidelity=min_fidelity, + max_fidelity=max_fidelity, eta=eta, n_workers=1) -def objective_function(x: ConfigSpace.Configuration, budget: float, **kwargs): +def objective_function(x: ConfigSpace.Configuration, fidelity: float, **kwargs): """Toy objective function. Args: x (ConfigSpace.Configuration): Configuration to evaluate - budget (float): Budget to evaluate x on + fidelity (float): fidelity to evaluate x on Returns: dict: Result dictionary @@ -70,7 +70,7 @@ class TestBudgetExhaustion(): def test_runtime_exhaustion(self): """Test for runtime budget exhaustion.""" cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, objective_function=objective_function) dehb.start = time.time() - 10 @@ -80,7 +80,7 @@ def test_runtime_exhaustion(self): def test_fevals_exhaustion(self): """Test for function evaluations budget exhaustion.""" cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, objective_function=objective_function) dehb.traj.append("Just needed for the test") @@ -90,7 +90,7 @@ def test_fevals_exhaustion(self): def test_brackets_exhaustion(self): """Test for bracket budget exhaustion.""" cs = create_toy_searchspace() - dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, objective_function=objective_function) dehb.iteration_counter = 5 @@ -99,16 +99,52 @@ def test_brackets_exhaustion(self): class TestInitialization: """Class that bundles all tests regarding the initialization of DEHB.""" - def test_higher_min_budget(self): - """Test that verifies, that DEHB breaks if min_budget > max_budget.""" + def test_higher_min_fidelity(self): + """Test that verifies, that DEHB breaks if min_fidelity > max_fidelity.""" cs = create_toy_searchspace() with pytest.raises(AssertionError): - create_toy_optimizer(configspace=cs, min_budget=28, max_budget=27, eta=3, + create_toy_optimizer(configspace=cs, min_fidelity=28, max_fidelity=27, eta=3, objective_function=objective_function) - def test_equal_min_max_budget(self): - """Test that verifies, that DEHB breaks if min_budget == max_budget.""" + def test_equal_min_max_fidelity(self): + """Test that verifies, that DEHB breaks if min_fidelity == max_fidelity.""" cs = create_toy_searchspace() with pytest.raises(AssertionError): - create_toy_optimizer(configspace=cs, min_budget=27, max_budget=27, eta=3, + create_toy_optimizer(configspace=cs, min_fidelity=27, max_fidelity=27, eta=3, objective_function=objective_function) + +class TestConfigID: + """Class that bundles all tests regarding config ID functionality.""" + def test_initialization(self): + """Verifies, that the initial population is properly tracked by the config repository.""" + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + objective_function=objective_function) + # calculate how many configurations have been sampled for the initial populations + num_configs = 0 + for de_inst in dehb.de.values(): + num_configs += len(de_inst.population) + + # config repository should be exactly this long + assert len(dehb.config_repository.configs) == num_configs + + def test_single_bracket(self): + """Verifies, that the population is continously tracked over the run of a single bracket.""" + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + objective_function=objective_function) + # calculate how many configurations have been sampled for the initial populations + num_initial_configs = 0 + for de_inst in dehb.de.values(): + num_initial_configs += len(de_inst.population) + + # run for a single bracket + dehb.run(brackets=1, verbose=True) + + # for the first bracket, we only mutate on the lowest fidelity and then promote the best + # configs to the next fidelity. Please note, that this is only the case for the first + # DEHB bracket! + # Note: The final + 1 is due to the inner workings of DEHB. If the run budget is exhausted, + # we keep evolving new configurations without evaluating them, since we are only waiting to + # to fetch all results started ahead of the budget exhaustion. + assert len(dehb.config_repository.configs) == num_initial_configs + 9 + 1 \ No newline at end of file diff --git a/utils/README.md b/utils/README.md index 65f4058..5297e17 100644 --- a/utils/README.md +++ b/utils/README.md @@ -36,8 +36,8 @@ For example, running a DEHB optimization by specifiying `scheduler_file` makes t connect to the Dask cluster runnning. ```bash python examples/03_pytorch_mnist_hpo.py \ - --min_budget 1 \ - --max_budget 9 \ + --min_fidelity 1 \ + --max_fidelity 9 \ --runtime 200 \ --seed 123 \ --scheduler_file scheduler/scheduler_gpu.json \ From 7f9d18eb78b53613d3fc9bf6f0a76e8dcfd3b1bd Mon Sep 17 00:00:00 2001 From: Bronzila Date: Sat, 17 Feb 2024 15:45:49 +0100 Subject: [PATCH 7/9] Revert "Revert "Merge pull request #69 from automl/feat/ask_tell"" This reverts commit 652fe0baf25aa00b9b238bf0e8bbc54cbba8515b. --- src/dehb/optimizers/de.py | 13 ++- src/dehb/optimizers/dehb.py | 175 +++++++++++++++++++--------- src/dehb/utils/bracket_manager.py | 13 +++ src/dehb/utils/config_repository.py | 38 ++++-- tests/test_config_repository.py | 70 ++++++++++- tests/test_dehb.py | 94 ++++++++++++++- 6 files changed, 330 insertions(+), 73 deletions(-) diff --git a/src/dehb/optimizers/de.py b/src/dehb/optimizers/de.py index d1c40a2..be0e660 100644 --- a/src/dehb/optimizers/de.py +++ b/src/dehb/optimizers/de.py @@ -163,7 +163,7 @@ def vector_to_configspace(self, vector: np.ndarray) -> ConfigSpace.Configuration ''' # creates a ConfigSpace object dict with all hyperparameters present, the inactive too new_config = ConfigSpace.util.impute_inactive_values( - self.cs.sample_configuration() + self.cs.get_default_configuration() ).get_dictionary() # iterates over all hyperparameters and normalizes each based on its type for i, hyper in enumerate(self.cs.get_hyperparameters()): @@ -304,12 +304,17 @@ def f_objective(self, x, fidelity=None, **kwargs): raise NotImplementedError("An objective function needs to be passed.") if self.encoding: x = self.map_to_original(x) + + # Only convert config if configspace is used + configuration has not been converted yet if self.configspace: - # converts [0, 1] vector to a ConfigSpace object - config = self.vector_to_configspace(x) + if not isinstance(x, ConfigSpace.Configuration): + # converts [0, 1] vector to a ConfigSpace object + config = self.vector_to_configspace(x) + else: + config = x else: - # can insert custom scaling/transform function here config = x.copy() + if fidelity is not None: # to be used when called by multi-fidelity based optimizers res = self.f(config, fidelity=fidelity, **kwargs) else: diff --git a/src/dehb/optimizers/dehb.py b/src/dehb/optimizers/dehb.py index 612b496..6ac3f04 100644 --- a/src/dehb/optimizers/dehb.py +++ b/src/dehb/optimizers/dehb.py @@ -245,14 +245,18 @@ def _f_objective(self, job_info): res = self.de[fidelity].f_objective(config, fidelity, **kwargs) info = res["info"] if "info" in res else {} run_info = { - "fitness": res["fitness"], - "cost": res["cost"], - "config": config, - "config_id": config_id, - "fidelity": fidelity, - "parent_id": parent_id, - "bracket_id": bracket_id, - "info": info, + "job_info": { + "config": config, + "config_id": config_id, + "fidelity": fidelity, + "parent_id": parent_id, + "bracket_id": bracket_id, + }, + "result": { + "fitness": res["fitness"], + "cost": res["cost"], + "info": info, + }, } if "gpu_devices" in job_info: @@ -542,7 +546,10 @@ def _acquire_config(self, bracket, fidelity): return config, config_id, parent_id def _get_next_job(self): - """ Loads a configuration and fidelity to be evaluated next by a free worker + """Loads a configuration and fidelity to be evaluated next. + + Returns: + dict: Dicitonary containing all necessary information of the next job. """ bracket = None if len(self.active_brackets) == 0 or \ @@ -564,16 +571,50 @@ def _get_next_job(self): # fidelity that the SH bracket allots fidelity = bracket.get_next_job_fidelity() config, config_id, parent_id = self._acquire_config(bracket, fidelity) + + # transform config to proper representation + if self.configspace: + # converts [0, 1] vector to a ConfigSpace object + config = self.de[fidelity].vector_to_configspace(config) + # notifies the Bracket Manager that a single config is to run for the fidelity chosen job_info = { "config": config, "config_id": config_id, "fidelity": fidelity, "parent_id": parent_id, - "bracket_id": bracket.bracket_id + "bracket_id": bracket.bracket_id, } + + # pass information of job submission to Bracket Manager + for bracket in self.active_brackets: + if bracket.bracket_id == job_info['bracket_id']: + # registering is IMPORTANT for Bracket Manager to perform SH + bracket.register_job(job_info['fidelity']) + break return job_info + def ask(self, n_configs: int=1): + """Get the next configuration to run from the optimizer. + + The retrieved configuration can then be evaluated by the user. + After evaluation use `tell` to report the results back to the optimizer. + For more information, please refer to the description of `tell`. + + Args: + n_configs (int, optional): Number of configs to ask for. Defaults to 1. + + Returns: + dict or list of dict: Job info(s) of next configuration to evaluate. + """ + if n_configs == 1: + return self._get_next_job() + + jobs = [] + for _ in range(n_configs): + jobs.append(self._get_next_job()) + return jobs + def _get_gpu_id_with_low_load(self): candidates = [] for k, v in self.gpu_usage.items(): @@ -594,7 +635,7 @@ def submit_job(self, job_info, **kwargs): """ Asks a free worker to run the objective function on config and fidelity """ job_info["kwargs"] = self.shared_data if self.shared_data is not None else kwargs - # submit to to Dask client + # submit to Dask client if self.n_workers > 1 or isinstance(self.client, Client): if self.single_node_with_gpus: # managing GPU allocation for the job to be submitted @@ -606,13 +647,6 @@ def submit_job(self, job_info, **kwargs): # skipping scheduling to Dask worker to avoid added overheads in the synchronous case self.futures.append(self._f_objective(job_info)) - # pass information of job submission to Bracket Manager - for bracket in self.active_brackets: - if bracket.bracket_id == job_info['bracket_id']: - # registering is IMPORTANT for Bracket Manager to perform SH - bracket.register_job(job_info['fidelity']) - break - def _fetch_results_from_workers(self): """ Iterate over futures and collect results from finished workers """ @@ -636,40 +670,20 @@ def _fetch_results_from_workers(self): else: # Dask not invoked in the synchronous case run_info = future - # update bracket information - fitness, cost = run_info["fitness"], run_info["cost"] - info = run_info["info"] if "info" in run_info else dict() - fidelity, parent_id = run_info["fidelity"], run_info["parent_id"] - config, config_id = run_info["config"], run_info["config_id"] - bracket_id = run_info["bracket_id"] - for bracket in self.active_brackets: - if bracket.bracket_id == bracket_id: - # bracket job complete - bracket.complete_job(fidelity) # IMPORTANT to perform synchronous SH - - self.config_repository.tell_result(config_id, fidelity, fitness, cost, info) - - # carry out DE selection - if fitness <= self.de[fidelity].fitness[parent_id]: - self.de[fidelity].population[parent_id] = config - self.de[fidelity].population_ids[parent_id] = config_id - self.de[fidelity].fitness[parent_id] = fitness - # updating incumbents - if self.de[fidelity].fitness[parent_id] < self.inc_score: - self._update_incumbents( - config=self.de[fidelity].population[parent_id], - score=self.de[fidelity].fitness[parent_id], - info=info - ) - # book-keeping - self._update_trackers( - traj=self.inc_score, runtime=cost, history=( - config.tolist(), float(fitness), float(cost), float(fidelity), info - ) - ) + # tell result + self.tell(run_info["job_info"], run_info["result"]) # remove processed future self.futures = np.delete(self.futures, [i for i, _ in done_list]).tolist() + def _adjust_budgets(self, fevals=None, brackets=None): + # only update budgets if it is not the first run + if fevals is not None and len(self.traj) > 0: + fevals = len(self.traj) + fevals + elif brackets is not None and self.iteration_counter > -1: + brackets = self.iteration_counter + brackets + + return fevals, brackets + def _is_run_budget_exhausted(self, fevals=None, brackets=None, total_cost=None): """ Checks if the DEHB run should be terminated or continued """ @@ -745,6 +759,54 @@ def _verbosity_runtime(self, fevals, brackets, total_cost): "{}/{} {}".format(remaining[0], remaining[1], remaining[2]) ) + def tell(self, job_info: dict, result: dict): + """Feed a result back to the optimizer. + + In order to correctly interpret the results, the `job_info` dict, retrieved by `ask`, + has to be given. Moreover, the `result` dict has to contain the keys `fitness` and `cost`. + It is also possible to add the field `info` to the `result` in order to store additional, + user-specific information. + + Args: + job_info (dict): Job info returned by ask(). + result (dict): Result dictionary with mandatory keys `fitness` and `cost`. + """ + # update bracket information + fitness, cost = result["fitness"], result["cost"] + info = result["info"] if "info" in result else dict() + fidelity, parent_id = job_info["fidelity"], job_info["parent_id"] + config, config_id = job_info["config"], job_info["config_id"] + bracket_id = job_info["bracket_id"] + for bracket in self.active_brackets: + if bracket.bracket_id == bracket_id: + # bracket job complete + bracket.complete_job(fidelity) # IMPORTANT to perform synchronous SH + + self.config_repository.tell_result(config_id, fidelity, fitness, cost, info) + + # get hypercube representation from config repo + if self.configspace: + config = self.config_repository.get(config_id) + + # carry out DE selection + if fitness <= self.de[fidelity].fitness[parent_id]: + self.de[fidelity].population[parent_id] = config + self.de[fidelity].population_ids[parent_id] = config_id + self.de[fidelity].fitness[parent_id] = fitness + # updating incumbents + if self.de[fidelity].fitness[parent_id] < self.inc_score: + self._update_incumbents( + config=self.de[fidelity].population[parent_id], + score=self.de[fidelity].fitness[parent_id], + info=info + ) + # book-keeping + self._update_trackers( + traj=self.inc_score, runtime=cost, history=( + config.tolist(), float(fitness), float(cost), float(fidelity), info + ) + ) + @logger.catch def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus=False, verbose=False, debug=False, save_intermediate=True, save_history=True, name=None, **kwargs): @@ -761,6 +823,12 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus 2) Number of Successive Halving brackets run under Hyperband (brackets) 3) Total computational cost (in seconds) aggregated by all function evaluations (total_cost) """ + # check if run has already been called before + if self.start is not None: + logger.warning("DEHB has already been run. Calling 'run' twice could lead to unintended" + + " behavior. Please restart DEHB with an increased compute budget" + + " instead of calling 'run' twice.") + # checks if a Dask client exists if len(kwargs) > 0 and self.n_workers > 1 and isinstance(self.client, Client): # broadcasts all additional data passed as **kwargs to all client workers @@ -774,7 +842,8 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus if self.single_node_with_gpus: self.distribute_gpus() - self.start = time.time() + self.start = self.start = time.time() + fevals, brackets = self._adjust_budgets(fevals, brackets) if verbose: print("\nLogging at {} for optimization starting at {}\n".format( os.path.join(os.getcwd(), self.log_filename), @@ -786,11 +855,11 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus if self._is_run_budget_exhausted(fevals, brackets, total_cost): break if self.is_worker_available(): - job_info = self._get_next_job() + job_info = self.ask() if brackets is not None and job_info["bracket_id"] >= brackets: # ignore submission and only collect results # when brackets are chosen as run budget, an extra bracket is created - # since iteration_counter is incremented in _get_next_job() and then checked + # since iteration_counter is incremented in ask() and then checked # in _is_run_budget_exhausted(), therefore, need to skip suggestions # coming from the extra allocated bracket # _is_run_budget_exhausted() will not return True until all the lower brackets @@ -850,4 +919,6 @@ def run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus self.logger.info("{}".format(self.inc_config)) self._save_incumbent(name) self._save_history(name) + # reset waiting jobs of active bracket to allow for continuation + self.active_brackets[0].reset_waiting_jobs() return np.array(self.traj), np.array(self.runtime), np.array(self.history, dtype=object) diff --git a/src/dehb/utils/bracket_manager.py b/src/dehb/utils/bracket_manager.py index 2642223..598b92a 100644 --- a/src/dehb/utils/bracket_manager.py +++ b/src/dehb/utils/bracket_manager.py @@ -125,6 +125,19 @@ def is_waiting(self): """ return np.any([self._is_rung_waiting(i) > 0 for i, _ in enumerate(self.fidelities)]) + def reset_waiting_jobs(self): + """Resets all waiting jobs and updates the current_rung pointer accordingly.""" + for i, fidelity in enumerate(self.fidelities): + pending = self.sh_bracket[fidelity] + done = self._sh_bracket[fidelity] + waiting = np.abs(self.n_configs[i] - pending - done) + + # update current_rung pointer to the lowest rung with waiting jobs + if waiting > 0 and self.current_rung > i: + self.current_rung = i + # reset waiting jobs + self.sh_bracket[fidelity] += waiting + def __repr__(self): cell_width = 10 cell = "{{:^{}}}".format(cell_width) diff --git a/src/dehb/utils/config_repository.py b/src/dehb/utils/config_repository.py index 126b28f..2d2bab1 100644 --- a/src/dehb/utils/config_repository.py +++ b/src/dehb/utils/config_repository.py @@ -82,28 +82,26 @@ def announce_population(self, population: np.ndarray, fidelity=None) -> np.ndarr population_ids.append(conf_id) return np.array(population_ids) - def announce_fidelity(self, config_id: int, fidelity: float) -> bool: + def announce_fidelity(self, config_id: int, fidelity: float): """Announce the evaluation of a new fidelity for a given config. This function may only be used if the config already exists in the repository. + Note: This function is currently unused, but might be used later in order to + allow for continuation. Args: config_id (int): ID of Configuration fidelity (float): Fidelity on which the config will be evaluated - - Returns: - bool: Success/Failure of operation """ - if config_id >= len(self.configs) or config_id < 0: - # TODO: Error message - return False + try: + config_item = self.configs[config_id] + except IndexError as e: + raise IndexError("Config with the given ID can not be found.") from e - config_item = self.configs[config_id] result_item = { fidelity: ResultItem(np.inf, -1, {}), } config_item.results[fidelity] = result_item - return True def tell_result(self, config_id: int, fidelity: float, score: float, cost: float, info: dict): """Logs the achieved performance, cost etc. of a specific configuration-fidelity pair. @@ -115,7 +113,10 @@ def tell_result(self, config_id: int, fidelity: float, score: float, cost: float cost (float): Cost, given by objective function info (dict): Run info, given by objective function """ - config_item = self.configs[config_id] + try: + config_item = self.configs[config_id] + except IndexError as e: + raise IndexError("Config with the given ID can not be found.") from e # If configuration has been promoted, there is no fidelity information yet if fidelity not in config_item.results: @@ -124,4 +125,19 @@ def tell_result(self, config_id: int, fidelity: float, score: float, cost: float # ResultItem already given for specified fidelity --> update entries config_item.results[fidelity].score = score config_item.results[fidelity].cost = cost - config_item.results[fidelity].info = info \ No newline at end of file + config_item.results[fidelity].info = info + + def get(self, config_id: int) -> np.ndarray: + """Get the configuration with the given ID. + + Args: + config_id (int): ID of config + + Returns: + np.ndarray: Config in hypercube representation + """ + try: + config_item = self.configs[config_id] + except IndexError as e: + raise IndexError("Config with the given ID can not be found.") from e + return config_item.config \ No newline at end of file diff --git a/tests/test_config_repository.py b/tests/test_config_repository.py index f63e870..76cc1ae 100644 --- a/tests/test_config_repository.py +++ b/tests/test_config_repository.py @@ -1,21 +1,37 @@ import typing import numpy as np +import pytest from src.dehb.utils import ConfigRepository class TestConfigAnnouncing(): """Class that bundles all tests for announcing configurations to the repository.""" - def test_single_config(self): - """Tests announcing single config.""" + def test_single_config_fidelity(self): + """Tests announcing single config with a specified fidelity.""" repo = ConfigRepository() config = np.array([0.5]) - config_id = repo.announce_config(config, 2) + config_id = repo.announce_config(config, 2.) assert len(repo.configs) == 1 assert config_id == 0 assert repo.configs[config_id].config == config + # result entry properly given + assert repo.configs[config_id].results[2.] is not None + + def test_single_config_no_fidelity(self): + """Tests announcing single config with a specified fidelity.""" + repo = ConfigRepository() + config = np.array([0.5]) + + config_id = repo.announce_config(config) + + assert len(repo.configs) == 1 + assert config_id == 0 + assert repo.configs[config_id].config == config + # result entry properly given + assert repo.configs[config_id].results[0.] is not None def test_population(self): """Tests announcing a whole population.""" @@ -31,4 +47,50 @@ def test_population(self): assert len(repo.configs) == 10 for conf_id in config_ids: - assert repo.configs[conf_id].config == pop[conf_id] \ No newline at end of file + assert repo.configs[conf_id].config == pop[conf_id] + +class TestGetConfig(): + """Class that bundles all tests regarding retrieving of configs via config ID.""" + def test_get_successful(self): + """Test that get retrieves the right configuration.""" + repo = ConfigRepository() + config = np.array([0.5]) + + config_id = repo.announce_config(config) + + retrieved_config = repo.get(config_id) + + assert config == retrieved_config + + def test_get_failure(self): + """Test to verify that get returns the right error if config ID is unkown.""" + repo = ConfigRepository() + config = np.array([0.5]) + + config_id = repo.announce_config(config) + + with pytest.raises(IndexError): + repo.get(config_id + 1) + +class TestTellResult(): + """This class bundles all tests regarding the `tell_result` method.""" + def test_tell_result_successful(self): + repo = ConfigRepository() + config = np.array([0.5]) + + fidelity = 2.0 + config_id = repo.announce_config(config, fidelity) + score = 1 + cost = 2 + info = { + "test": 123, + } + repo.tell_result(config_id, fidelity, score, cost, info) + + config_item = repo.configs[config_id] + results = config_item.results + + assert len(results) == 1 + assert results[fidelity].score == score + assert results[fidelity].cost == cost + assert results[fidelity].info == info \ No newline at end of file diff --git a/tests/test_dehb.py b/tests/test_dehb.py index 277e959..d0a8e5c 100644 --- a/tests/test_dehb.py +++ b/tests/test_dehb.py @@ -37,7 +37,7 @@ def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_fideli Returns: _type_: _description_ """ - dim = len(configspace.get_hyperparameters()) + dim = len(configspace.get_hyperparameters()) if configspace else 1 return DEHB(f=objective_function, cs=configspace, dimensions=dim, min_fidelity=min_fidelity, max_fidelity=max_fidelity, eta=eta, n_workers=1) @@ -147,4 +147,94 @@ def test_single_bracket(self): # Note: The final + 1 is due to the inner workings of DEHB. If the run budget is exhausted, # we keep evolving new configurations without evaluating them, since we are only waiting to # to fetch all results started ahead of the budget exhaustion. - assert len(dehb.config_repository.configs) == num_initial_configs + 9 + 1 \ No newline at end of file + assert len(dehb.config_repository.configs) == num_initial_configs + 9 + 1 + +class TestAskTell: + """Class that bundles all tests regarding the ask and tell functionality of DEHB.""" + def test_all_fields_available(self): + """Verifies, that all fields needed are present in job info returned by ask.""" + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + objective_function=objective_function) + conf = dehb.ask() + assert "config" in conf + assert "bracket_id" in conf + assert "config_id" in conf + assert "fidelity" in conf + + def test_format_configspace(self): + """Verifies, that the returned config by ask() is of type Configuration + if a configspace is passed. + """ + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + objective_function=objective_function) + job_info = dehb.ask() + assert isinstance(job_info["config"], ConfigSpace.Configuration) + + def test_format_no_configspace(self): + """Verifies, that the returned config by ask() is of type Configuration + if a configspace is passed. + """ + dehb = create_toy_optimizer(configspace=None, min_fidelity=3, max_fidelity=27, eta=3, + objective_function=objective_function) + job_info = dehb.ask() + assert isinstance(job_info["config"], np.ndarray) + + def test_ask_multiple(self): + """Verifies, that ask can return multiple configs.""" + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + objective_function=objective_function) + job_infos = dehb.ask(2) + + assert len(job_infos) == 2 + assert job_infos[0]["config"] != job_infos[1]["config"] + + def test_ask_twice_different(self): + """Verifies, that ask can return multiple configs.""" + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + objective_function=objective_function) + job_info_a = dehb.ask() + job_info_b = dehb.ask() + assert job_info_a != job_info_b + + def test_tell_successful(self): + """Verifies, that tell successfully saves results.""" + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + objective_function=objective_function) + job_info = dehb.ask() + id = job_info["config_id"] + fid = job_info["fidelity"] + conf = job_info["config"] + + # before telling, entry should be empty + saved_score = dehb.config_repository.configs[id].results[fid].score + assert saved_score == np.inf + + result = objective_function(conf, fid) + dehb.tell(job_info, result) + + # after telling, score should be saved + saved_score = dehb.config_repository.configs[id].results[fid].score + assert saved_score == result["fitness"] + + def test_tell_error(self): + """Verifies, that tell throws an error if config ID is non-existent.""" + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_fidelity=3, max_fidelity=27, eta=3, + objective_function=objective_function) + # get config + job_info = dehb.ask() + # adjust config id to non existing id + job_info["config_id"] = 1337 + # create random result item + result = { + "fitness": 42, + "cost": 123 + } + # telling with wrong config_id should throw an error + with pytest.raises(IndexError): + dehb.tell(job_info, result) \ No newline at end of file From d1583b4e9ca3ddf0c393a7243b3d1e707fc02ac5 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Sat, 17 Feb 2024 19:26:38 +0100 Subject: [PATCH 8/9] Add optional dependencies for examples doc generation --- .github/workflows/docs.yml | 2 +- pyproject.toml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7ca2001..1b95904 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,7 +32,7 @@ jobs: with: file: 'pyproject.toml' field: 'project.version' - - run: pip install ".[dev]" + - run: pip install ".[dev, examples]" - name: Configure Git user run: | git config --local user.email "github-actions[bot]@users.noreply.github.com" diff --git a/pyproject.toml b/pyproject.toml index 32b1771..240a9af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,9 @@ dev = [ "black", "pre-commit", ] +examples = [ + "sklearn" +] [tool.pytest.ini_options] testpaths = ["tests"] # path to the test directory From 28e48c343ed0aa35027550b5bdf202c361165679 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Sat, 17 Feb 2024 19:34:56 +0100 Subject: [PATCH 9/9] Adjust sklearn dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 240a9af..953e3cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ dev = [ "pre-commit", ] examples = [ - "sklearn" + "scikit-learn" ] [tool.pytest.ini_options]