From 38f4d32c0d13c8d6fd2db553431bba39900404fc Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Sat, 17 Feb 2024 18:40:20 +0000
Subject: [PATCH] Deployed 28e48c3 to 0.1.0 with MkDocs 1.5.3 and mike 2.0.0

---
 0.1.0/examples/00_interfacing_DEHB/index.html |  30 +-
 .../index.html                                | 264 ++++++++----------
 .../index.html                                | 198 ++++---------
 .../index.html                                | 114 ++------
 .../getting_started/single_worker/index.html  |  12 +-
 0.1.0/search/search_index.json                |   2 +-
 0.1.0/sitemap.xml.gz                          | Bin 127 -> 127 bytes
 7 files changed, 212 insertions(+), 408 deletions(-)

diff --git a/0.1.0/examples/00_interfacing_DEHB/index.html b/0.1.0/examples/00_interfacing_DEHB/index.html
index 38c7e49..896b8f5 100644
--- a/0.1.0/examples/00_interfacing_DEHB/index.html
+++ b/0.1.0/examples/00_interfacing_DEHB/index.html
@@ -1549,7 +1549,7 @@ <h3 id="defining-a-sample-search-space">Defining a sample search space<a class="
 <div class="jp-OutputPrompt jp-OutputArea-prompt">Out[5]:</div>
 <div class="jp-RenderedText jp-OutputArea-output jp-OutputArea-executeResult" data-mime-type="text/plain" tabindex="0">
 <pre>Configuration(values={
-  'x0': 8.627121338040169,
+  'x0': 7.785477780551136,
 })</pre>
 </div>
 </div>
@@ -1771,7 +1771,7 @@ <h4 id="1-running-dehb-for-a-certain-number-of-successive-halving-brackets">1) R
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
 <div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
 <pre>Configuration(values={
-  'x0': 6.34288307846435,
+  'x0': 6.460117941570684,
 })
 </pre>
 </div>
@@ -1831,9 +1831,9 @@ <h4 id="2-running-dehb-for-total-number-of-function-evaluations">2) Running DEHB
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
 <div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
-<pre><span class="ansi-green-fg">2024-02-17 14:47:50.136</span> | <span class="ansi-bold">INFO    </span> | <span class="ansi-cyan-fg">dehb.optimizers.dehb</span>:<span class="ansi-cyan-fg">reset</span>:<span class="ansi-cyan-fg">121</span> - <span class="ansi-bold">
+<pre><span class="ansi-green-fg">2024-02-17 18:38:08.099</span> | <span class="ansi-bold">INFO    </span> | <span class="ansi-cyan-fg">dehb.optimizers.dehb</span>:<span class="ansi-cyan-fg">reset</span>:<span class="ansi-cyan-fg">121</span> - <span class="ansi-bold">
 
-RESET at 02/17/24 14:47:50 UTC
+RESET at 02/17/24 18:38:08 UTC
 
 </span>
 </pre>
@@ -1843,8 +1843,8 @@ <h4 id="2-running-dehb-for-total-number-of-function-evaluations">2) Running DEHB
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
 <div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
 <pre>(Configuration(values={
-  'x0': 9.263648958886906,
-}), 0.018104529362575228)
+  'x0': 7.74688038121089,
+}), 0.1103633789699231)
 </pre>
 </div>
 </div>
@@ -1903,9 +1903,9 @@ <h4 id="3-running-dehb-for-total-amount-of-wallclock-time">3) Running DEHB for t
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
 <div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
-<pre><span class="ansi-green-fg">2024-02-17 14:47:53.035</span> | <span class="ansi-bold">INFO    </span> | <span class="ansi-cyan-fg">dehb.optimizers.dehb</span>:<span class="ansi-cyan-fg">reset</span>:<span class="ansi-cyan-fg">121</span> - <span class="ansi-bold">
+<pre><span class="ansi-green-fg">2024-02-17 18:38:10.370</span> | <span class="ansi-bold">INFO    </span> | <span class="ansi-cyan-fg">dehb.optimizers.dehb</span>:<span class="ansi-cyan-fg">reset</span>:<span class="ansi-cyan-fg">121</span> - <span class="ansi-bold">
 
-RESET at 02/17/24 14:47:53 UTC
+RESET at 02/17/24 18:38:10 UTC
 
 </span>
 </pre>
@@ -1915,8 +1915,8 @@ <h4 id="3-running-dehb-for-total-amount-of-wallclock-time">3) Running DEHB for t
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
 <div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
 <pre>(Configuration(values={
-  'x0': 5.566303183270801,
-}), 0.02030815133180952)
+  'x0': 4.290333531372179,
+}), 0.004440353907169903)
 </pre>
 </div>
 </div>
@@ -1995,8 +1995,8 @@ <h4 id="3-running-dehb-for-total-amount-of-wallclock-time">3) Running DEHB for t
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
 <div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
 <pre>(Configuration(values={
-  'x0': 4.447543364972189,
-}), 0.018084296582581594)
+  'x0': 3.9501812728673467,
+}), 0.0031614215460380457)
 </pre>
 </div>
 </div>
@@ -2037,7 +2037,7 @@ <h4 id="3-running-dehb-for-total-amount-of-wallclock-time">3) Running DEHB for t
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
 <div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
 <pre>Configuration(values={
-  'x0': 4.447543364972189,
+  'x0': 3.9501812728673467,
 })
 </pre>
 </div>
@@ -2080,9 +2080,9 @@ <h4 id="3-running-dehb-for-total-amount-of-wallclock-time">3) Running DEHB for t
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
 <div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
-<pre>0.018084296582581594 0.018084296582581594
+<pre>0.0031614215460380457 0.0031614215460380457
 Configuration(values={
-  'x0': 4.447543364972189,
+  'x0': 3.9501812728673467,
 })
 </pre>
 </div>
diff --git a/0.1.0/examples/01.1_Optimizing_RandomForest_using_DEHB/index.html b/0.1.0/examples/01.1_Optimizing_RandomForest_using_DEHB/index.html
index 26e7fd3..9e36612 100644
--- a/0.1.0/examples/01.1_Optimizing_RandomForest_using_DEHB/index.html
+++ b/0.1.0/examples/01.1_Optimizing_RandomForest_using_DEHB/index.html
@@ -1643,8 +1643,8 @@ <h4 id="1-preparing-dataset-and-splits">1 ) Preparing dataset and splits<a class
 </div>
 </div>
 </div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -1675,28 +1675,9 @@ <h4 id="1-preparing-dataset-and-splits">1 ) Preparing dataset and splits<a class
 </div>
 </div>
 </div>
-<div class="jp-Cell-outputWrapper">
-<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
-</div>
-<div class="jp-OutputArea jp-Cell-outputArea">
-<div class="jp-OutputArea-child">
-<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">ModuleNotFoundError</span>                       Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[6], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">datasets</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> load_iris, load_digits, load_wine
-<span class="ansi-green-intense-fg ansi-bold">      4</span> classification <span style="color: rgb(98,98,98)">=</span> {<span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">iris</span><span style="color: rgb(175,0,0)">"</span>: load_iris, <span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">digits</span><span style="color: rgb(175,0,0)">"</span>: load_digits, <span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">wine</span><span style="color: rgb(175,0,0)">"</span>: load_wine}
-
-<span class="ansi-red-fg">ModuleNotFoundError</span>: No module named 'sklearn'</pre>
-</div>
-</div>
-</div>
-</div>
 </div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -1779,26 +1760,6 @@ <h4 id="1-preparing-dataset-and-splits">1 ) Preparing dataset and splits<a class
 </div>
 </div>
 </div>
-<div class="jp-Cell-outputWrapper">
-<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
-</div>
-<div class="jp-OutputArea jp-Cell-outputArea">
-<div class="jp-OutputArea-child">
-<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">ModuleNotFoundError</span>                       Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[7], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">model_selection</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> train_test_split
-<span class="ansi-green-intense-fg ansi-bold">      4</span> <span class="ansi-bold" style="color: rgb(0,135,0)">def</span> <span style="color: rgb(0,0,255)">prepare_dataset</span>(model_type<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">classification</span><span style="color: rgb(175,0,0)">"</span>, dataset<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">None</span>):
-<span class="ansi-green-intense-fg ansi-bold">      6</span>     <span class="ansi-bold" style="color: rgb(0,135,0)">if</span> model_type <span style="color: rgb(98,98,98)">==</span> <span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">classification</span><span style="color: rgb(175,0,0)">"</span>:
-
-<span class="ansi-red-fg">ModuleNotFoundError</span>: No module named 'sklearn'</pre>
-</div>
-</div>
-</div>
-</div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
 <div class="jp-Cell jp-CodeCell jp-Notebook-cell">
@@ -1844,19 +1805,12 @@ <h4 id="1-preparing-dataset-and-splits">1 ) Preparing dataset and splits<a class
 <div class="jp-OutputArea jp-Cell-outputArea">
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[8], line 2</span>
-<span class="ansi-green-intense-fg ansi-bold">      1</span> train_X, train_y, valid_X, valid_y, test_X, test_y, dataset <span style="color: rgb(98,98,98)">=</span> \
-<span class="ansi-green-fg">----&gt; 2</span>     <span class="ansi-yellow-bg">prepare_dataset</span>(model_type<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">classification</span><span style="color: rgb(175,0,0)">"</span>)
-<span class="ansi-green-intense-fg ansi-bold">      4</span> <span style="color: rgb(0,135,0)">print</span>(dataset)
-<span class="ansi-green-intense-fg ansi-bold">      5</span> <span style="color: rgb(0,135,0)">print</span>(<span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">Train size: </span><span class="ansi-bold" style="color: rgb(175,95,135)">{}</span><span class="ansi-bold" style="color: rgb(175,95,0)">\n</span><span style="color: rgb(175,0,0)">Valid size: </span><span class="ansi-bold" style="color: rgb(175,95,135)">{}</span><span class="ansi-bold" style="color: rgb(175,95,0)">\n</span><span style="color: rgb(175,0,0)">Test size: </span><span class="ansi-bold" style="color: rgb(175,95,135)">{}</span><span style="color: rgb(175,0,0)">"</span><span style="color: rgb(98,98,98)">.</span>format(
-<span class="ansi-green-intense-fg ansi-bold">      6</span>     train_X<span style="color: rgb(98,98,98)">.</span>shape, valid_X<span style="color: rgb(98,98,98)">.</span>shape, test_X<span style="color: rgb(98,98,98)">.</span>shape
-<span class="ansi-green-intense-fg ansi-bold">      7</span> ))
-
-<span class="ansi-red-fg">NameError</span>: name 'prepare_dataset' is not defined</pre>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre>wine
+Train size: (124, 13)
+Valid size: (36, 13)
+Test size: (18, 13)
+</pre>
 </div>
 </div>
 </div>
@@ -1873,8 +1827,8 @@ <h4 id="2-function-interface-with-dehb">2 ) Function interface with DEHB<a class
 </div>
 </div>
 </div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -1907,26 +1861,6 @@ <h4 id="2-function-interface-with-dehb">2 ) Function interface with DEHB<a class
 </div>
 </div>
 </div>
-<div class="jp-Cell-outputWrapper">
-<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
-</div>
-<div class="jp-OutputArea jp-Cell-outputArea">
-<div class="jp-OutputArea-child">
-<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">ModuleNotFoundError</span>                       Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[9], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">ensemble</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> RandomForestClassifier
-<span class="ansi-green-intense-fg ansi-bold">      2</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">metrics</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> accuracy_score, make_scorer
-<span class="ansi-green-intense-fg ansi-bold">      5</span> accuracy_scorer <span style="color: rgb(98,98,98)">=</span> make_scorer(accuracy_score)
-
-<span class="ansi-red-fg">ModuleNotFoundError</span>: No module named 'sklearn'</pre>
-</div>
-</div>
-</div>
-</div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
@@ -2100,8 +2034,8 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 </div>
 </div>
 </div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -2148,35 +2082,6 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 </div>
 </div>
 </div>
-<div class="jp-Cell-outputWrapper">
-<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
-</div>
-<div class="jp-OutputArea jp-Cell-outputArea">
-<div class="jp-OutputArea-child">
-<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[12], line 7</span>
-<span class="ansi-green-intense-fg ansi-bold">      1</span> trajectory, runtime, history <span style="color: rgb(98,98,98)">=</span> dehb<span style="color: rgb(98,98,98)">.</span>run(
-<span class="ansi-green-intense-fg ansi-bold">      2</span>     total_cost<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(98,98,98)">10</span>,
-<span class="ansi-green-intense-fg ansi-bold">      3</span>     verbose<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">False</span>,
-<span class="ansi-green-intense-fg ansi-bold">      4</span>     save_intermediate<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">False</span>,
-<span class="ansi-green-intense-fg ansi-bold">      5</span>     <span style="color: rgb(95,135,135)"># parameters expected as **kwargs in target_function is passed here</span>
-<span class="ansi-green-intense-fg ansi-bold">      6</span>     seed<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(98,98,98)">123</span>,
-<span class="ansi-green-fg">----&gt; 7</span>     train_X<span style="color: rgb(98,98,98)">=</span><span class="ansi-yellow-bg">train_X</span>,
-<span class="ansi-green-intense-fg ansi-bold">      8</span>     train_y<span style="color: rgb(98,98,98)">=</span>train_y,
-<span class="ansi-green-intense-fg ansi-bold">      9</span>     valid_X<span style="color: rgb(98,98,98)">=</span>valid_X,
-<span class="ansi-green-intense-fg ansi-bold">     10</span>     valid_y<span style="color: rgb(98,98,98)">=</span>valid_y,
-<span class="ansi-green-intense-fg ansi-bold">     11</span>     max_fidelity<span style="color: rgb(98,98,98)">=</span>dehb<span style="color: rgb(98,98,98)">.</span>max_fidelity
-<span class="ansi-green-intense-fg ansi-bold">     12</span> )
-
-<span class="ansi-red-fg">NameError</span>: name 'train_X' is not defined</pre>
-</div>
-</div>
-</div>
-</div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
 <div class="jp-Cell jp-CodeCell jp-Notebook-cell">
@@ -2230,16 +2135,18 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 <div class="jp-OutputArea jp-Cell-outputArea">
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[13], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> <span style="color: rgb(0,135,0)">print</span>(<span style="color: rgb(0,135,0)">len</span>(<span class="ansi-yellow-bg">trajectory</span>), <span style="color: rgb(0,135,0)">len</span>(runtime), <span style="color: rgb(0,135,0)">len</span>(history), end<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(175,0,0)">"</span><span class="ansi-bold" style="color: rgb(175,95,0)">\n</span><span class="ansi-bold" style="color: rgb(175,95,0)">\n</span><span style="color: rgb(175,0,0)">"</span>)
-<span class="ansi-green-intense-fg ansi-bold">      3</span> <span style="color: rgb(95,135,135)"># Last recorded function evaluation</span>
-<span class="ansi-green-intense-fg ansi-bold">      4</span> last_eval <span style="color: rgb(98,98,98)">=</span> history[<span style="color: rgb(98,98,98)">-</span><span style="color: rgb(98,98,98)">1</span>]
-
-<span class="ansi-red-fg">NameError</span>: name 'trajectory' is not defined</pre>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre>416 416 416
+
+Last evaluated configuration, 
+Configuration(values={
+  'max_depth': 6,
+  'max_features': 0.5908227041967594,
+  'min_samples_leaf': 2,
+  'min_samples_split': 5,
+})got a score of -1.0, was evaluated at a fidelity of 16.67 and took 0.018 seconds to run.
+The additional info attached: {'test_score': 1.0, 'fidelity': 16.666666666666664}
+</pre>
 </div>
 </div>
 </div>
@@ -2359,9 +2266,9 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
 <div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
-<pre><span class="ansi-green-fg">2024-02-17 14:48:33.183</span> | <span class="ansi-bold">INFO    </span> | <span class="ansi-cyan-fg">dehb.optimizers.dehb</span>:<span class="ansi-cyan-fg">reset</span>:<span class="ansi-cyan-fg">121</span> - <span class="ansi-bold">
+<pre><span class="ansi-green-fg">2024-02-17 18:38:59.841</span> | <span class="ansi-bold">INFO    </span> | <span class="ansi-cyan-fg">dehb.optimizers.dehb</span>:<span class="ansi-cyan-fg">reset</span>:<span class="ansi-cyan-fg">121</span> - <span class="ansi-bold">
 
-RESET at 02/17/24 14:48:33 UTC
+RESET at 02/17/24 18:38:59 UTC
 
 </span>
 </pre>
@@ -2369,28 +2276,46 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 </div>
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[14], line 14</span>
-<span class="ansi-green-intense-fg ansi-bold">      7</span> dehb<span style="color: rgb(98,98,98)">.</span>reset()
-<span class="ansi-green-intense-fg ansi-bold">      8</span> <span style="color: rgb(95,135,135)"># Executing a run of DEHB optimization lasting for 10s</span>
-<span class="ansi-green-intense-fg ansi-bold">      9</span> trajectory, runtime, history <span style="color: rgb(98,98,98)">=</span> dehb<span style="color: rgb(98,98,98)">.</span>run(
-<span class="ansi-green-intense-fg ansi-bold">     10</span>     total_cost<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(98,98,98)">10</span>,
-<span class="ansi-green-intense-fg ansi-bold">     11</span>     verbose<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">False</span>,
-<span class="ansi-green-intense-fg ansi-bold">     12</span>     save_intermediate<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">False</span>,
-<span class="ansi-green-intense-fg ansi-bold">     13</span>     seed<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(98,98,98)">123</span>,
-<span class="ansi-green-fg">---&gt; 14</span>     train_X<span style="color: rgb(98,98,98)">=</span><span class="ansi-yellow-bg">train_X</span>,
-<span class="ansi-green-intense-fg ansi-bold">     15</span>     train_y<span style="color: rgb(98,98,98)">=</span>train_y,
-<span class="ansi-green-intense-fg ansi-bold">     16</span>     valid_X<span style="color: rgb(98,98,98)">=</span>valid_X,
-<span class="ansi-green-intense-fg ansi-bold">     17</span>     valid_y<span style="color: rgb(98,98,98)">=</span>valid_y,
-<span class="ansi-green-intense-fg ansi-bold">     18</span>     max_fidelity<span style="color: rgb(98,98,98)">=</span>dehb<span style="color: rgb(98,98,98)">.</span>max_fidelity
-<span class="ansi-green-intense-fg ansi-bold">     19</span> )
-<span class="ansi-green-intense-fg ansi-bold">     20</span> best_config <span style="color: rgb(98,98,98)">=</span> dehb<span style="color: rgb(98,98,98)">.</span>vector_to_configspace(dehb<span style="color: rgb(98,98,98)">.</span>inc_config)
-<span class="ansi-green-intense-fg ansi-bold">     22</span> <span style="color: rgb(95,135,135)"># Creating a model using the best configuration found</span>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre><span class="ansi-green-fg">2024-02-17 18:39:09.921</span> | <span class="ansi-bold">INFO    </span> | <span class="ansi-cyan-fg">dehb.optimizers.dehb</span>:<span class="ansi-cyan-fg">reset</span>:<span class="ansi-cyan-fg">121</span> - <span class="ansi-bold">
 
-<span class="ansi-red-fg">NameError</span>: name 'train_X' is not defined</pre>
+RESET at 02/17/24 18:39:09 UTC
+
+</span>
+</pre>
+</div>
+</div>
+<div class="jp-OutputArea-child">
+<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre><span class="ansi-green-fg">2024-02-17 18:39:19.994</span> | <span class="ansi-bold">INFO    </span> | <span class="ansi-cyan-fg">dehb.optimizers.dehb</span>:<span class="ansi-cyan-fg">reset</span>:<span class="ansi-cyan-fg">121</span> - <span class="ansi-bold">
+
+RESET at 02/17/24 18:39:19 UTC
+
+</span>
+</pre>
+</div>
+</div>
+<div class="jp-OutputArea-child">
+<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre><span class="ansi-green-fg">2024-02-17 18:39:30.078</span> | <span class="ansi-bold">INFO    </span> | <span class="ansi-cyan-fg">dehb.optimizers.dehb</span>:<span class="ansi-cyan-fg">reset</span>:<span class="ansi-cyan-fg">121</span> - <span class="ansi-bold">
+
+RESET at 02/17/24 18:39:30 UTC
+
+</span>
+</pre>
+</div>
+</div>
+<div class="jp-OutputArea-child">
+<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre><span class="ansi-green-fg">2024-02-17 18:39:40.139</span> | <span class="ansi-bold">INFO    </span> | <span class="ansi-cyan-fg">dehb.optimizers.dehb</span>:<span class="ansi-cyan-fg">reset</span>:<span class="ansi-cyan-fg">121</span> - <span class="ansi-bold">
+
+RESET at 02/17/24 18:39:40 UTC
+
+</span>
+</pre>
 </div>
 </div>
 </div>
@@ -2431,16 +2356,16 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
 <div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
-<pre>Mean score across trials:  nan
-Std. dev. of score across trials:  nan
+<pre>Mean score across trials:  1.0
+Std. dev. of score across trials:  0.0
 </pre>
 </div>
 </div>
 </div>
 </div>
 </div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -2469,6 +2394,53 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 </div>
 </div>
 </div>
+<div class="jp-Cell-outputWrapper">
+<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
+</div>
+<div class="jp-OutputArea jp-Cell-outputArea">
+<div class="jp-OutputArea-child">
+<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre>Configuration(values={
+  'max_depth': 10,
+  'max_features': 0.6478776855196833,
+  'min_samples_leaf': 1,
+  'min_samples_split': 23,
+}) got an accuracy of 1.0 on the test set.
+
+Configuration(values={
+  'max_depth': 4,
+  'max_features': 0.7985430014261752,
+  'min_samples_leaf': 1,
+  'min_samples_split': 14,
+}) got an accuracy of 1.0 on the test set.
+
+Configuration(values={
+  'max_depth': 12,
+  'max_features': 0.5994266493364799,
+  'min_samples_leaf': 3,
+  'min_samples_split': 3,
+}) got an accuracy of 1.0 on the test set.
+
+Configuration(values={
+  'max_depth': 11,
+  'max_features': 0.3430438209075988,
+  'min_samples_leaf': 2,
+  'min_samples_split': 17,
+}) got an accuracy of 1.0 on the test set.
+
+Configuration(values={
+  'max_depth': 6,
+  'max_features': 0.5899184020756805,
+  'min_samples_leaf': 11,
+  'min_samples_split': 8,
+}) got an accuracy of 1.0 on the test set.
+
+</pre>
+</div>
+</div>
+</div>
+</div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
diff --git a/0.1.0/examples/01.2_Optimizing_RandomForest_using_Ask_Tell/index.html b/0.1.0/examples/01.2_Optimizing_RandomForest_using_Ask_Tell/index.html
index b4764a8..8417417 100644
--- a/0.1.0/examples/01.2_Optimizing_RandomForest_using_Ask_Tell/index.html
+++ b/0.1.0/examples/01.2_Optimizing_RandomForest_using_Ask_Tell/index.html
@@ -1643,8 +1643,8 @@ <h4 id="1-preparing-dataset-and-splits">1 ) Preparing dataset and splits<a class
 </div>
 </div>
 </div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -1675,28 +1675,9 @@ <h4 id="1-preparing-dataset-and-splits">1 ) Preparing dataset and splits<a class
 </div>
 </div>
 </div>
-<div class="jp-Cell-outputWrapper">
-<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
 </div>
-<div class="jp-OutputArea jp-Cell-outputArea">
-<div class="jp-OutputArea-child">
-<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">ModuleNotFoundError</span>                       Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[6], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">datasets</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> load_iris, load_digits, load_wine
-<span class="ansi-green-intense-fg ansi-bold">      4</span> classification <span style="color: rgb(98,98,98)">=</span> {<span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">iris</span><span style="color: rgb(175,0,0)">"</span>: load_iris, <span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">digits</span><span style="color: rgb(175,0,0)">"</span>: load_digits, <span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">wine</span><span style="color: rgb(175,0,0)">"</span>: load_wine}
-
-<span class="ansi-red-fg">ModuleNotFoundError</span>: No module named 'sklearn'</pre>
-</div>
-</div>
-</div>
-</div>
-</div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -1779,26 +1760,6 @@ <h4 id="1-preparing-dataset-and-splits">1 ) Preparing dataset and splits<a class
 </div>
 </div>
 </div>
-<div class="jp-Cell-outputWrapper">
-<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
-</div>
-<div class="jp-OutputArea jp-Cell-outputArea">
-<div class="jp-OutputArea-child">
-<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">ModuleNotFoundError</span>                       Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[7], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">model_selection</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> train_test_split
-<span class="ansi-green-intense-fg ansi-bold">      4</span> <span class="ansi-bold" style="color: rgb(0,135,0)">def</span> <span style="color: rgb(0,0,255)">prepare_dataset</span>(model_type<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">classification</span><span style="color: rgb(175,0,0)">"</span>, dataset<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">None</span>):
-<span class="ansi-green-intense-fg ansi-bold">      6</span>     <span class="ansi-bold" style="color: rgb(0,135,0)">if</span> model_type <span style="color: rgb(98,98,98)">==</span> <span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">classification</span><span style="color: rgb(175,0,0)">"</span>:
-
-<span class="ansi-red-fg">ModuleNotFoundError</span>: No module named 'sklearn'</pre>
-</div>
-</div>
-</div>
-</div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
 <div class="jp-Cell jp-CodeCell jp-Notebook-cell">
@@ -1844,19 +1805,12 @@ <h4 id="1-preparing-dataset-and-splits">1 ) Preparing dataset and splits<a class
 <div class="jp-OutputArea jp-Cell-outputArea">
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[8], line 2</span>
-<span class="ansi-green-intense-fg ansi-bold">      1</span> train_X, train_y, valid_X, valid_y, test_X, test_y, dataset <span style="color: rgb(98,98,98)">=</span> \
-<span class="ansi-green-fg">----&gt; 2</span>     <span class="ansi-yellow-bg">prepare_dataset</span>(model_type<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">classification</span><span style="color: rgb(175,0,0)">"</span>)
-<span class="ansi-green-intense-fg ansi-bold">      4</span> <span style="color: rgb(0,135,0)">print</span>(dataset)
-<span class="ansi-green-intense-fg ansi-bold">      5</span> <span style="color: rgb(0,135,0)">print</span>(<span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">Train size: </span><span class="ansi-bold" style="color: rgb(175,95,135)">{}</span><span class="ansi-bold" style="color: rgb(175,95,0)">\n</span><span style="color: rgb(175,0,0)">Valid size: </span><span class="ansi-bold" style="color: rgb(175,95,135)">{}</span><span class="ansi-bold" style="color: rgb(175,95,0)">\n</span><span style="color: rgb(175,0,0)">Test size: </span><span class="ansi-bold" style="color: rgb(175,95,135)">{}</span><span style="color: rgb(175,0,0)">"</span><span style="color: rgb(98,98,98)">.</span>format(
-<span class="ansi-green-intense-fg ansi-bold">      6</span>     train_X<span style="color: rgb(98,98,98)">.</span>shape, valid_X<span style="color: rgb(98,98,98)">.</span>shape, test_X<span style="color: rgb(98,98,98)">.</span>shape
-<span class="ansi-green-intense-fg ansi-bold">      7</span> ))
-
-<span class="ansi-red-fg">NameError</span>: name 'prepare_dataset' is not defined</pre>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre>wine
+Train size: (124, 13)
+Valid size: (36, 13)
+Test size: (18, 13)
+</pre>
 </div>
 </div>
 </div>
@@ -1873,8 +1827,8 @@ <h4 id="2-function-interface-with-dehb">2 ) Function interface with DEHB<a class
 </div>
 </div>
 </div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -1907,26 +1861,6 @@ <h4 id="2-function-interface-with-dehb">2 ) Function interface with DEHB<a class
 </div>
 </div>
 </div>
-<div class="jp-Cell-outputWrapper">
-<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
-</div>
-<div class="jp-OutputArea jp-Cell-outputArea">
-<div class="jp-OutputArea-child">
-<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">ModuleNotFoundError</span>                       Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[9], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">ensemble</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> RandomForestClassifier
-<span class="ansi-green-intense-fg ansi-bold">      2</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">metrics</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> accuracy_score, make_scorer
-<span class="ansi-green-intense-fg ansi-bold">      5</span> accuracy_scorer <span style="color: rgb(98,98,98)">=</span> make_scorer(accuracy_score)
-
-<span class="ansi-red-fg">ModuleNotFoundError</span>: No module named 'sklearn'</pre>
-</div>
-</div>
-</div>
-</div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
@@ -2100,8 +2034,8 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 </div>
 </div>
 </div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -2176,39 +2110,6 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 </div>
 </div>
 </div>
-<div class="jp-Cell-outputWrapper">
-<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
-</div>
-<div class="jp-OutputArea jp-Cell-outputArea">
-<div class="jp-OutputArea-child">
-<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[12], line 15</span>
-<span class="ansi-green-intense-fg ansi-bold">      5</span> job_info <span style="color: rgb(98,98,98)">=</span> dehb<span style="color: rgb(98,98,98)">.</span>ask()
-<span class="ansi-green-intense-fg ansi-bold">      7</span> <span style="color: rgb(95,135,135)"># Evaluate the configuration on the given fidelity. Here you are free to use</span>
-<span class="ansi-green-intense-fg ansi-bold">      8</span> <span style="color: rgb(95,135,135)"># any technique to compute the result. This job could e.g. be forwarded to</span>
-<span class="ansi-green-intense-fg ansi-bold">      9</span> <span style="color: rgb(95,135,135)"># a worker on your cluster (Which is not required to use Dask).</span>
-<span class="ansi-green-intense-fg ansi-bold">     10</span> <span style="color: rgb(95,135,135)"># The results dict has to contain the keys "cost" and "fitness" with an additional "info"</span>
-<span class="ansi-green-intense-fg ansi-bold">     11</span> <span style="color: rgb(95,135,135)"># dict for additional, user-specific data.</span>
-<span class="ansi-green-intense-fg ansi-bold">     12</span> res <span style="color: rgb(98,98,98)">=</span> target_function(job_info[<span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">config</span><span style="color: rgb(175,0,0)">"</span>], job_info[<span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">fidelity</span><span style="color: rgb(175,0,0)">"</span>],
-<span class="ansi-green-intense-fg ansi-bold">     13</span>                       <span style="color: rgb(95,135,135)"># parameters as **kwargs in target_function</span>
-<span class="ansi-green-intense-fg ansi-bold">     14</span>                       seed<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(98,98,98)">123</span>,
-<span class="ansi-green-fg">---&gt; 15</span>                       train_X<span style="color: rgb(98,98,98)">=</span><span class="ansi-yellow-bg">train_X</span>,
-<span class="ansi-green-intense-fg ansi-bold">     16</span>                       train_y<span style="color: rgb(98,98,98)">=</span>train_y,
-<span class="ansi-green-intense-fg ansi-bold">     17</span>                       valid_X<span style="color: rgb(98,98,98)">=</span>valid_X,
-<span class="ansi-green-intense-fg ansi-bold">     18</span>                       valid_y<span style="color: rgb(98,98,98)">=</span>valid_y,
-<span class="ansi-green-intense-fg ansi-bold">     19</span>                       max_fidelity<span style="color: rgb(98,98,98)">=</span>dehb<span style="color: rgb(98,98,98)">.</span>max_fidelity)
-<span class="ansi-green-intense-fg ansi-bold">     21</span> <span style="color: rgb(95,135,135)"># When the evaluation is done, report the results back to the DEHB controller.</span>
-<span class="ansi-green-intense-fg ansi-bold">     22</span> dehb<span style="color: rgb(98,98,98)">.</span>tell(job_info, res)
-
-<span class="ansi-red-fg">NameError</span>: name 'train_X' is not defined</pre>
-</div>
-</div>
-</div>
-</div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
 <div class="jp-Cell jp-CodeCell jp-Notebook-cell">
@@ -2306,16 +2207,26 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 <div class="jp-OutputArea jp-Cell-outputArea">
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[13], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> <span style="color: rgb(0,135,0)">print</span>(<span style="color: rgb(0,135,0)">len</span>(<span class="ansi-yellow-bg">trajectory</span>), <span style="color: rgb(0,135,0)">len</span>(runtime), <span style="color: rgb(0,135,0)">len</span>(history), end<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(175,0,0)">"</span><span class="ansi-bold" style="color: rgb(175,95,0)">\n</span><span class="ansi-bold" style="color: rgb(175,95,0)">\n</span><span style="color: rgb(175,0,0)">"</span>)
-<span class="ansi-green-intense-fg ansi-bold">      3</span> <span style="color: rgb(95,135,135)"># Last recorded function evaluation</span>
-<span class="ansi-green-intense-fg ansi-bold">      4</span> last_eval <span style="color: rgb(98,98,98)">=</span> history[<span style="color: rgb(98,98,98)">-</span><span style="color: rgb(98,98,98)">1</span>]
-
-<span class="ansi-red-fg">NameError</span>: name 'trajectory' is not defined</pre>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre>50 50 50
+
+Last evaluated configuration, 
+Configuration(values={
+  'max_depth': 12,
+  'max_features': 0.6422911372304849,
+  'min_samples_leaf': 9,
+  'min_samples_split': 8,
+})got a score of -0.9722222222222222, was evaluated at a fidelity of 16.67 and took 0.018 seconds to run.
+The additional info attached: {'test_score': 1.0, 'fidelity': 16.666666666666664}
+
+Best evaluated configuration, 
+Configuration(values={
+  'max_depth': 12,
+  'max_features': 0.6422911372304849,
+  'min_samples_leaf': 8,
+  'min_samples_split': 8,
+}) got an accuracy of 1.0 on the test set.
+</pre>
 </div>
 </div>
 </div>
@@ -2452,27 +2363,26 @@ <h2 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 <div class="jp-OutputArea jp-Cell-outputArea">
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[14], line 7</span>
-<span class="ansi-green-intense-fg ansi-bold">      1</span> <span style="color: rgb(95,135,135)"># Continuing the ask/tell run of DEHB optimization for another 10s</span>
-<span class="ansi-green-intense-fg ansi-bold">      2</span> trajectory, runtime, history <span style="color: rgb(98,98,98)">=</span> dehb<span style="color: rgb(98,98,98)">.</span>run(
-<span class="ansi-green-intense-fg ansi-bold">      3</span>     total_cost<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(98,98,98)">10</span>,
-<span class="ansi-green-intense-fg ansi-bold">      4</span>     verbose<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">False</span>,
-<span class="ansi-green-intense-fg ansi-bold">      5</span>     save_intermediate<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">False</span>,
-<span class="ansi-green-intense-fg ansi-bold">      6</span>     seed<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(98,98,98)">123</span>,
-<span class="ansi-green-fg">----&gt; 7</span>     train_X<span style="color: rgb(98,98,98)">=</span><span class="ansi-yellow-bg">train_X</span>,
-<span class="ansi-green-intense-fg ansi-bold">      8</span>     train_y<span style="color: rgb(98,98,98)">=</span>train_y,
-<span class="ansi-green-intense-fg ansi-bold">      9</span>     valid_X<span style="color: rgb(98,98,98)">=</span>valid_X,
-<span class="ansi-green-intense-fg ansi-bold">     10</span>     valid_y<span style="color: rgb(98,98,98)">=</span>valid_y,
-<span class="ansi-green-intense-fg ansi-bold">     11</span>     max_fidelity<span style="color: rgb(98,98,98)">=</span>dehb<span style="color: rgb(98,98,98)">.</span>max_fidelity
-<span class="ansi-green-intense-fg ansi-bold">     12</span> )
-<span class="ansi-green-intense-fg ansi-bold">     13</span> best_config <span style="color: rgb(98,98,98)">=</span> dehb<span style="color: rgb(98,98,98)">.</span>vector_to_configspace(dehb<span style="color: rgb(98,98,98)">.</span>inc_config)
-<span class="ansi-green-intense-fg ansi-bold">     15</span> <span style="color: rgb(95,135,135)"># Creating a model using the best configuration found</span>
-
-<span class="ansi-red-fg">NameError</span>: name 'train_X' is not defined</pre>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre>460 460 460
+
+Last evaluated configuration, 
+Configuration(values={
+  'max_depth': 8,
+  'max_features': 0.3567601906530009,
+  'min_samples_leaf': 5,
+  'min_samples_split': 2,
+})got a score of -1.0, was evaluated at a fidelity of 50.00 and took 0.050 seconds to run.
+The additional info attached: {'test_score': 1.0, 'fidelity': 50.0}
+
+Best evaluated configuration, 
+Configuration(values={
+  'max_depth': 10,
+  'max_features': 0.6012257129543677,
+  'min_samples_leaf': 2,
+  'min_samples_split': 3,
+}) got an accuracy of 1.0 on the test set.
+</pre>
 </div>
 </div>
 </div>
diff --git a/0.1.0/examples/02_using_DEHB_without_ConfigSpace/index.html b/0.1.0/examples/02_using_DEHB_without_ConfigSpace/index.html
index 1405824..9f78733 100644
--- a/0.1.0/examples/02_using_DEHB_without_ConfigSpace/index.html
+++ b/0.1.0/examples/02_using_DEHB_without_ConfigSpace/index.html
@@ -1705,16 +1705,12 @@ <h3 id="defining-the-target_function">Defining the target_function<a class="anch
 <div class="jp-OutputArea jp-Cell-outputArea">
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">ModuleNotFoundError</span>                       Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[3], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">datasets</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> load_digits, load_wine
-<span class="ansi-green-intense-fg ansi-bold">      2</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">model_selection</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> train_test_split
-<span class="ansi-green-intense-fg ansi-bold">      3</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">sklearn</span><span class="ansi-bold" style="color: rgb(0,0,255)">.</span><span class="ansi-bold" style="color: rgb(0,0,255)">ensemble</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> RandomForestClassifier
-
-<span class="ansi-red-fg">ModuleNotFoundError</span>: No module named 'sklearn'</pre>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre>digits
+Train size: (1257, 64)
+Valid size: (540, 64)
+Test size: (180, 64)
+</pre>
 </div>
 </div>
 </div>
@@ -1731,8 +1727,8 @@ <h3 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 </div>
 </div>
 </div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -1777,35 +1773,9 @@ <h3 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 </div>
 </div>
 </div>
-<div class="jp-Cell-outputWrapper">
-<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
-</div>
-<div class="jp-OutputArea jp-Cell-outputArea">
-<div class="jp-OutputArea-child">
-<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[4], line 5</span>
-<span class="ansi-green-intense-fg ansi-bold">      1</span> <span class="ansi-bold" style="color: rgb(0,135,0)">from</span> <span class="ansi-bold" style="color: rgb(0,0,255)">dehb</span> <span class="ansi-bold" style="color: rgb(0,135,0)">import</span> DEHB
-<span class="ansi-green-intense-fg ansi-bold">      4</span> dehb <span style="color: rgb(98,98,98)">=</span> DEHB(
-<span class="ansi-green-fg">----&gt; 5</span>     f<span style="color: rgb(98,98,98)">=</span><span class="ansi-yellow-bg">target_function</span>, 
-<span class="ansi-green-intense-fg ansi-bold">      6</span>     dimensions<span style="color: rgb(98,98,98)">=</span>dimensions, 
-<span class="ansi-green-intense-fg ansi-bold">      7</span>     min_fidelity<span style="color: rgb(98,98,98)">=</span>min_fidelity, 
-<span class="ansi-green-intense-fg ansi-bold">      8</span>     max_fidelity<span style="color: rgb(98,98,98)">=</span>max_fidelity,
-<span class="ansi-green-intense-fg ansi-bold">      9</span>     n_workers<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(98,98,98)">1</span>,
-<span class="ansi-green-intense-fg ansi-bold">     10</span>     output_path<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">./temp</span><span style="color: rgb(175,0,0)">"</span>
-<span class="ansi-green-intense-fg ansi-bold">     11</span> )
-
-<span class="ansi-red-fg">NameError</span>: name 'target_function' is not defined</pre>
-</div>
-</div>
-</div>
-</div>
 </div>
-</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell">
+</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -1852,35 +1822,6 @@ <h3 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 </div>
 </div>
 </div>
-<div class="jp-Cell-outputWrapper">
-<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
-</div>
-<div class="jp-OutputArea jp-Cell-outputArea">
-<div class="jp-OutputArea-child">
-<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[5], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> trajectory, runtime, history <span style="color: rgb(98,98,98)">=</span> <span class="ansi-yellow-bg">dehb</span><span style="color: rgb(98,98,98)">.</span>run(
-<span class="ansi-green-intense-fg ansi-bold">      2</span>     total_cost<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(98,98,98)">10</span>, 
-<span class="ansi-green-intense-fg ansi-bold">      3</span>     verbose<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">False</span>,
-<span class="ansi-green-intense-fg ansi-bold">      4</span>     save_intermediate<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">False</span>,
-<span class="ansi-green-intense-fg ansi-bold">      5</span>     seed<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(98,98,98)">123</span>,
-<span class="ansi-green-intense-fg ansi-bold">      6</span>     train_X<span style="color: rgb(98,98,98)">=</span>train_X,
-<span class="ansi-green-intense-fg ansi-bold">      7</span>     train_y<span style="color: rgb(98,98,98)">=</span>train_y,
-<span class="ansi-green-intense-fg ansi-bold">      8</span>     valid_X<span style="color: rgb(98,98,98)">=</span>valid_X,
-<span class="ansi-green-intense-fg ansi-bold">      9</span>     valid_y<span style="color: rgb(98,98,98)">=</span>valid_y,
-<span class="ansi-green-intense-fg ansi-bold">     10</span>     max_fidelity<span style="color: rgb(98,98,98)">=</span>dehb<span style="color: rgb(98,98,98)">.</span>max_fidelity,
-<span class="ansi-green-intense-fg ansi-bold">     11</span>     param_space<span style="color: rgb(98,98,98)">=</span>param_space
-<span class="ansi-green-intense-fg ansi-bold">     12</span> )
-
-<span class="ansi-red-fg">NameError</span>: name 'dehb' is not defined</pre>
-</div>
-</div>
-</div>
-</div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell">
 <div class="jp-Cell jp-CodeCell jp-Notebook-cell">
@@ -1916,15 +1857,11 @@ <h3 id="running-dehb">Running DEHB<a class="anchor-link" href="#running-dehb">¶
 <div class="jp-OutputArea jp-Cell-outputArea">
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[6], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> <span style="color: rgb(0,135,0)">print</span>(<span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">Incumbent score: </span><span class="ansi-bold" style="color: rgb(175,95,135)">{}</span><span style="color: rgb(175,0,0)">"</span><span style="color: rgb(98,98,98)">.</span>format(<span class="ansi-yellow-bg">dehb</span><span style="color: rgb(98,98,98)">.</span>inc_score))
-<span class="ansi-green-intense-fg ansi-bold">      2</span> <span style="color: rgb(0,135,0)">print</span>(<span style="color: rgb(175,0,0)">"</span><span style="color: rgb(175,0,0)">Incumbent configuration:</span><span class="ansi-bold" style="color: rgb(175,95,0)">\n</span><span class="ansi-bold" style="color: rgb(175,95,135)">{}</span><span style="color: rgb(175,0,0)">"</span><span style="color: rgb(98,98,98)">.</span>format(transform_space(param_space, dehb<span style="color: rgb(98,98,98)">.</span>inc_config)))
-
-<span class="ansi-red-fg">NameError</span>: name 'dehb' is not defined</pre>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre>Incumbent score: -0.9611111111111111
+Incumbent configuration:
+{'max_depth': 9, 'min_samples_split': 3, 'max_features': 0.3990411414400532, 'min_samples_leaf': 1}
+</pre>
 </div>
 </div>
 </div>
@@ -1995,24 +1932,9 @@ <h5 id="evaluating-the-incumbent">Evaluating the incumbent<a class="anchor-link"
 <div class="jp-OutputArea jp-Cell-outputArea">
 <div class="jp-OutputArea-child">
 <div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
-<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr" tabindex="0">
-<pre>
-<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
-<span class="ansi-red-fg">NameError</span>                                 Traceback (most recent call last)
-Cell <span class="ansi-green-fg">In[7], line 1</span>
-<span class="ansi-green-fg">----&gt; 1</span> model <span style="color: rgb(98,98,98)">=</span> <span class="ansi-yellow-bg">RandomForestClassifier</span>(
-<span class="ansi-green-intense-fg ansi-bold">      2</span>     <span style="color: rgb(98,98,98)">*</span><span style="color: rgb(98,98,98)">*</span>transform_space(param_space, dehb<span style="color: rgb(98,98,98)">.</span>inc_config),
-<span class="ansi-green-intense-fg ansi-bold">      3</span>     n_estimators<span style="color: rgb(98,98,98)">=</span><span style="color: rgb(0,135,0)">int</span>(max_fidelity),
-<span class="ansi-green-intense-fg ansi-bold">      4</span>     bootstrap<span style="color: rgb(98,98,98)">=</span><span class="ansi-bold" style="color: rgb(0,135,0)">True</span>,
-<span class="ansi-green-intense-fg ansi-bold">      5</span>     random_state<span style="color: rgb(98,98,98)">=</span>seed,
-<span class="ansi-green-intense-fg ansi-bold">      6</span> )
-<span class="ansi-green-intense-fg ansi-bold">      7</span> model<span style="color: rgb(98,98,98)">.</span>fit(
-<span class="ansi-green-intense-fg ansi-bold">      8</span>     np<span style="color: rgb(98,98,98)">.</span>concatenate((train_X, valid_X)),
-<span class="ansi-green-intense-fg ansi-bold">      9</span>     np<span style="color: rgb(98,98,98)">.</span>concatenate((train_y, valid_y))
-<span class="ansi-green-intense-fg ansi-bold">     10</span> )
-<span class="ansi-green-intense-fg ansi-bold">     11</span> test_accuracy <span style="color: rgb(98,98,98)">=</span> accuracy_scorer(model, test_X, test_y)
-
-<span class="ansi-red-fg">NameError</span>: name 'RandomForestClassifier' is not defined</pre>
+<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="text/plain" tabindex="0">
+<pre>Test accuracy: 0.9944444444444445
+</pre>
 </div>
 </div>
 </div>
diff --git a/0.1.0/getting_started/single_worker/index.html b/0.1.0/getting_started/single_worker/index.html
index 8f209d5..7bdbc45 100644
--- a/0.1.0/getting_started/single_worker/index.html
+++ b/0.1.0/getting_started/single_worker/index.html
@@ -794,10 +794,10 @@ <h3 id="basic-single-worker-setup">Basic single worker setup<a class="headerlink
 </code></pre></div>
 <div class="result">
 <div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="n">Configuration</span><span class="p">(</span><span class="n">values</span><span class="o">=</span><span class="p">{</span>
-<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a>  <span class="s1">&#39;x0&#39;</span><span class="p">:</span> <span class="mf">5.309818877459886</span><span class="p">,</span>
-<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a>  <span class="s1">&#39;x1&#39;</span><span class="p">:</span> <span class="s1">&#39;red&#39;</span><span class="p">,</span>
+<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a>  <span class="s1">&#39;x0&#39;</span><span class="p">:</span> <span class="mf">8.081442413593486</span><span class="p">,</span>
+<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a>  <span class="s1">&#39;x1&#39;</span><span class="p">:</span> <span class="s1">&#39;green&#39;</span><span class="p">,</span>
 <a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="p">})</span>
-<a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="p">{</span><span class="s1">&#39;fitness&#39;</span><span class="p">:</span> <span class="mf">6.085055062123084</span><span class="p">,</span> <span class="s1">&#39;cost&#39;</span><span class="p">:</span> <span class="mf">5.309818877459886</span><span class="p">}</span>
+<a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="p">{</span><span class="s1">&#39;fitness&#39;</span><span class="p">:</span> <span class="mf">8.426257816719632</span><span class="p">,</span> <span class="s1">&#39;cost&#39;</span><span class="p">:</span> <span class="mf">8.081442413593486</span><span class="p">}</span>
 </code></pre></div>
 </div></p>
 <p>Finally, we can setup our optimizer and run DEHB:</p>
@@ -824,9 +824,9 @@ <h3 id="basic-single-worker-setup">Basic single worker setup<a class="headerlink
 <a id="__codelineno-0-21" name="__codelineno-0-21" href="#__codelineno-0-21"></a><span class="nb">print</span><span class="p">(</span><span class="s2">&quot;fidelity&quot;</span><span class="p">,</span> <span class="n">fidelity</span><span class="p">)</span>
 </code></pre></div>
 <div class="result">
-<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="n">config</span> <span class="p">[</span><span class="mf">0.6940348967446345</span><span class="p">,</span> <span class="mf">0.25</span><span class="p">]</span>
-<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="n">fitness</span> <span class="mf">8.181542779533341</span>
-<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="n">runtime</span> <span class="mf">7.8582442772124415</span>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="n">config</span> <span class="p">[</span><span class="mf">0.4899593969425501</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">]</span>
+<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="n">fitness</span> <span class="mf">6.898280767484525</span>
+<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="n">runtime</span> <span class="mf">6.429715778597851</span>
 <a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="n">fidelity</span> <span class="mf">3.0</span>
 </code></pre></div>
 </div>
diff --git a/0.1.0/search/search_index.json b/0.1.0/search/search_index.json
index bae3052..c7e3a64 100644
--- a/0.1.0/search/search_index.json
+++ b/0.1.0/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Welcome to DEHB's documentation!","text":""},{"location":"#introduction","title":"Introduction","text":"<p><code>dehb</code> is a python package implementing the DEHB algorithm. It offers an intuitive interface to optimize user-defined problems using DEHB.</p> <p>This documentation explains how to use <code>dehb</code> and demonstrates its features. In the following section you will be guided how to install the <code>dehb</code> package and how to use it in your own projects. Examples with more hands-on material can be found in the examples folder.</p>"},{"location":"#installation","title":"Installation","text":"<p>To start using the <code>dehb</code> package, you can install it via pip. You can either install the package right from git or install it as an editable package to modify the code and rerun your experiments:</p> <pre><code># Install from pypi\npip install dehb\n</code></pre> <p>From Source</p> <p>To install directly from from source</p> <pre><code>git clone https://github.com/automl/DEHB.git\npip install -e DEHB  # -e stands for editable, lets you modify the code and rerun things\n</code></pre>"},{"location":"#contributing","title":"Contributing","text":"<p>Please have a look at our contributing guidelines.</p>"},{"location":"#to-cite-the-paper-or-code","title":"To cite the paper or code","text":"<p>If you use DEHB in one of your research projects, please cite our paper(s): <pre><code>@inproceedings{awad-ijcai21,\n  author    = {N. Awad and N. Mallik and F. Hutter},\n  title     = {{DEHB}: Evolutionary Hyberband for Scalable, Robust and Efficient Hyperparameter Optimization},\n  pages     = {2147--2153},\n  booktitle = {Proceedings of the Thirtieth International Joint Conference on\n               Artificial Intelligence, {IJCAI-21}},\n  publisher = {ijcai.org},\n  editor    = {Z. Zhou},\n  year      = {2021}\n}\n</code></pre></p>"},{"location":"examples/00_interfacing_DEHB/","title":"Interfacing DEHB","text":"In\u00a0[1]: Copied! <pre>import time\nimport warnings\nimport numpy as np\nimport ConfigSpace\nfrom typing import Dict, Union, List\n\nwarnings.filterwarnings('ignore')\n</pre> import time import warnings import numpy as np import ConfigSpace from typing import Dict, Union, List  warnings.filterwarnings('ignore') <p>DEHB was designed to be an algorithm for Hyper Parameter Optimization (HPO). DEHB uses Differential Evolution (DE) under-the-hood as an Evolutionary Algorithm to power the black-box optimization that HPO problems pose. DE is a black-box optimization algorithm that generates candidate configurations $x$, to the black-box function $f(x)$, that is being optimized. The $x$ is evaluated by the black-box and the corresponding response $y$ is made available to the DE algorithm, which can then use this observation ($x$, $y$), along with previous such observations, to suggest a new candidate $x$ for the next evaluation.</p> <p>DEHB also uses Hyperband along with DE, to allow for cheaper approximations of the actual evaluations of $x$. Let $f(x)$ be the validation error of training a multilayer perceptron (MLP) on the complete training set. Multi-fidelity algorithms such as Hyperband, allow for cheaper approximations along a possible fidelity. For the MLP, a subset of the dataset maybe a cheaper approximation to the full data set evaluation. Whereas the fidelity can be quantifies as the fraction of the dataset used to evaluate the configuration $x$, instead of the full dataset. Such approximations can allow sneak-peek into the black-box, potentially revealing certain landscape feature of f(x), thus rendering it a gray-box and not completely opaque and black!</p> <p>The $z$ parameter is the fidelity parameter to the black-box function. If $z \\in [fidelity_{min}, fidelity_{max}]$, then $f(x, fidelity_{max})$ would be equivalent to the black-box case of $f(x)$.</p> <p></p> <p>HPO algorithms optimize such black/gray box by wrapping around this target function an interface, by which the algorithms can suggest new $x$ and also consume the result of the corresponding evaluation to store a collection of such ($x$, $y$) pairs. Therefore, to run DEHB, the most essential component required as input is the target function to optimize. Since DEHB can leverage a Hyperband, the target function interface should account for possible input of fidelity too.</p> In\u00a0[2]: Copied! <pre>def target_function(\n    x: Union[ConfigSpace.Configuration, List, np.array], \n    fidelity: Union[int, float] = None,\n    **kwargs\n) -&gt; Dict:\n    \"\"\" Target/objective function to optimize\n    \n    Parameters\n    ----------\n    x : configuration that DEHB wants to evaluate\n    fidelity : parameter determining cheaper evaluations\n    \n    Returns\n    -------\n    dict\n    \"\"\"\n    # ...\n    # write your code here\n    # ...\n    \n    # remove the code snippet below\n    start = time.time()\n    y = np.random.uniform()  # placeholder response of evaluation\n    time.sleep(fidelity)       # simulates runtime (mostly proportional to fidelity)\n    cost = time.time() - start\n    \n    # result dict passed to DE/DEHB as function evaluation output\n    result = {\n        \"fitness\": y,  # must-have key that DE/DEHB minimizes\n        \"cost\": cost,  # must-have key that associates cost/runtime \n        \"info\": dict() # optional key containing a dictionary of additional info\n    }\n    return result\n</pre> def target_function(     x: Union[ConfigSpace.Configuration, List, np.array],      fidelity: Union[int, float] = None,     **kwargs ) -&gt; Dict:     \"\"\" Target/objective function to optimize          Parameters     ----------     x : configuration that DEHB wants to evaluate     fidelity : parameter determining cheaper evaluations          Returns     -------     dict     \"\"\"     # ...     # write your code here     # ...          # remove the code snippet below     start = time.time()     y = np.random.uniform()  # placeholder response of evaluation     time.sleep(fidelity)       # simulates runtime (mostly proportional to fidelity)     cost = time.time() - start          # result dict passed to DE/DEHB as function evaluation output     result = {         \"fitness\": y,  # must-have key that DE/DEHB minimizes         \"cost\": cost,  # must-have key that associates cost/runtime          \"info\": dict() # optional key containing a dictionary of additional info     }     return result     <p>This <code>target_function</code> is the problem that needs to be solved, or the function to be optimized. The other prerequisite for this function is therefore the domain for its input $x$. In other words, the definition and constraints of the search space for DEHB.</p> <p>The DE component inside DEHB, assumes that the input domain is scaled to a unit hypercube. This is essential for effective search. If the ConfigSpace library is used to define the domain of $x$, or the parameters of the search space, DEHB can internally handle the scaling to and from the unit hypercube required for search. If ConfigSpace is not used, one needs to additionally handle the scaling of the parameters as an extra interface between DEHB and the target function (or encode it within the target function).</p> <p>For this template notebook, we will illustrate how a ConfigSpace parameter space can be created.</p> In\u00a0[3]: Copied! <pre>import ConfigSpace\n\n\ndef create_search_space():\n    # Creating a one-dimensional search space of real numbers in [3, 10]\n    cs = ConfigSpace.ConfigurationSpace()\n    cs.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter(\"x0\", lower=3, upper=10, log=False))\n    return cs\n\n\ncs = create_search_space()\nprint(cs)\n</pre> import ConfigSpace   def create_search_space():     # Creating a one-dimensional search space of real numbers in [3, 10]     cs = ConfigSpace.ConfigurationSpace()     cs.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter(\"x0\", lower=3, upper=10, log=False))     return cs   cs = create_search_space() print(cs) <pre>Configuration space object:\n  Hyperparameters:\n    x0, Type: UniformFloat, Range: [3.0, 10.0], Default: 6.5\n\n</pre> In\u00a0[4]: Copied! <pre># Finding dimensionality of search space\ndimensions = len(cs.get_hyperparameters())\nprint(dimensions)\n</pre> # Finding dimensionality of search space dimensions = len(cs.get_hyperparameters()) print(dimensions) <pre>1\n</pre> In\u00a0[5]: Copied! <pre># Sampling a random configuration\ncs.sample_configuration()\n</pre> # Sampling a random configuration cs.sample_configuration() Out[5]: <pre>Configuration(values={\n  'x0': 8.627121338040169,\n})</pre> <p>The ConfigSpace documentation can be referred to for more complicated search space creation.</p> <p>In a similar vein, for a complete gray-box definition, the fidelity domain needs to be defined too. For the earlier example of dataset fractions, the fidelity upper limit cannot clearly exceed 1, and therefore $[0.3, 1]$ is a legitimate definition for such a fidelity. In this template example, we shall simply define the lower and upper range of the fidelity as two parameters that can be input to DEHB. Given that fidelity is being used to simulate cost of runtime in our sample <code>target_function</code>, we shall use a reasonable time range as a placeholder for the fidelity in this case.</p> In\u00a0[6]: Copied! <pre>min_fidelity, max_fidelity = (0.1, 3)\n</pre> min_fidelity, max_fidelity = (0.1, 3)  <p>The above definitions are all the information that DEHB needs about a problem. We are now in a position to call upon DEHB and start running it, to tune $x$.</p> In\u00a0[7]: Copied! <pre>from dehb import DEHB\n\n\ndehb = DEHB(\n    f=target_function,\n    dimensions=dimensions,\n    cs=cs,\n    min_fidelity=min_fidelity,\n    max_fidelity=max_fidelity,\n    output_path=\"./temp\",\n    n_workers=1        # set to &gt;1 to utilize parallel workers\n)\n\n# NOTE: the other hyperparameters to DEHB have been set to certain defaults that were \n# empirically determined through related literature, ablation analysis and other experiments,\n# but can be tuned as desired\n</pre> from dehb import DEHB   dehb = DEHB(     f=target_function,     dimensions=dimensions,     cs=cs,     min_fidelity=min_fidelity,     max_fidelity=max_fidelity,     output_path=\"./temp\",     n_workers=1        # set to &gt;1 to utilize parallel workers )  # NOTE: the other hyperparameters to DEHB have been set to certain defaults that were  # empirically determined through related literature, ablation analysis and other experiments, # but can be tuned as desired In\u00a0[8]: Copied! <pre>?dehb.run\n</pre> ?dehb.run <p>DEHB allows the option of 3 different resources for its runtime budget:</p> In\u00a0[9]: Copied! <pre>_, _, _ = dehb.run(brackets=1, verbose=False, save_intermediate=True)\nprint(dehb.vector_to_configspace(dehb.inc_config))\n</pre> _, _, _ = dehb.run(brackets=1, verbose=False, save_intermediate=True) print(dehb.vector_to_configspace(dehb.inc_config)) <pre>Configuration(values={\n  'x0': 6.34288307846435,\n})\n</pre> In\u00a0[10]: Copied! <pre># allows optimization to restart from the beginning by forgetting all observations\ndehb.reset()  \n\n_, _, _ = dehb.run(fevals=20, verbose=False, save_intermediate=True)\nprint(dehb.get_incumbents())\n</pre> # allows optimization to restart from the beginning by forgetting all observations dehb.reset()    _, _, _ = dehb.run(fevals=20, verbose=False, save_intermediate=True) print(dehb.get_incumbents()) <pre>2024-02-17 14:47:50.136 | INFO     | dehb.optimizers.dehb:reset:121 - \n\nRESET at 02/17/24 14:47:50 UTC\n\n\n</pre> <pre>(Configuration(values={\n  'x0': 9.263648958886906,\n}), 0.018104529362575228)\n</pre> In\u00a0[11]: Copied! <pre># allows optimization to restart from the beginning by forgetting all observations\ndehb.reset()  \n\n_, _, _ = dehb.run(total_cost=10, verbose=False, save_intermediate=True)  # run for 10s\nprint(dehb.get_incumbents())\n</pre> # allows optimization to restart from the beginning by forgetting all observations dehb.reset()    _, _, _ = dehb.run(total_cost=10, verbose=False, save_intermediate=True)  # run for 10s print(dehb.get_incumbents()) <pre>2024-02-17 14:47:53.035 | INFO     | dehb.optimizers.dehb:reset:121 - \n\nRESET at 02/17/24 14:47:53 UTC\n\n\n</pre> <pre>(Configuration(values={\n  'x0': 5.566303183270801,\n}), 0.02030815133180952)\n</pre> <p>Each <code>dehb</code> object initialized maintains a <code>log</code> file in the <code>output_path</code> specified, where the progress and other debugging information is updated. While every alternative DEHB evaluation (and after full optimization), an <code>incumbent.json</code> file is written to disk <code>output_path</code>, with the incumbent (best seen so far) configuration and its corresponding score.</p> <p>We now rerun DEHB in parallel with 2 workers, and show that the incumbents can be retrieved in any of the following manner:</p> In\u00a0[12]: Copied! <pre>dehb = DEHB(\n    f=target_function,\n    dimensions=dimensions,\n    cs=cs,\n    min_fidelity=min_fidelity,\n    max_fidelity=max_fidelity,\n    output_path=\"./temp\",\n    n_workers=2\n)\ntrajectory, runtime, history = dehb.run(\n    total_cost=20, verbose=False,\n)\n\nprint(dehb.get_incumbents())\n</pre> dehb = DEHB(     f=target_function,     dimensions=dimensions,     cs=cs,     min_fidelity=min_fidelity,     max_fidelity=max_fidelity,     output_path=\"./temp\",     n_workers=2 ) trajectory, runtime, history = dehb.run(     total_cost=20, verbose=False, )  print(dehb.get_incumbents()) <pre>(Configuration(values={\n  'x0': 4.447543364972189,\n}), 0.018084296582581594)\n</pre> In\u00a0[13]: Copied! <pre>print(dehb.vector_to_configspace(dehb.inc_config))  # config as ConfigSpace configuration\n</pre> print(dehb.vector_to_configspace(dehb.inc_config))  # config as ConfigSpace configuration <pre>Configuration(values={\n  'x0': 4.447543364972189,\n})\n</pre> In\u00a0[14]: Copied! <pre>print(trajectory[-1], dehb.inc_score)\nprint(dehb.vector_to_configspace(dehb.inc_config))\n</pre> print(trajectory[-1], dehb.inc_score) print(dehb.vector_to_configspace(dehb.inc_config)) <pre>0.018084296582581594 0.018084296582581594\nConfiguration(values={\n  'x0': 4.447543364972189,\n})\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/00_interfacing_DEHB/#interfacing-dehb","title":"Interfacing DEHB\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#how-to-read-this-notebook","title":"How to read this notebook\u00b6","text":"<p>This notebook is designed to serve as a high-level, highly abstracted view of DEHB and how it can be used. The examples here are mere placeholders and only offer an interface to run DEHB on toy or actual problems.</p>"},{"location":"examples/00_interfacing_DEHB/#getting-started-with-dehb","title":"Getting started with DEHB\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#sample-interface-for-target-function-that-dehb-optimizes","title":"Sample interface for target function that DEHB optimizes\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#defining-a-sample-search-space","title":"Defining a sample search space\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#defining-fidelity-range-for-the-target-function","title":"Defining fidelity range for the target function\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#instantiating-and-running-dehb","title":"Instantiating and running DEHB\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#1-running-dehb-for-a-certain-number-of-successive-halving-brackets","title":"1) Running DEHB for a certain number of (successive halving) brackets\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#2-running-dehb-for-total-number-of-function-evaluations","title":"2) Running DEHB for total number of function evaluations\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#3-running-dehb-for-total-amount-of-wallclock-time","title":"3) Running DEHB for total amount of wallclock time\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#conclusion","title":"Conclusion\u00b6","text":"<p>As detailed above, the problem definition needs to be input to DEHB as the following information:</p> <ul> <li>the target_function (<code>f</code>) that is the primary black-box function to optimize</li> <li>the fidelity range of <code>min_fidelity</code> and <code>max_fidelity</code> that allows the cheaper, faster gray-box optimization of <code>f</code></li> <li>the search space or the input domain of the function <code>f</code>, that can be represented as a <code>ConfigSpace</code> object and passed to DEHB at initialization</li> </ul> <p>Following which, DEHB can be run for any amount of practical real-world budget. It can be run for either:</p> <ul> <li>a total amount of actual wallclock time, example one day (~86400 seconds), or</li> <li>a total number of function evaluations, or the number of times we want the black-box to be accessed for evaluation, across all fidelities</li> <li>the total number of brackets we want to run the DEHB algorithm for</li> </ul> <p>DEHB will terminate once its chosen runtime budget is exhausted, and report the incumbent found. DEHB, as an anytime algorithm, constantly writes to disk a lightweight <code>json</code> file with the best found configuration and its score seen till that point.</p>"},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/","title":"Optimizing RandomForest using DEHB","text":"In\u00a0[1]: Copied! <pre>import time\nimport numpy as np\nimport warnings\n\n\nseed = 123\nnp.random.seed(seed)\nwarnings.filterwarnings('ignore')\n</pre> import time import numpy as np import warnings   seed = 123 np.random.seed(seed) warnings.filterwarnings('ignore') <p>The problem defined here is to optimize a Random Forest model, on any given dataset, using DEHB. The hyperparameters chosen to be optimized are:</p> <ul> <li><code>max_depth</code></li> <li><code>min_samples_split</code></li> <li><code>max_features</code></li> <li><code>min_samples_leaf</code> while the <code>n_estimators</code> hyperparameter to the Random Forest is chosen to be a fidelity parameter instead. Lesser number of trees ($&lt;10$) in the Random Forest may not allow adequate ensembling for the grouped prediction to be significantly better than the individual tree predictions. Whereas a large number of trees (~$100$) often give accurate predictions but is naturally slower to train and predict on account of more trees to train. Therefore, a smaller <code>n_estimators</code> can be used as a cheaper approximation of the actual fidelity of <code>n_estimators=100</code>.</li> </ul> In\u00a0[2]: Copied! <pre>min_fidelity, max_fidelity = 2, 50\n</pre> min_fidelity, max_fidelity = 2, 50 <p>For the remaining $4$ hyperparameters, the search space can be created as a <code>ConfigSpace</code> object, with the domain of individual parameters defined.</p> In\u00a0[3]: Copied! <pre>import ConfigSpace as CS\n\n\ndef create_search_space(seed=123):\n    \"\"\"Parameter space to be optimized --- contains the hyperparameters\n    \"\"\"\n    cs = CS.ConfigurationSpace(seed=seed)\n\n    cs.add_hyperparameters([\n        CS.UniformIntegerHyperparameter(\n            'max_depth', lower=1, upper=15, default_value=2, log=False\n        ),\n        CS.UniformIntegerHyperparameter(\n            'min_samples_split', lower=2, upper=128, default_value=2, log=True\n        ),\n        CS.UniformFloatHyperparameter(\n            'max_features', lower=0.1, upper=0.9, default_value=0.5, log=False\n        ),\n        CS.UniformIntegerHyperparameter(\n            'min_samples_leaf', lower=1, upper=64, default_value=1, log=True\n        ),\n    ])\n    return cs\n</pre> import ConfigSpace as CS   def create_search_space(seed=123):     \"\"\"Parameter space to be optimized --- contains the hyperparameters     \"\"\"     cs = CS.ConfigurationSpace(seed=seed)      cs.add_hyperparameters([         CS.UniformIntegerHyperparameter(             'max_depth', lower=1, upper=15, default_value=2, log=False         ),         CS.UniformIntegerHyperparameter(             'min_samples_split', lower=2, upper=128, default_value=2, log=True         ),         CS.UniformFloatHyperparameter(             'max_features', lower=0.1, upper=0.9, default_value=0.5, log=False         ),         CS.UniformIntegerHyperparameter(             'min_samples_leaf', lower=1, upper=64, default_value=1, log=True         ),     ])     return cs In\u00a0[4]: Copied! <pre>cs = create_search_space(seed)\nprint(cs)\n</pre> cs = create_search_space(seed) print(cs) <pre>Configuration space object:\n  Hyperparameters:\n    max_depth, Type: UniformInteger, Range: [1, 15], Default: 2\n    max_features, Type: UniformFloat, Range: [0.1, 0.9], Default: 0.5\n    min_samples_leaf, Type: UniformInteger, Range: [1, 64], Default: 1, on log-scale\n    min_samples_split, Type: UniformInteger, Range: [2, 128], Default: 2, on log-scale\n\n</pre> In\u00a0[5]: Copied! <pre>dimensions = len(cs.get_hyperparameters())\nprint(\"Dimensionality of search space: {}\".format(dimensions))\n</pre> dimensions = len(cs.get_hyperparameters()) print(\"Dimensionality of search space: {}\".format(dimensions)) <pre>Dimensionality of search space: 4\n</pre> <p>Now the primary black/gray-box interface to the Random Forest model needs to be built for DEHB to query. As given in the <code>00_interfacing_DEHB</code> notebook, this function will have a signature akin to: <code>target_function(config, fidelity)</code>, and return a two-element tuple of the <code>score</code> and <code>cost</code>. It must be noted that DEHB minimizes and therefore the <code>score</code> being returned by this <code>target_function</code> should account for it.</p> <p>In this example, the target function trains a Random Forest model on a dataset. We load a dataset here and maintain a fixed, train-validation-test split for one complete run. Multiple DEHB runs can therefore optimize on the same validation split, and evaluate final performance on the same test set.</p> In\u00a0[6]: Copied! <pre>from sklearn.datasets import load_iris, load_digits, load_wine\n\n\nclassification = {\"iris\": load_iris, \"digits\": load_digits, \"wine\": load_wine}\n</pre> from sklearn.datasets import load_iris, load_digits, load_wine   classification = {\"iris\": load_iris, \"digits\": load_digits, \"wine\": load_wine} <pre>\n---------------------------------------------------------------------------\nModuleNotFoundError                       Traceback (most recent call last)\nCell In[6], line 1\n----&gt; 1 from sklearn.datasets import load_iris, load_digits, load_wine\n      4 classification = {\"iris\": load_iris, \"digits\": load_digits, \"wine\": load_wine}\n\nModuleNotFoundError: No module named 'sklearn'</pre> In\u00a0[7]: Copied! <pre>from sklearn.model_selection import train_test_split\n\n\ndef prepare_dataset(model_type=\"classification\", dataset=None):\n\n    if model_type == \"classification\":\n        if dataset is None:\n            dataset = np.random.choice(list(classification.keys())) \n        _data = classification[dataset]()\n    else:\n        if dataset is None:\n            dataset = np.random.choice(list(regression.keys()))\n        _data = regression[dataset]()\n\n    train_X, rest_X, train_y, rest_y = train_test_split(\n      _data.get(\"data\"), \n      _data.get(\"target\"), \n      train_size=0.7, \n      shuffle=True, \n      random_state=seed\n    )\n    \n    # 10% test and 20% validation data\n    valid_X, test_X, valid_y, test_y = train_test_split(\n      rest_X, rest_y,\n      test_size=0.3333, \n      shuffle=True, \n      random_state=seed\n    )\n    return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset\n</pre> from sklearn.model_selection import train_test_split   def prepare_dataset(model_type=\"classification\", dataset=None):      if model_type == \"classification\":         if dataset is None:             dataset = np.random.choice(list(classification.keys()))          _data = classification[dataset]()     else:         if dataset is None:             dataset = np.random.choice(list(regression.keys()))         _data = regression[dataset]()      train_X, rest_X, train_y, rest_y = train_test_split(       _data.get(\"data\"),        _data.get(\"target\"),        train_size=0.7,        shuffle=True,        random_state=seed     )          # 10% test and 20% validation data     valid_X, test_X, valid_y, test_y = train_test_split(       rest_X, rest_y,       test_size=0.3333,        shuffle=True,        random_state=seed     )     return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset <pre>\n---------------------------------------------------------------------------\nModuleNotFoundError                       Traceback (most recent call last)\nCell In[7], line 1\n----&gt; 1 from sklearn.model_selection import train_test_split\n      4 def prepare_dataset(model_type=\"classification\", dataset=None):\n      6     if model_type == \"classification\":\n\nModuleNotFoundError: No module named 'sklearn'</pre> In\u00a0[8]: Copied! <pre>train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\\n    prepare_dataset(model_type=\"classification\")\n\nprint(dataset)\nprint(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(\n    train_X.shape, valid_X.shape, test_X.shape\n))\n</pre> train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\     prepare_dataset(model_type=\"classification\")  print(dataset) print(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(     train_X.shape, valid_X.shape, test_X.shape )) <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[8], line 2\n      1 train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\\n----&gt; 2     prepare_dataset(model_type=\"classification\")\n      4 print(dataset)\n      5 print(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(\n      6     train_X.shape, valid_X.shape, test_X.shape\n      7 ))\n\nNameError: name 'prepare_dataset' is not defined</pre> In\u00a0[9]: Copied! <pre>from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, make_scorer\n\n\naccuracy_scorer = make_scorer(accuracy_score)\n</pre> from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score, make_scorer   accuracy_scorer = make_scorer(accuracy_score) <pre>\n---------------------------------------------------------------------------\nModuleNotFoundError                       Traceback (most recent call last)\nCell In[9], line 1\n----&gt; 1 from sklearn.ensemble import RandomForestClassifier\n      2 from sklearn.metrics import accuracy_score, make_scorer\n      5 accuracy_scorer = make_scorer(accuracy_score)\n\nModuleNotFoundError: No module named 'sklearn'</pre> In\u00a0[10]: Copied! <pre>def target_function(config, fidelity, **kwargs):\n    # Extracting support information\n    seed = kwargs[\"seed\"]\n    train_X = kwargs[\"train_X\"]\n    train_y = kwargs[\"train_y\"]\n    valid_X = kwargs[\"valid_X\"]\n    valid_y = kwargs[\"valid_y\"]\n    max_fidelity = kwargs[\"max_fidelity\"]\n    \n    if fidelity is None:\n        fidelity = max_fidelity\n    \n    start = time.time()\n    # Building model \n    model = RandomForestClassifier(\n        **config.get_dictionary(),\n        n_estimators=int(fidelity),\n        bootstrap=True,\n        random_state=seed,\n    )\n    # Training the model on the complete training set\n    model.fit(train_X, train_y)\n    \n    # Evaluating the model on the validation set\n    valid_accuracy = accuracy_scorer(model, valid_X, valid_y)\n    cost = time.time() - start\n    \n    # Evaluating the model on the test set as additional info\n    test_accuracy = accuracy_scorer(model, test_X, test_y)\n    \n    result = {\n        \"fitness\": -valid_accuracy,  # DE/DEHB minimizes\n        \"cost\": cost,\n        \"info\": {\n            \"test_score\": test_accuracy,\n            \"fidelity\": fidelity\n        }\n    }\n    return result\n</pre> def target_function(config, fidelity, **kwargs):     # Extracting support information     seed = kwargs[\"seed\"]     train_X = kwargs[\"train_X\"]     train_y = kwargs[\"train_y\"]     valid_X = kwargs[\"valid_X\"]     valid_y = kwargs[\"valid_y\"]     max_fidelity = kwargs[\"max_fidelity\"]          if fidelity is None:         fidelity = max_fidelity          start = time.time()     # Building model      model = RandomForestClassifier(         **config.get_dictionary(),         n_estimators=int(fidelity),         bootstrap=True,         random_state=seed,     )     # Training the model on the complete training set     model.fit(train_X, train_y)          # Evaluating the model on the validation set     valid_accuracy = accuracy_scorer(model, valid_X, valid_y)     cost = time.time() - start          # Evaluating the model on the test set as additional info     test_accuracy = accuracy_scorer(model, test_X, test_y)          result = {         \"fitness\": -valid_accuracy,  # DE/DEHB minimizes         \"cost\": cost,         \"info\": {             \"test_score\": test_accuracy,             \"fidelity\": fidelity         }     }     return result <p>We now have all components to define the problem to be optimized. DEHB can be initialized using all these information.</p> In\u00a0[11]: Copied! <pre>from dehb import DEHB\n\ndehb = DEHB(\n    f=target_function, \n    cs=cs, \n    dimensions=dimensions, \n    min_fidelity=min_fidelity, \n    max_fidelity=max_fidelity,\n    n_workers=1,\n    output_path=\"./temp\"\n)\n</pre> from dehb import DEHB  dehb = DEHB(     f=target_function,      cs=cs,      dimensions=dimensions,      min_fidelity=min_fidelity,      max_fidelity=max_fidelity,     n_workers=1,     output_path=\"./temp\" ) In\u00a0[12]: Copied! <pre>trajectory, runtime, history = dehb.run(\n    total_cost=10,\n    verbose=False,\n    save_intermediate=False,\n    # parameters expected as **kwargs in target_function is passed here\n    seed=123,\n    train_X=train_X,\n    train_y=train_y,\n    valid_X=valid_X,\n    valid_y=valid_y,\n    max_fidelity=dehb.max_fidelity\n)\n</pre> trajectory, runtime, history = dehb.run(     total_cost=10,     verbose=False,     save_intermediate=False,     # parameters expected as **kwargs in target_function is passed here     seed=123,     train_X=train_X,     train_y=train_y,     valid_X=valid_X,     valid_y=valid_y,     max_fidelity=dehb.max_fidelity ) <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[12], line 7\n      1 trajectory, runtime, history = dehb.run(\n      2     total_cost=10,\n      3     verbose=False,\n      4     save_intermediate=False,\n      5     # parameters expected as **kwargs in target_function is passed here\n      6     seed=123,\n----&gt; 7     train_X=train_X,\n      8     train_y=train_y,\n      9     valid_X=valid_X,\n     10     valid_y=valid_y,\n     11     max_fidelity=dehb.max_fidelity\n     12 )\n\nNameError: name 'train_X' is not defined</pre> In\u00a0[13]: Copied! <pre>print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")\n\n# Last recorded function evaluation\nlast_eval = history[-1]\nconfig, score, cost, fidelity, _info = last_eval\n\nprint(\"Last evaluated configuration, \")\nprint(dehb.vector_to_configspace(config), end=\"\")\nprint(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"\n      \"took {:.3f} seconds to run.\".format(score, fidelity, cost))\nprint(\"The additional info attached: {}\".format(_info))\n</pre> print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")  # Last recorded function evaluation last_eval = history[-1] config, score, cost, fidelity, _info = last_eval  print(\"Last evaluated configuration, \") print(dehb.vector_to_configspace(config), end=\"\") print(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"       \"took {:.3f} seconds to run.\".format(score, fidelity, cost)) print(\"The additional info attached: {}\".format(_info)) <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[13], line 1\n----&gt; 1 print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")\n      3 # Last recorded function evaluation\n      4 last_eval = history[-1]\n\nNameError: name 'trajectory' is not defined</pre> <p>Below, we let DEHB optimize for $5$ different runs. The <code>reset()</code> allows DEHB to begin optimization from the beginning by cleaning all history and starting with random samples. Each run of DEHB optimization is for just $10$ seconds as set by <code>total_cost=10</code>. We then report the mean and the standard deviation of the best score seen across these $5$ runs.</p> In\u00a0[14]: Copied! <pre>runs = 5\n\nbest_config_list = []\n\nfor i in range(runs):\n    # Resetting to begin optimization again\n    dehb.reset()\n    # Executing a run of DEHB optimization lasting for 10s\n    trajectory, runtime, history = dehb.run(\n        total_cost=10,\n        verbose=False,\n        save_intermediate=False,\n        seed=123,\n        train_X=train_X,\n        train_y=train_y,\n        valid_X=valid_X,\n        valid_y=valid_y,\n        max_fidelity=dehb.max_fidelity\n    )\n    best_config = dehb.vector_to_configspace(dehb.inc_config)\n    \n    # Creating a model using the best configuration found\n    model = RandomForestClassifier(\n        **best_config.get_dictionary(),\n        n_estimators=int(max_fidelity),\n        bootstrap=True,\n        random_state=seed,\n    )\n    # Training the model on the complete training set\n    model.fit(\n        np.concatenate((train_X, valid_X)), \n        np.concatenate((train_y, valid_y))\n    )\n    # Evaluating the model on the held-out test set\n    test_accuracy = accuracy_scorer(model, test_X, test_y)\n    best_config_list.append((best_config, test_accuracy))\n</pre> runs = 5  best_config_list = []  for i in range(runs):     # Resetting to begin optimization again     dehb.reset()     # Executing a run of DEHB optimization lasting for 10s     trajectory, runtime, history = dehb.run(         total_cost=10,         verbose=False,         save_intermediate=False,         seed=123,         train_X=train_X,         train_y=train_y,         valid_X=valid_X,         valid_y=valid_y,         max_fidelity=dehb.max_fidelity     )     best_config = dehb.vector_to_configspace(dehb.inc_config)          # Creating a model using the best configuration found     model = RandomForestClassifier(         **best_config.get_dictionary(),         n_estimators=int(max_fidelity),         bootstrap=True,         random_state=seed,     )     # Training the model on the complete training set     model.fit(         np.concatenate((train_X, valid_X)),          np.concatenate((train_y, valid_y))     )     # Evaluating the model on the held-out test set     test_accuracy = accuracy_scorer(model, test_X, test_y)     best_config_list.append((best_config, test_accuracy)) <pre>2024-02-17 14:48:33.183 | INFO     | dehb.optimizers.dehb:reset:121 - \n\nRESET at 02/17/24 14:48:33 UTC\n\n\n</pre> <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[14], line 14\n      7 dehb.reset()\n      8 # Executing a run of DEHB optimization lasting for 10s\n      9 trajectory, runtime, history = dehb.run(\n     10     total_cost=10,\n     11     verbose=False,\n     12     save_intermediate=False,\n     13     seed=123,\n---&gt; 14     train_X=train_X,\n     15     train_y=train_y,\n     16     valid_X=valid_X,\n     17     valid_y=valid_y,\n     18     max_fidelity=dehb.max_fidelity\n     19 )\n     20 best_config = dehb.vector_to_configspace(dehb.inc_config)\n     22 # Creating a model using the best configuration found\n\nNameError: name 'train_X' is not defined</pre> In\u00a0[15]: Copied! <pre>print(\"Mean score across trials: \", np.mean([score for _, score in best_config_list]))\nprint(\"Std. dev. of score across trials: \", np.std([score for _, score in best_config_list]))\n</pre> print(\"Mean score across trials: \", np.mean([score for _, score in best_config_list])) print(\"Std. dev. of score across trials: \", np.std([score for _, score in best_config_list])) <pre>Mean score across trials:  nan\nStd. dev. of score across trials:  nan\n</pre> In\u00a0[16]: Copied! <pre>for config, score in best_config_list:\n    print(\"{} got an accuracy of {} on the test set.\".format(config, score))\n    print()\n</pre> for config, score in best_config_list:     print(\"{} got an accuracy of {} on the test set.\".format(config, score))     print() In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#optimizing-randomforest-using-dehb","title":"Optimizing RandomForest using DEHB\u00b6","text":"<p>This notebook aims to build on the template from <code>00_interfacing_DEHB</code> and use it on an actual problem, to optimize the hyperparameters of a Random Forest model, for a dataset.</p> <p>Additional requirements:</p> <ul> <li>scikit-learn&gt;=0.24</li> </ul>"},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#defining-fidelity-range","title":"Defining fidelity range\u00b6","text":""},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#creating-search-space","title":"Creating search space\u00b6","text":""},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#creating-target-function-to-optimize-2-parts","title":"Creating target function to optimize (2 parts)\u00b6","text":""},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#1-preparing-dataset-and-splits","title":"1 ) Preparing dataset and splits\u00b6","text":""},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#2-function-interface-with-dehb","title":"2 ) Function interface with DEHB\u00b6","text":""},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#running-dehb","title":"Running DEHB\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/","title":"Using the Ask &amp; Tell interface","text":"In\u00a0[1]: Copied! <pre>import time\nimport numpy as np\nimport warnings\n\n\nseed = 123\nnp.random.seed(seed)\nwarnings.filterwarnings('ignore')\n</pre> import time import numpy as np import warnings   seed = 123 np.random.seed(seed) warnings.filterwarnings('ignore') <p>The problem defined here is to optimize a Random Forest model, on any given dataset, using DEHB. The hyperparameters chosen to be optimized are:</p> <ul> <li><code>max_depth</code></li> <li><code>min_samples_split</code></li> <li><code>max_features</code></li> <li><code>min_samples_leaf</code> while the <code>n_estimators</code> hyperparameter to the Random Forest is chosen to be a fidelity parameter instead. Lesser number of trees ($&lt;10$) in the Random Forest may not allow adequate ensembling for the grouped prediction to be significantly better than the individual tree predictions. Whereas a large number of trees (~$100$) often give accurate predictions but is naturally slower to train and predict on account of more trees to train. Therefore, a smaller <code>n_estimators</code> can be used as a cheaper approximation of the actual fidelity of <code>n_estimators=100</code>.</li> </ul> In\u00a0[2]: Copied! <pre>min_fidelity, max_fidelity = 2, 50\n</pre> min_fidelity, max_fidelity = 2, 50 <p>For the remaining $4$ hyperparameters, the search space can be created as a <code>ConfigSpace</code> object, with the domain of individual parameters defined.</p> In\u00a0[3]: Copied! <pre>import ConfigSpace as CS\n\n\ndef create_search_space(seed=123):\n    \"\"\"Parameter space to be optimized --- contains the hyperparameters\n    \"\"\"\n    cs = CS.ConfigurationSpace(seed=seed)\n\n    cs.add_hyperparameters([\n        CS.UniformIntegerHyperparameter(\n            'max_depth', lower=1, upper=15, default_value=2, log=False\n        ),\n        CS.UniformIntegerHyperparameter(\n            'min_samples_split', lower=2, upper=128, default_value=2, log=True\n        ),\n        CS.UniformFloatHyperparameter(\n            'max_features', lower=0.1, upper=0.9, default_value=0.5, log=False\n        ),\n        CS.UniformIntegerHyperparameter(\n            'min_samples_leaf', lower=1, upper=64, default_value=1, log=True\n        ),\n    ])\n    return cs\n</pre> import ConfigSpace as CS   def create_search_space(seed=123):     \"\"\"Parameter space to be optimized --- contains the hyperparameters     \"\"\"     cs = CS.ConfigurationSpace(seed=seed)      cs.add_hyperparameters([         CS.UniformIntegerHyperparameter(             'max_depth', lower=1, upper=15, default_value=2, log=False         ),         CS.UniformIntegerHyperparameter(             'min_samples_split', lower=2, upper=128, default_value=2, log=True         ),         CS.UniformFloatHyperparameter(             'max_features', lower=0.1, upper=0.9, default_value=0.5, log=False         ),         CS.UniformIntegerHyperparameter(             'min_samples_leaf', lower=1, upper=64, default_value=1, log=True         ),     ])     return cs In\u00a0[4]: Copied! <pre>cs = create_search_space(seed)\nprint(cs)\n</pre> cs = create_search_space(seed) print(cs) <pre>Configuration space object:\n  Hyperparameters:\n    max_depth, Type: UniformInteger, Range: [1, 15], Default: 2\n    max_features, Type: UniformFloat, Range: [0.1, 0.9], Default: 0.5\n    min_samples_leaf, Type: UniformInteger, Range: [1, 64], Default: 1, on log-scale\n    min_samples_split, Type: UniformInteger, Range: [2, 128], Default: 2, on log-scale\n\n</pre> In\u00a0[5]: Copied! <pre>dimensions = len(cs.get_hyperparameters())\nprint(\"Dimensionality of search space: {}\".format(dimensions))\n</pre> dimensions = len(cs.get_hyperparameters()) print(\"Dimensionality of search space: {}\".format(dimensions)) <pre>Dimensionality of search space: 4\n</pre> <p>Now the primary black/gray-box interface to the Random Forest model needs to be built for DEHB to query. As given in the <code>00_interfacing_DEHB</code> notebook, this function will have a signature akin to: <code>target_function(config, fidelity)</code>, and return a two-element tuple of the <code>score</code> and <code>cost</code>. It must be noted that DEHB minimizes and therefore the <code>score</code> being returned by this <code>target_function</code> should account for it.</p> <p>In this example, the target function trains a Random Forest model on a dataset. We load a dataset here and maintain a fixed, train-validation-test split for one complete run. Multiple DEHB runs can therefore optimize on the same validation split, and evaluate final performance on the same test set.</p> In\u00a0[6]: Copied! <pre>from sklearn.datasets import load_iris, load_digits, load_wine\n\n\nclassification = {\"iris\": load_iris, \"digits\": load_digits, \"wine\": load_wine}\n</pre> from sklearn.datasets import load_iris, load_digits, load_wine   classification = {\"iris\": load_iris, \"digits\": load_digits, \"wine\": load_wine} <pre>\n---------------------------------------------------------------------------\nModuleNotFoundError                       Traceback (most recent call last)\nCell In[6], line 1\n----&gt; 1 from sklearn.datasets import load_iris, load_digits, load_wine\n      4 classification = {\"iris\": load_iris, \"digits\": load_digits, \"wine\": load_wine}\n\nModuleNotFoundError: No module named 'sklearn'</pre> In\u00a0[7]: Copied! <pre>from sklearn.model_selection import train_test_split\n\n\ndef prepare_dataset(model_type=\"classification\", dataset=None):\n\n    if model_type == \"classification\":\n        if dataset is None:\n            dataset = np.random.choice(list(classification.keys())) \n        _data = classification[dataset]()\n    else:\n        if dataset is None:\n            dataset = np.random.choice(list(regression.keys()))\n        _data = regression[dataset]()\n\n    train_X, rest_X, train_y, rest_y = train_test_split(\n      _data.get(\"data\"), \n      _data.get(\"target\"), \n      train_size=0.7, \n      shuffle=True, \n      random_state=seed\n    )\n    \n    # 10% test and 20% validation data\n    valid_X, test_X, valid_y, test_y = train_test_split(\n      rest_X, rest_y,\n      test_size=0.3333, \n      shuffle=True, \n      random_state=seed\n    )\n    return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset\n</pre> from sklearn.model_selection import train_test_split   def prepare_dataset(model_type=\"classification\", dataset=None):      if model_type == \"classification\":         if dataset is None:             dataset = np.random.choice(list(classification.keys()))          _data = classification[dataset]()     else:         if dataset is None:             dataset = np.random.choice(list(regression.keys()))         _data = regression[dataset]()      train_X, rest_X, train_y, rest_y = train_test_split(       _data.get(\"data\"),        _data.get(\"target\"),        train_size=0.7,        shuffle=True,        random_state=seed     )          # 10% test and 20% validation data     valid_X, test_X, valid_y, test_y = train_test_split(       rest_X, rest_y,       test_size=0.3333,        shuffle=True,        random_state=seed     )     return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset <pre>\n---------------------------------------------------------------------------\nModuleNotFoundError                       Traceback (most recent call last)\nCell In[7], line 1\n----&gt; 1 from sklearn.model_selection import train_test_split\n      4 def prepare_dataset(model_type=\"classification\", dataset=None):\n      6     if model_type == \"classification\":\n\nModuleNotFoundError: No module named 'sklearn'</pre> In\u00a0[8]: Copied! <pre>train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\\n    prepare_dataset(model_type=\"classification\")\n\nprint(dataset)\nprint(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(\n    train_X.shape, valid_X.shape, test_X.shape\n))\n</pre> train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\     prepare_dataset(model_type=\"classification\")  print(dataset) print(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(     train_X.shape, valid_X.shape, test_X.shape )) <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[8], line 2\n      1 train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\\n----&gt; 2     prepare_dataset(model_type=\"classification\")\n      4 print(dataset)\n      5 print(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(\n      6     train_X.shape, valid_X.shape, test_X.shape\n      7 ))\n\nNameError: name 'prepare_dataset' is not defined</pre> In\u00a0[9]: Copied! <pre>from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, make_scorer\n\n\naccuracy_scorer = make_scorer(accuracy_score)\n</pre> from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score, make_scorer   accuracy_scorer = make_scorer(accuracy_score) <pre>\n---------------------------------------------------------------------------\nModuleNotFoundError                       Traceback (most recent call last)\nCell In[9], line 1\n----&gt; 1 from sklearn.ensemble import RandomForestClassifier\n      2 from sklearn.metrics import accuracy_score, make_scorer\n      5 accuracy_scorer = make_scorer(accuracy_score)\n\nModuleNotFoundError: No module named 'sklearn'</pre> In\u00a0[10]: Copied! <pre>def target_function(config, fidelity, **kwargs):\n    # Extracting support information\n    seed = kwargs[\"seed\"]\n    train_X = kwargs[\"train_X\"]\n    train_y = kwargs[\"train_y\"]\n    valid_X = kwargs[\"valid_X\"]\n    valid_y = kwargs[\"valid_y\"]\n    max_fidelity = kwargs[\"max_fidelity\"]\n    \n    if fidelity is None:\n        fidelity = max_fidelity\n    \n    start = time.time()\n    # Building model \n    model = RandomForestClassifier(\n        **config.get_dictionary(),\n        n_estimators=int(fidelity),\n        bootstrap=True,\n        random_state=seed,\n    )\n    # Training the model on the complete training set\n    model.fit(train_X, train_y)\n    \n    # Evaluating the model on the validation set\n    valid_accuracy = accuracy_scorer(model, valid_X, valid_y)\n    cost = time.time() - start\n    \n    # Evaluating the model on the test set as additional info\n    test_accuracy = accuracy_scorer(model, test_X, test_y)\n    \n    result = {\n        \"fitness\": -valid_accuracy,  # DE/DEHB minimizes\n        \"cost\": cost,\n        \"info\": {\n            \"test_score\": test_accuracy,\n            \"fidelity\": fidelity\n        }\n    }\n    return result\n</pre> def target_function(config, fidelity, **kwargs):     # Extracting support information     seed = kwargs[\"seed\"]     train_X = kwargs[\"train_X\"]     train_y = kwargs[\"train_y\"]     valid_X = kwargs[\"valid_X\"]     valid_y = kwargs[\"valid_y\"]     max_fidelity = kwargs[\"max_fidelity\"]          if fidelity is None:         fidelity = max_fidelity          start = time.time()     # Building model      model = RandomForestClassifier(         **config.get_dictionary(),         n_estimators=int(fidelity),         bootstrap=True,         random_state=seed,     )     # Training the model on the complete training set     model.fit(train_X, train_y)          # Evaluating the model on the validation set     valid_accuracy = accuracy_scorer(model, valid_X, valid_y)     cost = time.time() - start          # Evaluating the model on the test set as additional info     test_accuracy = accuracy_scorer(model, test_X, test_y)          result = {         \"fitness\": -valid_accuracy,  # DE/DEHB minimizes         \"cost\": cost,         \"info\": {             \"test_score\": test_accuracy,             \"fidelity\": fidelity         }     }     return result <p>We now have all components to define the problem to be optimized. DEHB can be initialized using all these information.</p> In\u00a0[11]: Copied! <pre>from dehb import DEHB\n\ndehb = DEHB(\n    f=target_function, # Here we do not need to necessarily specify the target function, but it can still be useful to call 'run' later.\n    cs=cs, \n    dimensions=dimensions, \n    min_fidelity=min_fidelity, \n    max_fidelity=max_fidelity,\n    n_workers=1,\n    output_path=\"./temp\"\n)\n</pre> from dehb import DEHB  dehb = DEHB(     f=target_function, # Here we do not need to necessarily specify the target function, but it can still be useful to call 'run' later.     cs=cs,      dimensions=dimensions,      min_fidelity=min_fidelity,      max_fidelity=max_fidelity,     n_workers=1,     output_path=\"./temp\" ) In\u00a0[12]: Copied! <pre>n_function_evals = 50\n\nfor _ in range(n_function_evals):\n    # Ask for the job_info, including the configuration to run and the fidelity\n    job_info = dehb.ask()\n\n    # Evaluate the configuration on the given fidelity. Here you are free to use\n    # any technique to compute the result. This job could e.g. be forwarded to\n    # a worker on your cluster (Which is not required to use Dask).\n    # The results dict has to contain the keys \"cost\" and \"fitness\" with an additional \"info\"\n    # dict for additional, user-specific data.\n    res = target_function(job_info[\"config\"], job_info[\"fidelity\"],\n                          # parameters as **kwargs in target_function\n                          seed=123,\n                          train_X=train_X,\n                          train_y=train_y,\n                          valid_X=valid_X,\n                          valid_y=valid_y,\n                          max_fidelity=dehb.max_fidelity)\n    \n    # When the evaluation is done, report the results back to the DEHB controller.\n    dehb.tell(job_info, res)\n\ntrajectory = dehb.traj\nruntime = dehb.runtime\nhistory = dehb.history\n</pre> n_function_evals = 50  for _ in range(n_function_evals):     # Ask for the job_info, including the configuration to run and the fidelity     job_info = dehb.ask()      # Evaluate the configuration on the given fidelity. Here you are free to use     # any technique to compute the result. This job could e.g. be forwarded to     # a worker on your cluster (Which is not required to use Dask).     # The results dict has to contain the keys \"cost\" and \"fitness\" with an additional \"info\"     # dict for additional, user-specific data.     res = target_function(job_info[\"config\"], job_info[\"fidelity\"],                           # parameters as **kwargs in target_function                           seed=123,                           train_X=train_X,                           train_y=train_y,                           valid_X=valid_X,                           valid_y=valid_y,                           max_fidelity=dehb.max_fidelity)          # When the evaluation is done, report the results back to the DEHB controller.     dehb.tell(job_info, res)  trajectory = dehb.traj runtime = dehb.runtime history = dehb.history <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[12], line 15\n      5 job_info = dehb.ask()\n      7 # Evaluate the configuration on the given fidelity. Here you are free to use\n      8 # any technique to compute the result. This job could e.g. be forwarded to\n      9 # a worker on your cluster (Which is not required to use Dask).\n     10 # The results dict has to contain the keys \"cost\" and \"fitness\" with an additional \"info\"\n     11 # dict for additional, user-specific data.\n     12 res = target_function(job_info[\"config\"], job_info[\"fidelity\"],\n     13                       # parameters as **kwargs in target_function\n     14                       seed=123,\n---&gt; 15                       train_X=train_X,\n     16                       train_y=train_y,\n     17                       valid_X=valid_X,\n     18                       valid_y=valid_y,\n     19                       max_fidelity=dehb.max_fidelity)\n     21 # When the evaluation is done, report the results back to the DEHB controller.\n     22 dehb.tell(job_info, res)\n\nNameError: name 'train_X' is not defined</pre> In\u00a0[13]: Copied! <pre>print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")\n\n# Last recorded function evaluation\nlast_eval = history[-1]\nconfig, score, cost, fidelity, _info = last_eval\n\nprint(\"Last evaluated configuration, \")\nprint(dehb.vector_to_configspace(config), end=\"\")\nprint(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"\n      \"took {:.3f} seconds to run.\".format(score, fidelity, cost))\nprint(\"The additional info attached: {}\".format(_info))\n\nprint()\nprint(\"Best evaluated configuration, \")\n\nbest_config = dehb.vector_to_configspace(dehb.inc_config)\n\n# Creating a model using the best configuration found\nmodel = RandomForestClassifier(\n      **best_config.get_dictionary(),\n      n_estimators=int(max_fidelity),\n      bootstrap=True,\n      random_state=seed,\n)\n# Training the model on the complete training set\nmodel.fit(\n      np.concatenate((train_X, valid_X)), \n      np.concatenate((train_y, valid_y))\n)\n# Evaluating the model on the held-out test set\ntest_accuracy = accuracy_scorer(model, test_X, test_y)\n\nprint(f\"{best_config} got an accuracy of {test_accuracy} on the test set.\")\n</pre> print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")  # Last recorded function evaluation last_eval = history[-1] config, score, cost, fidelity, _info = last_eval  print(\"Last evaluated configuration, \") print(dehb.vector_to_configspace(config), end=\"\") print(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"       \"took {:.3f} seconds to run.\".format(score, fidelity, cost)) print(\"The additional info attached: {}\".format(_info))  print() print(\"Best evaluated configuration, \")  best_config = dehb.vector_to_configspace(dehb.inc_config)  # Creating a model using the best configuration found model = RandomForestClassifier(       **best_config.get_dictionary(),       n_estimators=int(max_fidelity),       bootstrap=True,       random_state=seed, ) # Training the model on the complete training set model.fit(       np.concatenate((train_X, valid_X)),        np.concatenate((train_y, valid_y)) ) # Evaluating the model on the held-out test set test_accuracy = accuracy_scorer(model, test_X, test_y)  print(f\"{best_config} got an accuracy of {test_accuracy} on the test set.\") <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[13], line 1\n----&gt; 1 print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")\n      3 # Last recorded function evaluation\n      4 last_eval = history[-1]\n\nNameError: name 'trajectory' is not defined</pre> <p>After running DEHB for 50 function evaluations using the ask and tell interface, we can still call the <code>run</code> function in order keep optimizing without specifically using ask and tell.</p> In\u00a0[14]: Copied! <pre># Continuing the ask/tell run of DEHB optimization for another 10s\ntrajectory, runtime, history = dehb.run(\n    total_cost=10,\n    verbose=False,\n    save_intermediate=False,\n    seed=123,\n    train_X=train_X,\n    train_y=train_y,\n    valid_X=valid_X,\n    valid_y=valid_y,\n    max_fidelity=dehb.max_fidelity\n)\nbest_config = dehb.vector_to_configspace(dehb.inc_config)\n\n# Creating a model using the best configuration found\nmodel = RandomForestClassifier(\n    **best_config.get_dictionary(),\n    n_estimators=int(max_fidelity),\n    bootstrap=True,\n    random_state=seed,\n)\n# Training the model on the complete training set\nmodel.fit(\n    np.concatenate((train_X, valid_X)), \n    np.concatenate((train_y, valid_y))\n)\n# Evaluating the model on the held-out test set\ntest_accuracy = accuracy_scorer(model, test_X, test_y)\n\n\nprint(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")\n\n# Last recorded function evaluation\nlast_eval = history[-1]\nconfig, score, cost, fidelity, _info = last_eval\n\nprint(\"Last evaluated configuration, \")\nprint(dehb.vector_to_configspace(config), end=\"\")\nprint(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"\n      \"took {:.3f} seconds to run.\".format(score, fidelity, cost))\nprint(\"The additional info attached: {}\".format(_info))\n\nprint()\nprint(\"Best evaluated configuration, \")\nprint(f\"{best_config} got an accuracy of {test_accuracy} on the test set.\")\n</pre> # Continuing the ask/tell run of DEHB optimization for another 10s trajectory, runtime, history = dehb.run(     total_cost=10,     verbose=False,     save_intermediate=False,     seed=123,     train_X=train_X,     train_y=train_y,     valid_X=valid_X,     valid_y=valid_y,     max_fidelity=dehb.max_fidelity ) best_config = dehb.vector_to_configspace(dehb.inc_config)  # Creating a model using the best configuration found model = RandomForestClassifier(     **best_config.get_dictionary(),     n_estimators=int(max_fidelity),     bootstrap=True,     random_state=seed, ) # Training the model on the complete training set model.fit(     np.concatenate((train_X, valid_X)),      np.concatenate((train_y, valid_y)) ) # Evaluating the model on the held-out test set test_accuracy = accuracy_scorer(model, test_X, test_y)   print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")  # Last recorded function evaluation last_eval = history[-1] config, score, cost, fidelity, _info = last_eval  print(\"Last evaluated configuration, \") print(dehb.vector_to_configspace(config), end=\"\") print(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"       \"took {:.3f} seconds to run.\".format(score, fidelity, cost)) print(\"The additional info attached: {}\".format(_info))  print() print(\"Best evaluated configuration, \") print(f\"{best_config} got an accuracy of {test_accuracy} on the test set.\") <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[14], line 7\n      1 # Continuing the ask/tell run of DEHB optimization for another 10s\n      2 trajectory, runtime, history = dehb.run(\n      3     total_cost=10,\n      4     verbose=False,\n      5     save_intermediate=False,\n      6     seed=123,\n----&gt; 7     train_X=train_X,\n      8     train_y=train_y,\n      9     valid_X=valid_X,\n     10     valid_y=valid_y,\n     11     max_fidelity=dehb.max_fidelity\n     12 )\n     13 best_config = dehb.vector_to_configspace(dehb.inc_config)\n     15 # Creating a model using the best configuration found\n\nNameError: name 'train_X' is not defined</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#using-the-ask-tell-interface","title":"Using the Ask &amp; Tell interface\u00b6","text":"<p>This notebook aims to build on the template from <code>00_interfacing_DEHB</code> and use it on an actual problem, to optimize the hyperparameters of a Random Forest model, for a dataset. Here we use DEHB with the built-in ask and tell functionality.</p> <p>Additional requirements:</p> <ul> <li>scikit-learn&gt;=0.24</li> </ul>"},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#defining-fidelity-range","title":"Defining fidelity range\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#creating-search-space","title":"Creating search space\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#creating-target-function-to-optimize-2-parts","title":"Creating target function to optimize (2 parts)\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#1-preparing-dataset-and-splits","title":"1 ) Preparing dataset and splits\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#2-function-interface-with-dehb","title":"2 ) Function interface with DEHB\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#running-dehb","title":"Running DEHB\u00b6","text":""},{"location":"examples/02_using_DEHB_without_ConfigSpace/","title":"Using DEHB without ConfigSpace","text":"In\u00a0[1]: Copied! <pre>import time\nimport numpy as np\nimport warnings\n\n\nseed = 123\nnp.random.seed(seed)\nwarnings.filterwarnings('ignore')\n</pre> import time import numpy as np import warnings   seed = 123 np.random.seed(seed) warnings.filterwarnings('ignore') <p>The hyperparameters chosen, along with their type, and ranges:</p> <ul> <li><code>max_depth</code> $-$ integer $-$ [1, 15]</li> <li><code>min_samples_split</code> $-$ integer $-$ [2, 128] $-$ log-spacing</li> <li><code>max_features</code> $-$ float $-$ [0.1, 0.9]</li> <li><code>min_samples_leaf</code> $-$ integer $-$ [1, 64] $-$ log-spacing</li> </ul> <p>DE, and therefore DEHB, work in the unit hypercube space. The random individuals sampled at the beginning of DEHB, performs a uniform random sampling in the [0, 1] range for each parameter/dimension. Hence, each configuration suggested by DEHB also is in the [0, 1] range. The <code>vector_to_configspace</code> included in the DEHB source code, can reliably handle the transformation of the [0, 1] space of DEHB configurations to the original parameter space required. In the absence of ConfigSpace usage, such a conversion needs to included as part of the objective/target function being passed.</p> In\u00a0[2]: Copied! <pre># Declaring the search space\nparam_space = {\n    \"max_depth\": [1, 15, int, False],\n    \"min_samples_split\": [2, 128, int, True],\n    \"max_features\": [0.1, 0.9, float, False],\n    \"min_samples_leaf\": [1, 64, int, True],\n}\ndimensions = len(param_space)\n\n# Declaring the fidelity range\nmin_fidelity, max_fidelity = 2, 50\n\n\ndef transform_space(param_space, configuration):\n    \"\"\" Scales the [0, 1]-ranged parameter linearly to [lower, upper]\n    \n    Parameters\n    ----------\n    param_space : a dict containing the parameters and their meta-info\n    configuration : a vector with each dimension in [0, 1] (from DEHB)\n    \n    Results\n    -------\n    a dict which can be passed to the model as named hyperparameters\n    \"\"\"\n    assert len(configuration) == len(param_space)\n    config_dict = dict()\n    for i, (k, v) in enumerate(param_space.items()):\n        value = configuration[i]\n        lower, upper = v[0], v[1]\n        is_log = v[3]\n        if is_log:\n            # performs linear scaling in the log-space\n            log_range = np.log(upper) - np.log(lower)\n            value = np.exp(np.log(lower) + log_range * value)\n        else:\n            # linear scaling within the range of the parameter\n            value = lower + (upper - lower) * value\n        if v[2] == int:\n            value = np.round(value).astype(int)\n        config_dict[k] = value\n    return config_dict\n</pre> # Declaring the search space param_space = {     \"max_depth\": [1, 15, int, False],     \"min_samples_split\": [2, 128, int, True],     \"max_features\": [0.1, 0.9, float, False],     \"min_samples_leaf\": [1, 64, int, True], } dimensions = len(param_space)  # Declaring the fidelity range min_fidelity, max_fidelity = 2, 50   def transform_space(param_space, configuration):     \"\"\" Scales the [0, 1]-ranged parameter linearly to [lower, upper]          Parameters     ----------     param_space : a dict containing the parameters and their meta-info     configuration : a vector with each dimension in [0, 1] (from DEHB)          Results     -------     a dict which can be passed to the model as named hyperparameters     \"\"\"     assert len(configuration) == len(param_space)     config_dict = dict()     for i, (k, v) in enumerate(param_space.items()):         value = configuration[i]         lower, upper = v[0], v[1]         is_log = v[3]         if is_log:             # performs linear scaling in the log-space             log_range = np.log(upper) - np.log(lower)             value = np.exp(np.log(lower) + log_range * value)         else:             # linear scaling within the range of the parameter             value = lower + (upper - lower) * value         if v[2] == int:             value = np.round(value).astype(int)         config_dict[k] = value     return config_dict     <p>NOTE: To handle categorical parameters would require custom representations for such cases. Categorical parameters don't have a lower or upper range but rather a possible list of discrete choices or values. Moreoever, categorical parameters can be string categories, boolean or even ordinal in nature.</p> <p>Given this <code>transform_space</code> function, everything else from <code>01_Optimizing_RandomForest_using_DEHB</code> can be largely reused. Only the <code>target_function</code> needs to be modified to include the <code>transform_space</code> function. Also, the <code>configspace</code> parameter needs to be set tp <code>False</code> while initializing DEHB.</p> In\u00a0[3]: Copied! <pre>from sklearn.datasets import load_digits, load_wine\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, make_scorer\n\n\naccuracy_scorer = make_scorer(accuracy_score)\n\n\ndef prepare_dataset(model_type=\"classification\"):\n\n    if model_type == \"classification\":\n        dataset = np.random.choice(list(classification.keys()))\n        _data = classification[dataset]()\n    else:\n        dataset = np.random.choice(list(regression.keys()))\n        _data = regression[dataset]()\n\n    train_X, test_X, train_y, test_y = train_test_split(\n        _data.get(\"data\"), \n        _data.get(\"target\"), \n        test_size=0.1, \n        shuffle=True, \n        random_state=seed\n    )\n    train_X, valid_X, train_y, valid_y = train_test_split(\n        _data.get(\"data\"), \n        _data.get(\"target\"), \n        test_size=0.3, \n        shuffle=True, \n        random_state=seed\n    )\n    return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset\n\n\ndef target_function(config, fidelity, **kwargs):\n    # Extracting support information\n    seed = kwargs[\"seed\"]\n    train_X = kwargs[\"train_X\"]\n    train_y = kwargs[\"train_y\"]\n    valid_X = kwargs[\"valid_X\"]\n    valid_y = kwargs[\"valid_y\"]\n    max_fidelity = kwargs[\"max_fidelity\"]\n    \n    # Mapping [0, 1]-vector to Sklearn parameters\n    param_space = kwargs[\"param_space\"]\n    config = transform_space(param_space, config)\n    \n    if fidelity is None:\n        fidelity = max_fidelity\n    \n    start = time.time()\n    # Building model \n    model = RandomForestClassifier(\n        **config,\n        n_estimators=int(fidelity),\n        bootstrap=True,\n        random_state=seed,\n    )\n    # Training the model on the complete training set\n    model.fit(train_X, train_y)\n    \n    # Evaluating the model on the validation set\n    valid_accuracy = accuracy_scorer(model, valid_X, valid_y)\n    cost = time.time() - start\n    \n    # Evaluating the model on the test set as additional info\n    test_accuracy = accuracy_scorer(model, test_X, test_y)\n    \n    result = {\n        \"fitness\": -valid_accuracy,  # DE/DEHB minimizes\n        \"cost\": cost,\n        \"info\": {\n            \"test_score\": test_accuracy,\n            \"fidelity\": fidelity\n        }\n    }\n    return result\n\n\nclassification = {\"digits\": load_digits, \"wine\": load_wine}\ntrain_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\\n    prepare_dataset(model_type=\"classification\")\n\nprint(dataset)\nprint(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(\n    train_X.shape, valid_X.shape, test_X.shape\n))\n</pre> from sklearn.datasets import load_digits, load_wine from sklearn.model_selection import train_test_split from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score, make_scorer   accuracy_scorer = make_scorer(accuracy_score)   def prepare_dataset(model_type=\"classification\"):      if model_type == \"classification\":         dataset = np.random.choice(list(classification.keys()))         _data = classification[dataset]()     else:         dataset = np.random.choice(list(regression.keys()))         _data = regression[dataset]()      train_X, test_X, train_y, test_y = train_test_split(         _data.get(\"data\"),          _data.get(\"target\"),          test_size=0.1,          shuffle=True,          random_state=seed     )     train_X, valid_X, train_y, valid_y = train_test_split(         _data.get(\"data\"),          _data.get(\"target\"),          test_size=0.3,          shuffle=True,          random_state=seed     )     return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset   def target_function(config, fidelity, **kwargs):     # Extracting support information     seed = kwargs[\"seed\"]     train_X = kwargs[\"train_X\"]     train_y = kwargs[\"train_y\"]     valid_X = kwargs[\"valid_X\"]     valid_y = kwargs[\"valid_y\"]     max_fidelity = kwargs[\"max_fidelity\"]          # Mapping [0, 1]-vector to Sklearn parameters     param_space = kwargs[\"param_space\"]     config = transform_space(param_space, config)          if fidelity is None:         fidelity = max_fidelity          start = time.time()     # Building model      model = RandomForestClassifier(         **config,         n_estimators=int(fidelity),         bootstrap=True,         random_state=seed,     )     # Training the model on the complete training set     model.fit(train_X, train_y)          # Evaluating the model on the validation set     valid_accuracy = accuracy_scorer(model, valid_X, valid_y)     cost = time.time() - start          # Evaluating the model on the test set as additional info     test_accuracy = accuracy_scorer(model, test_X, test_y)          result = {         \"fitness\": -valid_accuracy,  # DE/DEHB minimizes         \"cost\": cost,         \"info\": {             \"test_score\": test_accuracy,             \"fidelity\": fidelity         }     }     return result   classification = {\"digits\": load_digits, \"wine\": load_wine} train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\     prepare_dataset(model_type=\"classification\")  print(dataset) print(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(     train_X.shape, valid_X.shape, test_X.shape )) <pre>\n---------------------------------------------------------------------------\nModuleNotFoundError                       Traceback (most recent call last)\nCell In[3], line 1\n----&gt; 1 from sklearn.datasets import load_digits, load_wine\n      2 from sklearn.model_selection import train_test_split\n      3 from sklearn.ensemble import RandomForestClassifier\n\nModuleNotFoundError: No module named 'sklearn'</pre> In\u00a0[4]: Copied! <pre>from dehb import DEHB\n\n\ndehb = DEHB(\n    f=target_function, \n    dimensions=dimensions, \n    min_fidelity=min_fidelity, \n    max_fidelity=max_fidelity,\n    n_workers=1,\n    output_path=\"./temp\"\n)\n</pre> from dehb import DEHB   dehb = DEHB(     f=target_function,      dimensions=dimensions,      min_fidelity=min_fidelity,      max_fidelity=max_fidelity,     n_workers=1,     output_path=\"./temp\" ) <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[4], line 5\n      1 from dehb import DEHB\n      4 dehb = DEHB(\n----&gt; 5     f=target_function, \n      6     dimensions=dimensions, \n      7     min_fidelity=min_fidelity, \n      8     max_fidelity=max_fidelity,\n      9     n_workers=1,\n     10     output_path=\"./temp\"\n     11 )\n\nNameError: name 'target_function' is not defined</pre> In\u00a0[5]: Copied! <pre>trajectory, runtime, history = dehb.run(\n    total_cost=10, \n    verbose=False,\n    save_intermediate=False,\n    seed=123,\n    train_X=train_X,\n    train_y=train_y,\n    valid_X=valid_X,\n    valid_y=valid_y,\n    max_fidelity=dehb.max_fidelity,\n    param_space=param_space\n)\n</pre> trajectory, runtime, history = dehb.run(     total_cost=10,      verbose=False,     save_intermediate=False,     seed=123,     train_X=train_X,     train_y=train_y,     valid_X=valid_X,     valid_y=valid_y,     max_fidelity=dehb.max_fidelity,     param_space=param_space ) <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[5], line 1\n----&gt; 1 trajectory, runtime, history = dehb.run(\n      2     total_cost=10, \n      3     verbose=False,\n      4     save_intermediate=False,\n      5     seed=123,\n      6     train_X=train_X,\n      7     train_y=train_y,\n      8     valid_X=valid_X,\n      9     valid_y=valid_y,\n     10     max_fidelity=dehb.max_fidelity,\n     11     param_space=param_space\n     12 )\n\nNameError: name 'dehb' is not defined</pre> In\u00a0[6]: Copied! <pre>print(\"Incumbent score: {}\".format(dehb.inc_score))\nprint(\"Incumbent configuration:\\n{}\".format(transform_space(param_space, dehb.inc_config)))\n</pre> print(\"Incumbent score: {}\".format(dehb.inc_score)) print(\"Incumbent configuration:\\n{}\".format(transform_space(param_space, dehb.inc_config))) <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[6], line 1\n----&gt; 1 print(\"Incumbent score: {}\".format(dehb.inc_score))\n      2 print(\"Incumbent configuration:\\n{}\".format(transform_space(param_space, dehb.inc_config)))\n\nNameError: name 'dehb' is not defined</pre> In\u00a0[7]: Copied! <pre>model = RandomForestClassifier(\n    **transform_space(param_space, dehb.inc_config),\n    n_estimators=int(max_fidelity),\n    bootstrap=True,\n    random_state=seed,\n)\nmodel.fit(\n    np.concatenate((train_X, valid_X)),\n    np.concatenate((train_y, valid_y))\n)\ntest_accuracy = accuracy_scorer(model, test_X, test_y)\nprint(\"Test accuracy: {}\".format(test_accuracy))\n</pre> model = RandomForestClassifier(     **transform_space(param_space, dehb.inc_config),     n_estimators=int(max_fidelity),     bootstrap=True,     random_state=seed, ) model.fit(     np.concatenate((train_X, valid_X)),     np.concatenate((train_y, valid_y)) ) test_accuracy = accuracy_scorer(model, test_X, test_y) print(\"Test accuracy: {}\".format(test_accuracy)) <pre>\n---------------------------------------------------------------------------\nNameError                                 Traceback (most recent call last)\nCell In[7], line 1\n----&gt; 1 model = RandomForestClassifier(\n      2     **transform_space(param_space, dehb.inc_config),\n      3     n_estimators=int(max_fidelity),\n      4     bootstrap=True,\n      5     random_state=seed,\n      6 )\n      7 model.fit(\n      8     np.concatenate((train_X, valid_X)),\n      9     np.concatenate((train_y, valid_y))\n     10 )\n     11 test_accuracy = accuracy_scorer(model, test_X, test_y)\n\nNameError: name 'RandomForestClassifier' is not defined</pre> In\u00a0[8]: Copied! <pre>from matplotlib import pyplot as plt\n\nplt.plot(np.cumsum(runtime), np.array(trajectory) + 1)\nplt.xlabel(\"Wallclock time in seconds\")\nplt.ylabel(\"Regret (1 - accuracy)\");\n</pre> from matplotlib import pyplot as plt  plt.plot(np.cumsum(runtime), np.array(trajectory) + 1) plt.xlabel(\"Wallclock time in seconds\") plt.ylabel(\"Regret (1 - accuracy)\"); <pre>\n---------------------------------------------------------------------------\nModuleNotFoundError                       Traceback (most recent call last)\nCell In[8], line 1\n----&gt; 1 from matplotlib import pyplot as plt\n      3 plt.plot(np.cumsum(runtime), np.array(trajectory) + 1)\n      4 plt.xlabel(\"Wallclock time in seconds\")\n\nModuleNotFoundError: No module named 'matplotlib'</pre>"},{"location":"examples/02_using_DEHB_without_ConfigSpace/#using-dehb-without-configspace","title":"Using DEHB without ConfigSpace\u00b6","text":"<p>This notebook repeats the example from <code>01.1_Optimizing_RandomForest_using_DEHB</code> but without using <code>ConfigSpace</code> for the parameter space, or the search space definition.</p>"},{"location":"examples/02_using_DEHB_without_ConfigSpace/#defining-transformation-from-dehb-01-space-to-original-parameter-space","title":"Defining transformation from DEHB [0,1]-space to original parameter space\u00b6","text":""},{"location":"examples/02_using_DEHB_without_ConfigSpace/#defining-the-target_function","title":"Defining the target_function\u00b6","text":""},{"location":"examples/02_using_DEHB_without_ConfigSpace/#running-dehb","title":"Running DEHB\u00b6","text":""},{"location":"examples/02_using_DEHB_without_ConfigSpace/#evaluating-the-incumbent","title":"Evaluating the incumbent\u00b6","text":""},{"location":"examples/02_using_DEHB_without_ConfigSpace/#plotting-the-optimization-trace-with-the-update-of-incumbents-over-time","title":"Plotting the optimization trace with the update of incumbents over time\u00b6","text":""},{"location":"getting_started/parallel/","title":"Parallel","text":""},{"location":"getting_started/parallel/#running-dehb-in-a-parallel-setting","title":"Running DEHB in a parallel setting","text":"<p>DEHB has been designed to interface a Dask client. DEHB can either create a Dask client during instantiation and close/kill the client during garbage collection.  Or a client can be passed as an argument during instantiation.</p> <ul> <li>Setting <code>n_workers</code> during instantiation \\     If set to <code>1</code> (default) then the entire process is a sequential run without invoking Dask. \\     If set to <code>&gt;1</code> then a Dask Client is initialized with as many workers as <code>n_workers</code>. \\     This parameter is ignored if <code>client</code> is not None.</li> <li>Setting <code>client</code> during instantiation \\     When <code>None</code> (default), a Dask client is created using <code>n_workers</code> specified. \\     Else, any custom-configured Dask Client can be created and passed as the <code>client</code> argument to DEHB.</li> </ul>"},{"location":"getting_started/parallel/#using-gpus-in-a-parallel-run","title":"Using GPUs in a parallel run","text":"<p>Certain target function evaluations (especially for Deep Learning) require computations to be  carried out on GPUs. The GPU devices are often ordered by device ID and if not configured, all  spawned worker processes access these devices in the same order and can either run out of memory or not exhibit parallelism.</p> <p>For <code>n_workers&gt;1</code> and when running on a single node (or local), the <code>single_node_with_gpus</code> can be  passed to the <code>run()</code> call to DEHB. Setting it to <code>False</code> (default) has no effect on the default setup  of the machine. Setting it to <code>True</code> will reorder the GPU device IDs dynamically by setting the environment  variable <code>CUDA_VISIBLE_DEVICES</code> for each worker process executing a target function evaluation. The re-ordering  is done in a manner that the first priority device is the one with the least number of active jobs assigned  to it by that DEHB run.</p> <p>To run the PyTorch MNIST example on a single node using 2 workers: <pre><code>python examples/03_pytorch_mnist_hpo.py \\\n    --min_budget 1 \\\n    --max_budget 3 \\\n    --runtime 60 \\\n    --n_workers 2 \\\n    --single_node_with_gpus \\\n    --verbose\n</code></pre></p>"},{"location":"getting_started/parallel/#multi-node-runs","title":"Multi-node runs","text":"<p>Multi-node parallelism is often contingent on the cluster setup to be deployed on. Dask provides useful  frameworks to interface various cluster designs. As long as the <code>client</code> passed to DEHB during  instantiation is of type <code>dask.distributed.Client</code>, DEHB can interact with this client and  distribute its optimization process in a parallel manner. </p> <p>For instance, <code>Dask-CLI</code> can be used to create a <code>dask-scheduler</code> which can dump its connection  details to a file on a cluster node accessible to all processes. Multiple <code>dask-worker</code> can then be created to interface the <code>dask-scheduler</code> by connecting to the details read from the file dumped. Each dask-worker can be triggered on any remote machine. Each worker can be configured as required,  including mapping to specific GPU devices. </p> <p>Some helper scripts can be found here, that can be used as a reference to run DEHB in a multi-node  manner on clusters managed by SLURM. (not expected to work off-the-shelf)</p> <p>To run the PyTorch MNIST example on a multi-node setup using 4 workers: <pre><code>bash utils/run_dask_setup.sh \\\n    -n 4 \\\n    -f dask_dump/scheduler.json \\   # This is how the workers will be discovered by DEHB\n    -e env_name\n\n# Make sure to sleep to allow the workers to setup properly\nsleep 5\n\npython examples/03_pytorch_mnist_hpo.py \\\n    --min_fidelity 1 \\\n    --max_fidelity 3 \\\n    --runtime 60 \\\n    --scheduler_file dask_dump/scheduler.json \\\n    --verbose\n</code></pre></p>"},{"location":"getting_started/single_worker/","title":"Single Worker","text":""},{"location":"getting_started/single_worker/#basic-single-worker-setup","title":"Basic single worker setup","text":"<p>A basic setup for optimizing can be done as follows. Please note, that this is example should solely show a simple setup of <code>dehb</code>. More in-depth examples can be found in the examples folder. First we need to setup a <code>ConfigurationSpace</code>, from which Configurations will be sampled:</p> Configuration Space<pre><code>from ConfigSpace import ConfigurationSpace, Configuration\n\ncs = ConfigurationSpace({\"x0\": (3.0, 10.0), \"x1\": [\"red\", \"green\"]})\nprint(cs)\n</code></pre> <pre><code>Configuration space object:\n  Hyperparameters:\n    x0, Type: UniformFloat, Range: [3.0, 10.0], Default: 6.5\n    x1, Type: Categorical, Choices: {red, green}, Default: red\n</code></pre> <p>Next, we need an <code>object_function</code>, which we are aiming to optimize: Configuration Space<pre><code>import numpy as np\n\ndef objective_function(x: Configuration, fidelity: float, **kwargs):\n    # Replace this with your actual objective value (y) and cost.\n    cost = (10 if x[\"x1\"] == \"red\" else 100) + fidelity\n    y = x[\"x0\"] + np.random.uniform()\n    return {\"fitness\": y, \"cost\": x[\"x0\"]}\n\nsample_config = cs.sample_configuration()\nprint(sample_config)\n\nresult = objective_function(sample_config, fidelity=10)\nprint(result)\n</code></pre> <pre><code>Configuration(values={\n  'x0': 5.309818877459886,\n  'x1': 'red',\n})\n{'fitness': 6.085055062123084, 'cost': 5.309818877459886}\n</code></pre> </p> <p>Finally, we can setup our optimizer and run DEHB:</p> Configuration Space<pre><code>from dehb import DEHB\n\ndim = len(cs.get_hyperparameters())\noptimizer = DEHB(\n    f=objective_function,\n    cs=cs,\n    dimensions=dim,\n    min_fidelity=3,\n    max_fidelity=27,\n    eta=3,\n    n_workers=1,\n    output_path=\"./logs\",\n)\n\n# Run optimization for 1 bracket. Output files will be saved to ./logs\ntraj, runtime, history = optimizer.run(brackets=1, verbose=True)\nconfig, fitness, runtime, fidelity, _ = history[0]\nprint(\"config\", config)\nprint(\"fitness\", fitness)\nprint(\"runtime\", runtime)\nprint(\"fidelity\", fidelity)\n</code></pre> <pre><code>config [0.6940348967446345, 0.25]\nfitness 8.181542779533341\nruntime 7.8582442772124415\nfidelity 3.0\n</code></pre>"},{"location":"references/bracket_manager/","title":"Bracket manager","text":""},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager","title":"<code>SHBracketManager(n_configs, fidelities, bracket_id=None)</code>","text":"<p>             Bases: <code>object</code></p> <p>Synchronous Successive Halving utilities</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def __init__(self, n_configs, fidelities, bracket_id=None):\n    assert len(n_configs) == len(fidelities)\n    self.n_configs = n_configs\n    self.fidelities = fidelities\n    self.bracket_id = bracket_id\n    self.sh_bracket = {}\n    self._sh_bracket = {}\n    self._config_map = {}\n    for i, fidelity in enumerate(fidelities):\n        # sh_bracket keeps track of jobs/configs that are still to be scheduled/allocatted\n        # _sh_bracket keeps track of jobs/configs that have been run and results retrieved for\n        # (sh_bracket[i] + _sh_bracket[i]) == n_configs[i] is when no jobs have been scheduled\n        #   or all jobs for that fidelity/rung are over\n        # (sh_bracket[i] + _sh_bracket[i]) &lt; n_configs[i] indicates a job has been scheduled\n        #   and is queued/running and the bracket needs to be paused till results are retrieved\n        self.sh_bracket[fidelity] = n_configs[i]  # each scheduled job does -= 1\n        self._sh_bracket[fidelity] = 0  # each retrieved job does +=1\n    self.n_rungs = len(fidelities)\n    self.current_rung = 0\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.get_fidelity","title":"<code>get_fidelity(rung=None)</code>","text":"<p>Returns the exact fidelity that rung is pointing to.</p> <p>Returns current rung's fidelity if no rung is passed.</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def get_fidelity(self, rung=None):\n    \"\"\" Returns the exact fidelity that rung is pointing to.\n\n    Returns current rung's fidelity if no rung is passed.\n    \"\"\"\n    if rung is not None:\n        return self.fidelities[rung]\n    return self.fidelities[self.current_rung]\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.get_lower_fidelity_promotions","title":"<code>get_lower_fidelity_promotions(fidelity)</code>","text":"<p>Returns the immediate lower fidelity and the number of configs to be promoted from there</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def get_lower_fidelity_promotions(self, fidelity):\n    \"\"\" Returns the immediate lower fidelity and the number of configs to be promoted from there\n    \"\"\"\n    assert fidelity in self.fidelities\n    rung = np.where(fidelity == self.fidelities)[0][0]\n    prev_rung = np.clip(rung - 1, a_min=0, a_max=self.n_rungs-1)\n    lower_fidelity = self.fidelities[prev_rung]\n    num_promote_configs = self.n_configs[rung]\n    return lower_fidelity, num_promote_configs\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.get_next_job_fidelity","title":"<code>get_next_job_fidelity()</code>","text":"<p>Returns the fidelity that will be selected if current_rung is incremented by 1</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def get_next_job_fidelity(self):\n    \"\"\" Returns the fidelity that will be selected if current_rung is incremented by 1\n    \"\"\"\n    if self.sh_bracket[self.get_fidelity()] &gt; 0:\n        # the current rung still has unallocated jobs (&gt;0)\n        return self.get_fidelity()\n    else:\n        # the current rung has no more jobs to allocate, increment it\n        rung = (self.current_rung + 1) % self.n_rungs\n        if self.sh_bracket[self.get_fidelity(rung)] &gt; 0:\n            # the incremented rung has unallocated jobs (&gt;0)\n            return self.get_fidelity(rung)\n        else:\n            # all jobs for this bracket has been allocated/bracket is complete\n            # no more fidelities to evaluate and can return None\n            pass\n        return None\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.register_job","title":"<code>register_job(fidelity)</code>","text":"<p>Registers the allocation of a configuration for the fidelity and updates current rung</p> <p>This function must be called when scheduling a job in order to allow the bracket manager to continue job and fidelity allocation without waiting for jobs to finish and return results necessarily. This feature can be leveraged to run brackets asynchronously.</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def register_job(self, fidelity):\n    \"\"\" Registers the allocation of a configuration for the fidelity and updates current rung\n\n    This function must be called when scheduling a job in order to allow the bracket manager\n    to continue job and fidelity allocation without waiting for jobs to finish and return\n    results necessarily. This feature can be leveraged to run brackets asynchronously.\n    \"\"\"\n    assert fidelity in self.fidelities\n    assert self.sh_bracket[fidelity] &gt; 0\n    self.sh_bracket[fidelity] -= 1\n    if not self._is_rung_pending(self.current_rung):\n        # increment current rung if no jobs left in the rung\n        self.current_rung = (self.current_rung + 1) % self.n_rungs\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.complete_job","title":"<code>complete_job(fidelity)</code>","text":"<p>Notifies the bracket that a job for a fidelity has been completed</p> <p>This function must be called when a config for a fidelity has finished evaluation to inform the Bracket Manager that no job needs to be waited for and the next rung can begin for the synchronous Successive Halving case.</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def complete_job(self, fidelity):\n    \"\"\" Notifies the bracket that a job for a fidelity has been completed\n\n    This function must be called when a config for a fidelity has finished evaluation to inform\n    the Bracket Manager that no job needs to be waited for and the next rung can begin for the\n    synchronous Successive Halving case.\n    \"\"\"\n    assert fidelity in self.fidelities\n    _max_configs = self.n_configs[list(self.fidelities).index(fidelity)]\n    assert self._sh_bracket[fidelity] &lt; _max_configs\n    self._sh_bracket[fidelity] += 1\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.previous_rung_waits","title":"<code>previous_rung_waits()</code>","text":"<p>Returns True if none of the rungs &lt; current rung is waiting for results</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def previous_rung_waits(self):\n    \"\"\" Returns True if none of the rungs &lt; current rung is waiting for results\n    \"\"\"\n    for rung in range(self.current_rung):\n        if self._is_rung_waiting(rung) and not self._is_rung_pending(rung):\n            return True\n    return False\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.is_bracket_done","title":"<code>is_bracket_done()</code>","text":"<p>Returns True if all configs in all rungs in the bracket have been allocated</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def is_bracket_done(self):\n    \"\"\" Returns True if all configs in all rungs in the bracket have been allocated\n    \"\"\"\n    return ~self.is_pending() and ~self.is_waiting()\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.is_pending","title":"<code>is_pending()</code>","text":"<p>Returns True if any of the rungs/fidelities have still a configuration to submit</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def is_pending(self):\n    \"\"\" Returns True if any of the rungs/fidelities have still a configuration to submit\n    \"\"\"\n    return np.any([self._is_rung_pending(i) &gt; 0 for i, _ in enumerate(self.fidelities)])\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.is_waiting","title":"<code>is_waiting()</code>","text":"<p>Returns True if any of the rungs/fidelities have a configuration pending/running</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def is_waiting(self):\n    \"\"\" Returns True if any of the rungs/fidelities have a configuration pending/running\n    \"\"\"\n    return np.any([self._is_rung_waiting(i) &gt; 0 for i, _ in enumerate(self.fidelities)])\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.reset_waiting_jobs","title":"<code>reset_waiting_jobs()</code>","text":"<p>Resets all waiting jobs and updates the current_rung pointer accordingly.</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def reset_waiting_jobs(self):\n    \"\"\"Resets all waiting jobs and updates the current_rung pointer accordingly.\"\"\"\n    for i, fidelity in enumerate(self.fidelities):\n        pending = self.sh_bracket[fidelity]\n        done = self._sh_bracket[fidelity]\n        waiting = np.abs(self.n_configs[i] - pending - done)\n\n        # update current_rung pointer to the lowest rung with waiting jobs\n        if waiting &gt; 0 and self.current_rung &gt; i:\n            self.current_rung = i\n        # reset waiting jobs\n        self.sh_bracket[fidelity] += waiting\n</code></pre>"},{"location":"references/de/","title":"DE","text":""},{"location":"references/de/#dehb.optimizers.de.DEBase","title":"<code>DEBase(cs=None, f=None, dimensions=None, pop_size=None, max_age=None, mutation_factor=None, crossover_prob=None, strategy=None, boundary_fix_type='random', config_repository=None, **kwargs)</code>","text":"<p>Base class for Differential Evolution</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=None,\n             mutation_factor=None, crossover_prob=None, strategy=None,\n             boundary_fix_type='random', config_repository=None, **kwargs):\n    # Benchmark related variables\n    self.cs = cs\n    self.f = f\n    if dimensions is None and self.cs is not None:\n        self.dimensions = len(self.cs.get_hyperparameters())\n    else:\n        self.dimensions = dimensions\n\n    # DE related variables\n    self.pop_size = pop_size\n    self.max_age = max_age\n    self.mutation_factor = mutation_factor\n    self.crossover_prob = crossover_prob\n    self.strategy = strategy\n    self.fix_type = boundary_fix_type\n\n    # Miscellaneous\n    self.configspace = True if isinstance(self.cs, ConfigSpace.ConfigurationSpace) else False\n    self.hps = dict()\n    if self.configspace:\n        for i, hp in enumerate(cs.get_hyperparameters()):\n            # maps hyperparameter name to positional index in vector form\n            self.hps[hp.name] = i\n    self.output_path = kwargs['output_path'] if 'output_path' in kwargs else './'\n    os.makedirs(self.output_path, exist_ok=True)\n\n    if config_repository:\n        self.config_repository = config_repository\n    else:\n        self.config_repository = ConfigRepository()\n\n    # Global trackers\n    self.inc_score : float\n    self.inc_config : np.ndarray[float]\n    self.inc_id : int\n    self.population : np.ndarray[np.ndarray[float]]\n    self.population_ids :np.ndarray[int]\n    self.fitness : np.ndarray[float]\n    self.age : int\n    self.history : list[object]\n    self.reset()\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DEBase.sample_population","title":"<code>sample_population(size=3, alt_pop=None)</code>","text":"<p>Samples 'size' individuals</p> <p>If alt_pop is None or a list/array of None, sample from own population Else sample from the specified alternate population (alt_pop)</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def sample_population(self, size: int = 3, alt_pop: List = None) -&gt; List:\n    '''Samples 'size' individuals\n\n    If alt_pop is None or a list/array of None, sample from own population\n    Else sample from the specified alternate population (alt_pop)\n    '''\n    if isinstance(alt_pop, list) or isinstance(alt_pop, np.ndarray):\n        idx = [indv is None for indv in alt_pop]\n        if any(idx):\n            selection = np.random.choice(np.arange(len(self.population)), size, replace=False)\n            return self.population[selection]\n        else:\n            if len(alt_pop) &lt; 3:\n                alt_pop = np.vstack((alt_pop, self.population))\n            selection = np.random.choice(np.arange(len(alt_pop)), size, replace=False)\n            alt_pop = np.stack(alt_pop)\n            return alt_pop[selection]\n    else:\n        selection = np.random.choice(np.arange(len(self.population)), size, replace=False)\n        return self.population[selection]\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DEBase.boundary_check","title":"<code>boundary_check(vector)</code>","text":"<p>Checks whether each of the dimensions of the input vector are within [0, 1]. If not, values of those dimensions are replaced with the type of fix selected.</p> <p>if fix_type == 'random', the values are replaced with a random sampling from (0,1) if fix_type == 'clip', the values are clipped to the closest limit from {0, 1}</p>"},{"location":"references/de/#dehb.optimizers.de.DEBase.boundary_check--parameters","title":"Parameters","text":"<p>vector : array</p>"},{"location":"references/de/#dehb.optimizers.de.DEBase.boundary_check--returns","title":"Returns","text":"<p>array</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def boundary_check(self, vector: np.ndarray) -&gt; np.ndarray:\n    '''\n    Checks whether each of the dimensions of the input vector are within [0, 1].\n    If not, values of those dimensions are replaced with the type of fix selected.\n\n    if fix_type == 'random', the values are replaced with a random sampling from (0,1)\n    if fix_type == 'clip', the values are clipped to the closest limit from {0, 1}\n\n    Parameters\n    ----------\n    vector : array\n\n    Returns\n    -------\n    array\n    '''\n    violations = np.where((vector &gt; 1) | (vector &lt; 0))[0]\n    if len(violations) == 0:\n        return vector\n    if self.fix_type == 'random':\n        vector[violations] = np.random.uniform(low=0.0, high=1.0, size=len(violations))\n    else:\n        vector[violations] = np.clip(vector[violations], a_min=0, a_max=1)\n    return vector\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DEBase.vector_to_configspace","title":"<code>vector_to_configspace(vector)</code>","text":"<p>Converts numpy array to ConfigSpace object</p> <p>Works when self.cs is a ConfigSpace object and the input vector is in the domain [0, 1].</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def vector_to_configspace(self, vector: np.ndarray) -&gt; ConfigSpace.Configuration:\n    '''Converts numpy array to ConfigSpace object\n\n    Works when self.cs is a ConfigSpace object and the input vector is in the domain [0, 1].\n    '''\n    # creates a ConfigSpace object dict with all hyperparameters present, the inactive too\n    new_config = ConfigSpace.util.impute_inactive_values(\n        self.cs.get_default_configuration()\n    ).get_dictionary()\n    # iterates over all hyperparameters and normalizes each based on its type\n    for i, hyper in enumerate(self.cs.get_hyperparameters()):\n        if type(hyper) == ConfigSpace.OrdinalHyperparameter:\n            ranges = np.arange(start=0, stop=1, step=1/len(hyper.sequence))\n            param_value = hyper.sequence[np.where((vector[i] &lt; ranges) == False)[0][-1]]\n        elif type(hyper) == ConfigSpace.CategoricalHyperparameter:\n            ranges = np.arange(start=0, stop=1, step=1/len(hyper.choices))\n            param_value = hyper.choices[np.where((vector[i] &lt; ranges) == False)[0][-1]]\n        elif type(hyper) == ConfigSpace.Constant:\n            param_value = hyper.default_value\n        else:  # handles UniformFloatHyperparameter &amp; UniformIntegerHyperparameter\n            # rescaling continuous values\n            if hyper.log:\n                log_range = np.log(hyper.upper) - np.log(hyper.lower)\n                param_value = np.exp(np.log(hyper.lower) + vector[i] * log_range)\n            else:\n                param_value = hyper.lower + (hyper.upper - hyper.lower) * vector[i]\n            if type(hyper) == ConfigSpace.UniformIntegerHyperparameter:\n                param_value = int(np.round(param_value))  # converting to discrete (int)\n            else:\n                param_value = float(param_value)\n        new_config[hyper.name] = param_value\n    # the mapping from unit hypercube to the actual config space may lead to illegal\n    # configurations based on conditions defined, which need to be deactivated/removed\n    new_config = ConfigSpace.util.deactivate_inactive_hyperparameters(\n        configuration = new_config, configuration_space=self.cs\n    )\n    return new_config\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DEBase.configspace_to_vector","title":"<code>configspace_to_vector(config)</code>","text":"<p>Converts ConfigSpace object to numpy array scaled to [0,1]</p> <p>Works when self.cs is a ConfigSpace object and the input config is a ConfigSpace object. Handles conditional spaces implicitly by replacing illegal parameters with default values to maintain the dimensionality of the vector.</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def configspace_to_vector(self, config: ConfigSpace.Configuration) -&gt; np.ndarray:\n    '''Converts ConfigSpace object to numpy array scaled to [0,1]\n\n    Works when self.cs is a ConfigSpace object and the input config is a ConfigSpace object.\n    Handles conditional spaces implicitly by replacing illegal parameters with default values\n    to maintain the dimensionality of the vector.\n    '''\n    # the imputation replaces illegal parameter values with their default\n    config = ConfigSpace.util.impute_inactive_values(config)\n    dimensions = len(self.cs.get_hyperparameters())\n    vector = [np.nan for i in range(dimensions)]\n    for name in config:\n        i = self.hps[name]\n        hyper = self.cs.get_hyperparameter(name)\n        if type(hyper) == ConfigSpace.OrdinalHyperparameter:\n            nlevels = len(hyper.sequence)\n            vector[i] = hyper.sequence.index(config[name]) / nlevels\n        elif type(hyper) == ConfigSpace.CategoricalHyperparameter:\n            nlevels = len(hyper.choices)\n            vector[i] = hyper.choices.index(config[name]) / nlevels\n        elif type(hyper) == ConfigSpace.Constant:\n            vector[i] = 0 # set constant to 0, so that it wont be affected by mutation\n        else:\n            bounds = (hyper.lower, hyper.upper)\n            param_value = config[name]\n            if hyper.log:\n                vector[i] = np.log(param_value / bounds[0]) / np.log(bounds[1] / bounds[0])\n            else:\n                vector[i] = (config[name] - bounds[0]) / (bounds[1] - bounds[0])\n    return np.array(vector)\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE","title":"<code>DE(cs=None, f=None, dimensions=None, pop_size=20, max_age=np.inf, mutation_factor=None, crossover_prob=None, strategy='rand1_bin', encoding=False, dim_map=None, config_repository=None, **kwargs)</code>","text":"<p>             Bases: <code>DEBase</code></p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def __init__(self, cs=None, f=None, dimensions=None, pop_size=20, max_age=np.inf,\n             mutation_factor=None, crossover_prob=None, strategy='rand1_bin',\n             encoding=False, dim_map=None, config_repository=None, **kwargs):\n    super().__init__(cs=cs, f=f, dimensions=dimensions, pop_size=pop_size, max_age=max_age,\n                     mutation_factor=mutation_factor, crossover_prob=crossover_prob,\n                     strategy=strategy, config_repository=config_repository,\n                     **kwargs)\n    if self.strategy is not None:\n        self.mutation_strategy = self.strategy.split('_')[0]\n        self.crossover_strategy = self.strategy.split('_')[1]\n    else:\n        self.mutation_strategy = self.crossover_strategy = None\n    self.encoding = encoding\n    self.dim_map = dim_map\n    self._set_min_pop_size()\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.__getstate__","title":"<code>__getstate__()</code>","text":"<p>Allows the object to picklable while having Dask client as a class attribute.</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def __getstate__(self):\n    \"\"\" Allows the object to picklable while having Dask client as a class attribute.\n    \"\"\"\n    d = dict(self.__dict__)\n    d[\"client\"] = None  # hack to allow Dask client to be a class attribute\n    d[\"logger\"] = None  # hack to allow logger object to be a class attribute\n    return d\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.__del__","title":"<code>__del__()</code>","text":"<p>Ensures a clean kill of the Dask client and frees up a port.</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def __del__(self):\n    \"\"\" Ensures a clean kill of the Dask client and frees up a port.\n    \"\"\"\n    if hasattr(self, \"client\") and isinstance(self.client, Client):\n        self.client.close()\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.init_eval_pop","title":"<code>init_eval_pop(fidelity=None, eval=True, **kwargs)</code>","text":"<p>Creates new population of 'pop_size' and evaluates individuals.</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def init_eval_pop(self, fidelity=None, eval=True, **kwargs):\n    '''Creates new population of 'pop_size' and evaluates individuals.\n    '''\n    self.population = self.init_population(self.pop_size)\n    self.population_ids = self.config_repository.announce_population(self.population, fidelity)\n    self.fitness = np.array([np.inf for i in range(self.pop_size)])\n    self.age = np.array([self.max_age] * self.pop_size)\n\n    traj = []\n    runtime = []\n    history = []\n\n    if not eval:\n        return traj, runtime, history\n\n    for i in range(self.pop_size):\n        config = self.population[i]\n        config_id = self.population_ids[i]\n        res = self.f_objective(config, fidelity, **kwargs)\n        self.fitness[i], cost = res[\"fitness\"], res[\"cost\"]\n        info = res[\"info\"] if \"info\" in res else dict()\n        if self.fitness[i] &lt; self.inc_score:\n            self.inc_score = self.fitness[i]\n            self.inc_config = config\n            self.inc_id = config_id\n        self.config_repository.tell_result(config_id, float(fidelity or 0), res[\"fitness\"], res[\"cost\"], info)\n        traj.append(self.inc_score)\n        runtime.append(cost)\n        history.append((config.tolist(), float(self.fitness[i]), float(fidelity or 0), info))\n\n    return traj, runtime, history\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.eval_pop","title":"<code>eval_pop(population=None, population_ids=None, fidelity=None, **kwargs)</code>","text":"<p>Evaluates a population</p> <p>If population=None, the current population's fitness will be evaluated If population!=None, this population will be evaluated</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def eval_pop(self, population=None, population_ids=None, fidelity=None, **kwargs):\n    '''Evaluates a population\n\n    If population=None, the current population's fitness will be evaluated\n    If population!=None, this population will be evaluated\n    '''\n    pop = self.population if population is None else population\n    pop_ids = self.population_ids if population_ids is None else population_ids\n    pop_size = self.pop_size if population is None else len(pop)\n    traj = []\n    runtime = []\n    history = []\n    fitnesses = []\n    costs = []\n    ages = []\n    for i in range(pop_size):\n        res = self.f_objective(pop[i], fidelity, **kwargs)\n        fitness, cost = res[\"fitness\"], res[\"cost\"]\n        info = res[\"info\"] if \"info\" in res else dict()\n        if population is None:\n            self.fitness[i] = fitness\n        if fitness &lt;= self.inc_score:\n            self.inc_score = fitness\n            self.inc_config = pop[i]\n            self.inc_id = pop_ids[i]\n        self.config_repository.tell_result(pop_ids[i], float(fidelity or 0), info)\n        traj.append(self.inc_score)\n        runtime.append(cost)\n        history.append((pop[i].tolist(), float(fitness), float(fidelity or 0), info))\n        fitnesses.append(fitness)\n        costs.append(cost)\n        ages.append(self.max_age)\n    if population is None:\n        self.fitness = np.array(fitnesses)\n        return traj, runtime, history\n    else:\n        return traj, runtime, history, np.array(fitnesses), np.array(ages)\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.mutation_rand1","title":"<code>mutation_rand1(r1, r2, r3)</code>","text":"<p>Performs the 'rand1' type of DE mutation</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def mutation_rand1(self, r1, r2, r3):\n    '''Performs the 'rand1' type of DE mutation\n    '''\n    diff = r2 - r3\n    mutant = r1 + self.mutation_factor * diff\n    return mutant\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.mutation_rand2","title":"<code>mutation_rand2(r1, r2, r3, r4, r5)</code>","text":"<p>Performs the 'rand2' type of DE mutation</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def mutation_rand2(self, r1, r2, r3, r4, r5):\n    '''Performs the 'rand2' type of DE mutation\n    '''\n    diff1 = r2 - r3\n    diff2 = r4 - r5\n    mutant = r1 + self.mutation_factor * diff1 + self.mutation_factor * diff2\n    return mutant\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.mutation","title":"<code>mutation(current=None, best=None, alt_pop=None)</code>","text":"<p>Performs DE mutation</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def mutation(self, current=None, best=None, alt_pop=None):\n    '''Performs DE mutation\n    '''\n    if self.mutation_strategy == 'rand1':\n        r1, r2, r3 = self.sample_population(size=3, alt_pop=alt_pop)\n        mutant = self.mutation_rand1(r1, r2, r3)\n\n    elif self.mutation_strategy == 'rand2':\n        r1, r2, r3, r4, r5 = self.sample_population(size=5, alt_pop=alt_pop)\n        mutant = self.mutation_rand2(r1, r2, r3, r4, r5)\n\n    elif self.mutation_strategy == 'rand2dir':\n        r1, r2, r3 = self.sample_population(size=3, alt_pop=alt_pop)\n        mutant = self.mutation_rand2dir(r1, r2, r3)\n\n    elif self.mutation_strategy == 'best1':\n        r1, r2 = self.sample_population(size=2, alt_pop=alt_pop)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_rand1(best, r1, r2)\n\n    elif self.mutation_strategy == 'best2':\n        r1, r2, r3, r4 = self.sample_population(size=4, alt_pop=alt_pop)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_rand2(best, r1, r2, r3, r4)\n\n    elif self.mutation_strategy == 'currenttobest1':\n        r1, r2 = self.sample_population(size=2, alt_pop=alt_pop)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_currenttobest1(current, best, r1, r2)\n\n    elif self.mutation_strategy == 'randtobest1':\n        r1, r2, r3 = self.sample_population(size=3, alt_pop=alt_pop)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_currenttobest1(r1, best, r2, r3)\n\n    return mutant\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.crossover_bin","title":"<code>crossover_bin(target, mutant)</code>","text":"<p>Performs the binomial crossover of DE</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def crossover_bin(self, target, mutant):\n    '''Performs the binomial crossover of DE\n    '''\n    cross_points = np.random.rand(self.dimensions) &lt; self.crossover_prob\n    if not np.any(cross_points):\n        cross_points[np.random.randint(0, self.dimensions)] = True\n    offspring = np.where(cross_points, mutant, target)\n    return offspring\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.crossover_exp","title":"<code>crossover_exp(target, mutant)</code>","text":"<p>Performs the exponential crossover of DE</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def crossover_exp(self, target, mutant):\n    '''Performs the exponential crossover of DE\n    '''\n    n = np.random.randint(0, self.dimensions)\n    L = 0\n    while ((np.random.rand() &lt; self.crossover_prob) and L &lt; self.dimensions):\n        idx = (n+L) % self.dimensions\n        target[idx] = mutant[idx]\n        L = L + 1\n    return target\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.crossover","title":"<code>crossover(target, mutant)</code>","text":"<p>Performs DE crossover</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def crossover(self, target, mutant):\n    '''Performs DE crossover\n    '''\n    if self.crossover_strategy == 'bin':\n        offspring = self.crossover_bin(target, mutant)\n    elif self.crossover_strategy == 'exp':\n        offspring = self.crossover_exp(target, mutant)\n    return offspring\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.selection","title":"<code>selection(trials, trial_ids, fidelity=None, **kwargs)</code>","text":"<p>Carries out a parent-offspring competition given a set of trial population</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def selection(self, trials, trial_ids, fidelity=None, **kwargs):\n    '''Carries out a parent-offspring competition given a set of trial population\n    '''\n    traj = []\n    runtime = []\n    history = []\n    for i in range(len(trials)):\n        # evaluation of the newly created individuals\n        res = self.f_objective(trials[i], fidelity, **kwargs)\n        fitness, cost = res[\"fitness\"], res[\"cost\"]\n        info = res[\"info\"] if \"info\" in res else dict()\n        # log result to config repo\n        self.config_repository.tell_result(trial_ids[i], float(fidelity or 0), fitness, cost, info)\n        # selection -- competition between parent[i] -- child[i]\n        ## equality is important for landscape exploration\n        if fitness &lt;= self.fitness[i]:\n            self.population[i] = trials[i]\n            self.population_ids[i] = trial_ids[i]\n            self.fitness[i] = fitness\n            # resetting age since new individual in the population\n            self.age[i] = self.max_age\n        else:\n            # decreasing age by 1 of parent who is better than offspring/trial\n            self.age[i] -= 1\n        # updation of global incumbent for trajectory\n        if self.fitness[i] &lt; self.inc_score:\n            self.inc_score = self.fitness[i]\n            self.inc_config = self.population[i]\n            self.inc_id = self.population[i]\n        traj.append(self.inc_score)\n        runtime.append(cost)\n        history.append((trials[i].tolist(), float(fitness), float(fidelity or 0), info))\n    return traj, runtime, history\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.evolve_generation","title":"<code>evolve_generation(fidelity=None, best=None, alt_pop=None, **kwargs)</code>","text":"<p>Performs a complete DE evolution: mutation -&gt; crossover -&gt; selection</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs):\n    '''Performs a complete DE evolution: mutation -&gt; crossover -&gt; selection\n    '''\n    trials = []\n    trial_ids = []\n    for j in range(self.pop_size):\n        target = self.population[j]\n        donor = self.mutation(current=target, best=best, alt_pop=alt_pop)\n        trial = self.crossover(target, donor)\n        trial = self.boundary_check(trial)\n        trial_id = self.config_repository.announce_config(trial, float(fidelity or 0))\n        trials.append(trial)\n        trial_ids.append(trial_id)\n    trials = np.array(trials)\n    trial_ids = np.array(trial_ids)\n    traj, runtime, history = self.selection(trials, trial_ids, fidelity, **kwargs)\n    return traj, runtime, history\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.sample_mutants","title":"<code>sample_mutants(size, population=None)</code>","text":"<p>Generates 'size' mutants from the population using rand1</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def sample_mutants(self, size, population=None):\n    '''Generates 'size' mutants from the population using rand1\n    '''\n    if population is None:\n        population = self.population\n    elif len(population) &lt; 3:\n        population = np.vstack((self.population, population))\n\n    old_strategy = self.mutation_strategy\n    self.mutation_strategy = 'rand1'\n    mutants = np.random.uniform(low=0.0, high=1.0, size=(size, self.dimensions))\n    for i in range(size):\n        mutant = self.mutation(current=None, best=None, alt_pop=population)\n        mutants[i] = self.boundary_check(mutant)\n    self.mutation_strategy = old_strategy\n\n    return mutants\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.AsyncDE","title":"<code>AsyncDE(cs=None, f=None, dimensions=None, pop_size=None, max_age=np.inf, mutation_factor=None, crossover_prob=None, strategy='rand1_bin', async_strategy='immediate', config_repository=None, **kwargs)</code>","text":"<p>             Bases: <code>DE</code></p> <p>Extends DE to be Asynchronous with variations</p>"},{"location":"references/de/#dehb.optimizers.de.AsyncDE--parameters","title":"Parameters","text":"<p>async_strategy : str     'deferred' - target will be chosen sequentially from the population         the winner of the selection step will be included in the population only after         the entire population has had a selection step in that generation     'immediate' - target will be chosen sequentially from the population         the winner of the selection step is included in the population right away     'random' - target will be chosen randomly from the population for mutation-crossover         the winner of the selection step is included in the population right away     'worst' - the worst individual will be chosen as the target         the winner of the selection step is included in the population right away     {immediate, worst, random} implement Asynchronous-DE</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=np.inf,\n             mutation_factor=None, crossover_prob=None, strategy='rand1_bin',\n             async_strategy='immediate', config_repository=None, **kwargs):\n    '''Extends DE to be Asynchronous with variations\n\n    Parameters\n    ----------\n    async_strategy : str\n        'deferred' - target will be chosen sequentially from the population\n            the winner of the selection step will be included in the population only after\n            the entire population has had a selection step in that generation\n        'immediate' - target will be chosen sequentially from the population\n            the winner of the selection step is included in the population right away\n        'random' - target will be chosen randomly from the population for mutation-crossover\n            the winner of the selection step is included in the population right away\n        'worst' - the worst individual will be chosen as the target\n            the winner of the selection step is included in the population right away\n        {immediate, worst, random} implement Asynchronous-DE\n    '''\n    super().__init__(cs=cs, f=f, dimensions=dimensions, pop_size=pop_size, max_age=max_age,\n                     mutation_factor=mutation_factor, crossover_prob=crossover_prob,\n                     strategy=strategy, config_repository=config_repository,\n                     **kwargs)\n    if self.strategy is not None:\n        self.mutation_strategy = self.strategy.split('_')[0]\n        self.crossover_strategy = self.strategy.split('_')[1]\n    else:\n        self.mutation_strategy = self.crossover_strategy = None\n    self.async_strategy = async_strategy\n    assert self.async_strategy in ['immediate', 'random', 'worst', 'deferred'], \\\n            \"{} is not a valid choice for type of DE\".format(self.async_strategy)\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.AsyncDE.mutation","title":"<code>mutation(current=None, best=None, alt_pop=None)</code>","text":"<p>Performs DE mutation</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def mutation(self, current=None, best=None, alt_pop=None):\n    '''Performs DE mutation\n    '''\n    if self.mutation_strategy == 'rand1':\n        r1, r2, r3 = self._sample_population(size=3, alt_pop=alt_pop, target=current)\n        mutant = self.mutation_rand1(r1, r2, r3)\n\n    elif self.mutation_strategy == 'rand2':\n        r1, r2, r3, r4, r5 = self._sample_population(size=5, alt_pop=alt_pop, target=current)\n        mutant = self.mutation_rand2(r1, r2, r3, r4, r5)\n\n    elif self.mutation_strategy == 'rand2dir':\n        r1, r2, r3 = self._sample_population(size=3, alt_pop=alt_pop, target=current)\n        mutant = self.mutation_rand2dir(r1, r2, r3)\n\n    elif self.mutation_strategy == 'best1':\n        r1, r2 = self._sample_population(size=2, alt_pop=alt_pop, target=current)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_rand1(best, r1, r2)\n\n    elif self.mutation_strategy == 'best2':\n        r1, r2, r3, r4 = self._sample_population(size=4, alt_pop=alt_pop, target=current)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_rand2(best, r1, r2, r3, r4)\n\n    elif self.mutation_strategy == 'currenttobest1':\n        r1, r2 = self._sample_population(size=2, alt_pop=alt_pop, target=current)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_currenttobest1(current, best, r1, r2)\n\n    elif self.mutation_strategy == 'randtobest1':\n        r1, r2, r3 = self._sample_population(size=3, alt_pop=alt_pop, target=current)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_currenttobest1(r1, best, r2, r3)\n\n    return mutant\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.AsyncDE.sample_mutants","title":"<code>sample_mutants(size, population=None)</code>","text":"<p>Samples 'size' mutants from the population</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def sample_mutants(self, size, population=None):\n    '''Samples 'size' mutants from the population\n    '''\n    if population is None:\n        population = self.population\n\n    mutants = np.random.uniform(low=0.0, high=1.0, size=(size, self.dimensions))\n    for i in range(size):\n        j = np.random.choice(np.arange(len(population)))\n        mutant = self.mutation(current=population[j], best=self.inc_config, alt_pop=population)\n        mutants[i] = self.boundary_check(mutant)\n\n    return mutants\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.AsyncDE.evolve_generation","title":"<code>evolve_generation(fidelity=None, best=None, alt_pop=None, **kwargs)</code>","text":"<p>Performs a complete DE evolution, mutation -&gt; crossover -&gt; selection</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs):\n    '''Performs a complete DE evolution, mutation -&gt; crossover -&gt; selection\n    '''\n    traj = []\n    runtime = []\n    history = []\n\n    if self.async_strategy == \"deferred\":\n        trials = []\n        trial_ids = []\n        for j in range(self.pop_size):\n            target = self.population[j]\n            donor = self.mutation(current=target, best=best, alt_pop=alt_pop)\n            trial = self.crossover(target, donor)\n            trial = self.boundary_check(trial)\n            trial_id = self.config_repository.announce_config(trial, float(fidelity or 0))\n            trials.append(trial)\n            trial_ids.append(trial_id)\n        # selection takes place on a separate trial population only after\n        # one iteration through the population has taken place\n        trials = np.array(trials)\n        traj, runtime, history = self.selection(trials, trial_ids, fidelity, **kwargs)\n        return traj, runtime, history\n\n    elif self.async_strategy == \"immediate\":\n        for i in range(self.pop_size):\n            target = self.population[i]\n            donor = self.mutation(current=target, best=best, alt_pop=alt_pop)\n            trial = self.crossover(target, donor)\n            trial = self.boundary_check(trial)\n            trial_id = self.config_repository.announce_config(trial, float(fidelity or 0))\n            # evaluating a single trial population for the i-th individual\n            de_traj, de_runtime, de_history, fitnesses, costs = \\\n                self.eval_pop(trial.reshape(1, self.dimensions),\n                              np.array([trial_id]), fidelity=fidelity, **kwargs)\n            # one-vs-one selection\n            ## can replace the i-the population despite not completing one iteration\n            if fitnesses[0] &lt;= self.fitness[i]:\n                self.population[i] = trial\n                self.population_ids[i] = trial_id\n                self.fitness[i] = fitnesses[0]\n            traj.extend(de_traj)\n            runtime.extend(de_runtime)\n            history.extend(de_history)\n        return traj, runtime, history\n\n    else:  # async_strategy == 'random' or async_strategy == 'worst':\n        for count in range(self.pop_size):\n            # choosing target individual\n            if self.async_strategy == \"random\":\n                i = np.random.choice(np.arange(self.pop_size))\n            else:  # async_strategy == 'worst'\n                i = np.argsort(-self.fitness)[0]\n            target = self.population[i]\n            mutant = self.mutation(current=target, best=best, alt_pop=alt_pop)\n            trial = self.crossover(target, mutant)\n            trial = self.boundary_check(trial)\n            trial_id = self.config_repository.announce_config(trial, float(fidelity or 0))\n            # evaluating a single trial population for the i-th individual\n            de_traj, de_runtime, de_history, fitnesses, costs = \\\n                self.eval_pop(trial.reshape(1, self.dimensions), np.array([trial_id]),\n                               fidelity=fidelity, **kwargs)\n            # one-vs-one selection\n            ## can replace the i-the population despite not completing one iteration\n            if fitnesses[0] &lt;= self.fitness[i]:\n                self.population[i] = trial\n                self.fitness[i] = fitnesses[0]\n            traj.extend(de_traj)\n            runtime.extend(de_runtime)\n            history.extend(de_history)\n\n    return traj, runtime, history\n</code></pre>"},{"location":"references/dehb/","title":"DEHB","text":""},{"location":"references/dehb/#dehb.optimizers.dehb.DEHBBase","title":"<code>DEHBBase(cs=None, f=None, dimensions=None, mutation_factor=None, crossover_prob=None, strategy=None, min_fidelity=None, max_fidelity=None, eta=None, min_clip=None, max_clip=None, boundary_fix_type='random', max_age=np.inf, **kwargs)</code>","text":"Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None,\n             crossover_prob=None, strategy=None, min_fidelity=None,\n             max_fidelity=None, eta=None, min_clip=None, max_clip=None,\n             boundary_fix_type='random', max_age=np.inf, **kwargs):\n    # Miscellaneous\n    self._setup_logger(kwargs)\n    self.config_repository = ConfigRepository()\n\n    # Benchmark related variables\n    self.cs = cs\n    self.configspace = True if isinstance(self.cs, ConfigSpace.ConfigurationSpace) else False\n    if self.configspace:\n        self.dimensions = len(self.cs.get_hyperparameters())\n    elif dimensions is None or not isinstance(dimensions, (int, np.integer)):\n        assert \"Need to specify `dimensions` as an int when `cs` is not available/specified!\"\n    else:\n        self.dimensions = dimensions\n    self.f = f\n\n    # DE related variables\n    self.mutation_factor = mutation_factor\n    self.crossover_prob = crossover_prob\n    self.strategy = strategy\n    self.fix_type = boundary_fix_type\n    self.max_age = max_age\n    self.de_params = {\n        \"mutation_factor\": self.mutation_factor,\n        \"crossover_prob\": self.crossover_prob,\n        \"strategy\": self.strategy,\n        \"configspace\": self.configspace,\n        \"boundary_fix_type\": self.fix_type,\n        \"max_age\": self.max_age,\n        \"cs\": self.cs,\n        \"dimensions\": self.dimensions,\n        \"f\": f\n    }\n\n    # Hyperband related variables\n    self.min_fidelity = min_fidelity\n    self.max_fidelity = max_fidelity\n    if self.max_fidelity &lt;= self.min_fidelity:\n        self.logger.error(\"Only (Max Fidelity &gt; Min Fidelity) is supported for DEHB.\")\n        if self.max_fidelity == self.min_fidelity:\n            self.logger.error(\n                \"If you have a fixed fidelity, \" \\\n                \"you can instead run DE. For more information checkout: \" \\\n                \"https://automl.github.io/DEHB/references/de\")\n        raise AssertionError()\n    self.eta = eta\n    self.min_clip = min_clip\n    self.max_clip = max_clip\n\n    # Precomputing fidelity spacing and number of configurations for HB iterations\n    self.max_SH_iter = None\n    self.fidelities = None\n    if self.min_fidelity is not None and \\\n       self.max_fidelity is not None and \\\n       self.eta is not None:\n        self.max_SH_iter = -int(np.log(self.min_fidelity / self.max_fidelity) / np.log(self.eta)) + 1\n        self.fidelities = self.max_fidelity * np.power(self.eta,\n                                                 -np.linspace(start=self.max_SH_iter - 1,\n                                                              stop=0, num=self.max_SH_iter))\n\n    # Updating DE parameter list\n    self.de_params.update({\"output_path\": self.output_path})\n\n    # Global trackers\n    self.population = None\n    self.fitness = None\n    self.inc_score = np.inf\n    self.inc_config = None\n    self.history = []\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHBBase.get_next_iteration","title":"<code>get_next_iteration(iteration)</code>","text":"<p>Computes the Successive Halving spacing</p> <p>Given the iteration index, computes the fidelity spacing to be used and the number of configurations to be used for the SH iterations.</p>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHBBase.get_next_iteration--parameters","title":"Parameters","text":"<p>iteration : int     Iteration index clip : int, {1, 2, 3, ..., None}     If not None, clips the minimum number of configurations to 'clip'</p>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHBBase.get_next_iteration--returns","title":"Returns","text":"<p>ns : array fidelities : array</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def get_next_iteration(self, iteration):\n    '''Computes the Successive Halving spacing\n\n    Given the iteration index, computes the fidelity spacing to be used and\n    the number of configurations to be used for the SH iterations.\n\n    Parameters\n    ----------\n    iteration : int\n        Iteration index\n    clip : int, {1, 2, 3, ..., None}\n        If not None, clips the minimum number of configurations to 'clip'\n\n    Returns\n    -------\n    ns : array\n    fidelities : array\n    '''\n    # number of 'SH runs'\n    s = self.max_SH_iter - 1 - (iteration % self.max_SH_iter)\n    # fidelity spacing for this iteration\n    fidelities = self.fidelities[(-s-1):]\n    # number of configurations in that bracket\n    n0 = int(np.floor((self.max_SH_iter)/(s+1)) * self.eta**s)\n    ns = [max(int(n0*(self.eta**(-i))), 1) for i in range(s+1)]\n    if self.min_clip is not None and self.max_clip is not None:\n        ns = np.clip(ns, a_min=self.min_clip, a_max=self.max_clip)\n    elif self.min_clip is not None:\n        ns = np.clip(ns, a_min=self.min_clip, a_max=np.max(ns))\n\n    return ns, fidelities\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHBBase.get_incumbents","title":"<code>get_incumbents()</code>","text":"<p>Returns a tuple of the (incumbent configuration, incumbent score/fitness).</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def get_incumbents(self):\n    \"\"\" Returns a tuple of the (incumbent configuration, incumbent score/fitness). \"\"\"\n    if self.configspace:\n        return self.vector_to_configspace(self.inc_config), self.inc_score\n    return self.inc_config, self.inc_score\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB","title":"<code>DEHB(cs=None, f=None, dimensions=None, mutation_factor=0.5, crossover_prob=0.5, strategy='rand1_bin', min_fidelity=None, max_fidelity=None, eta=3, min_clip=None, max_clip=None, configspace=True, boundary_fix_type='random', max_age=np.inf, n_workers=None, client=None, async_strategy='immediate', **kwargs)</code>","text":"<p>             Bases: <code>DEHBBase</code></p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=0.5,\n             crossover_prob=0.5, strategy='rand1_bin', min_fidelity=None,\n             max_fidelity=None, eta=3, min_clip=None, max_clip=None, configspace=True,\n             boundary_fix_type='random', max_age=np.inf, n_workers=None, client=None,\n             async_strategy=\"immediate\", **kwargs):\n    super().__init__(cs=cs, f=f, dimensions=dimensions, mutation_factor=mutation_factor,\n                     crossover_prob=crossover_prob, strategy=strategy, min_fidelity=min_fidelity,\n                     max_fidelity=max_fidelity, eta=eta, min_clip=min_clip, max_clip=max_clip,\n                     configspace=configspace, boundary_fix_type=boundary_fix_type,\n                     max_age=max_age, **kwargs)\n    self.de_params.update({\"async_strategy\": async_strategy})\n    self.iteration_counter = -1\n    self.de = {}\n    self._max_pop_size = None\n    self.active_brackets = []  # list of SHBracketManager objects\n    self.traj = []\n    self.runtime = []\n    self.history = []\n    self.start = None\n\n    # Dask variables\n    if n_workers is None and client is None:\n        raise ValueError(\"Need to specify either 'n_workers'(&gt;0) or 'client' (a Dask client)!\")\n    if client is not None and isinstance(client, Client):\n        self.client = client\n        self.n_workers = len(client.ncores())\n    else:\n        self.n_workers = n_workers\n        if self.n_workers &gt; 1:\n            self.client = Client(\n                n_workers=self.n_workers, processes=True, threads_per_worker=1, scheduler_port=0\n            )  # port 0 makes Dask select a random free port\n        else:\n            self.client = None\n    self.futures = []\n    self.shared_data = None\n\n    # Initializing DE subpopulations\n    self._get_pop_sizes()\n    self._init_subpop()\n\n    # Misc.\n    self.available_gpus = None\n    self.gpu_usage = None\n    self.single_node_with_gpus = None\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.__getstate__","title":"<code>__getstate__()</code>","text":"<p>Allows the object to picklable while having Dask client as a class attribute.</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def __getstate__(self):\n    \"\"\" Allows the object to picklable while having Dask client as a class attribute.\n    \"\"\"\n    d = dict(self.__dict__)\n    d[\"client\"] = None  # hack to allow Dask client to be a class attribute\n    d[\"logger\"] = None  # hack to allow logger object to be a class attribute\n    return d\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.__del__","title":"<code>__del__()</code>","text":"<p>Ensures a clean kill of the Dask client and frees up a port.</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def __del__(self):\n    \"\"\" Ensures a clean kill of the Dask client and frees up a port.\n    \"\"\"\n    if hasattr(self, \"client\") and isinstance(self, Client):\n        self.client.close()\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.distribute_gpus","title":"<code>distribute_gpus()</code>","text":"<p>Function to create a GPU usage tracker dict.</p> <p>The idea is to extract the exact GPU device IDs available. During job submission, each submitted job is given a preference of a GPU device ID based on the GPU device with the least number of active running jobs. On retrieval of the result, this gpu usage dict is updated for the device ID that the finished job was mapped to.</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def distribute_gpus(self):\n    \"\"\" Function to create a GPU usage tracker dict.\n\n    The idea is to extract the exact GPU device IDs available. During job submission, each\n    submitted job is given a preference of a GPU device ID based on the GPU device with the\n    least number of active running jobs. On retrieval of the result, this gpu usage dict is\n    updated for the device ID that the finished job was mapped to.\n    \"\"\"\n    try:\n        available_gpus = os.environ[\"CUDA_VISIBLE_DEVICES\"]\n        available_gpus = available_gpus.strip().split(\",\")\n        self.available_gpus = [int(_id) for _id in available_gpus]\n    except KeyError as e:\n        print(\"Unable to find valid GPU devices. \"\n              \"Environment variable {} not visible!\".format(str(e)))\n        self.available_gpus = []\n    self.gpu_usage = dict()\n    for _id in self.available_gpus:\n        self.gpu_usage[_id] = 0\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.clean_inactive_brackets","title":"<code>clean_inactive_brackets()</code>","text":"<p>Removes brackets from the active list if it is done as communicated by Bracket Manager</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def clean_inactive_brackets(self):\n    \"\"\" Removes brackets from the active list if it is done as communicated by Bracket Manager\n    \"\"\"\n    if len(self.active_brackets) == 0:\n        return\n    self.active_brackets = [\n        bracket for bracket in self.active_brackets if ~bracket.is_bracket_done()\n    ]\n    return\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.is_worker_available","title":"<code>is_worker_available(verbose=False)</code>","text":"<p>Checks if at least one worker is available to run a job</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def is_worker_available(self, verbose=False):\n    \"\"\" Checks if at least one worker is available to run a job\n    \"\"\"\n    if self.n_workers == 1 or self.client is None or not isinstance(self.client, Client):\n        # in the synchronous case, one worker is always available\n        return True\n    # checks the absolute number of workers mapped to the client scheduler\n    # client.ncores() should return a dict with the keys as unique addresses to these workers\n    # treating the number of available workers in this manner\n    workers = self._get_worker_count()  # len(self.client.ncores())\n    if len(self.futures) &gt;= workers:\n        # pause/wait if active worker count greater allocated workers\n        return False\n    return True\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.ask","title":"<code>ask(n_configs=1)</code>","text":"<p>Get the next configuration to run from the optimizer.</p> <p>The retrieved configuration can then be evaluated by the user. After evaluation use <code>tell</code> to report the results back to the optimizer. For more information, please refer to the description of <code>tell</code>.</p> PARAMETER  DESCRIPTION <code>n_configs</code> <p>Number of configs to ask for. Defaults to 1.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> RETURNS DESCRIPTION <p>dict or list of dict: Job info(s) of next configuration to evaluate.</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def ask(self, n_configs: int=1):\n    \"\"\"Get the next configuration to run from the optimizer.\n\n    The retrieved configuration can then be evaluated by the user.\n    After evaluation use `tell` to report the results back to the optimizer.\n    For more information, please refer to the description of `tell`.\n\n    Args:\n        n_configs (int, optional): Number of configs to ask for. Defaults to 1.\n\n    Returns:\n        dict or list of dict: Job info(s) of next configuration to evaluate.\n    \"\"\"\n    if n_configs == 1:\n        return self._get_next_job()\n\n    jobs = []\n    for _ in range(n_configs):\n        jobs.append(self._get_next_job())\n    return jobs\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.submit_job","title":"<code>submit_job(job_info, **kwargs)</code>","text":"<p>Asks a free worker to run the objective function on config and fidelity</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def submit_job(self, job_info, **kwargs):\n    \"\"\" Asks a free worker to run the objective function on config and fidelity\n    \"\"\"\n    job_info[\"kwargs\"] = self.shared_data if self.shared_data is not None else kwargs\n    # submit to Dask client\n    if self.n_workers &gt; 1 or isinstance(self.client, Client):\n        if self.single_node_with_gpus:\n            # managing GPU allocation for the job to be submitted\n            job_info.update({\"gpu_devices\": self._get_gpu_id_with_low_load()})\n        self.futures.append(\n            self.client.submit(self._f_objective, job_info)\n        )\n    else:\n        # skipping scheduling to Dask worker to avoid added overheads in the synchronous case\n        self.futures.append(self._f_objective(job_info))\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.tell","title":"<code>tell(job_info, result)</code>","text":"<p>Feed a result back to the optimizer.</p> <p>In order to correctly interpret the results, the <code>job_info</code> dict, retrieved by <code>ask</code>, has to be given. Moreover, the <code>result</code> dict has to contain the keys <code>fitness</code> and <code>cost</code>. It is also possible to add the field <code>info</code> to the <code>result</code> in order to store additional, user-specific information.</p> PARAMETER  DESCRIPTION <code>job_info</code> <p>Job info returned by ask().</p> <p> TYPE: <code>dict</code> </p> <code>result</code> <p>Result dictionary with mandatory keys <code>fitness</code> and <code>cost</code>.</p> <p> TYPE: <code>dict</code> </p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def tell(self, job_info: dict, result: dict):\n    \"\"\"Feed a result back to the optimizer.\n\n    In order to correctly interpret the results, the `job_info` dict, retrieved by `ask`,\n    has to be given. Moreover, the `result` dict has to contain the keys `fitness` and `cost`.\n    It is also possible to add the field `info` to the `result` in order to store additional,\n    user-specific information.\n\n    Args:\n        job_info (dict): Job info returned by ask().\n        result (dict): Result dictionary with mandatory keys `fitness` and `cost`.\n    \"\"\"\n    # update bracket information\n    fitness, cost = result[\"fitness\"], result[\"cost\"]\n    info = result[\"info\"] if \"info\" in result else dict()\n    fidelity, parent_id = job_info[\"fidelity\"], job_info[\"parent_id\"]\n    config, config_id = job_info[\"config\"], job_info[\"config_id\"]\n    bracket_id = job_info[\"bracket_id\"]\n    for bracket in self.active_brackets:\n        if bracket.bracket_id == bracket_id:\n            # bracket job complete\n            bracket.complete_job(fidelity)  # IMPORTANT to perform synchronous SH\n\n    self.config_repository.tell_result(config_id, fidelity, fitness, cost, info)\n\n    # get hypercube representation from config repo\n    if self.configspace:\n        config = self.config_repository.get(config_id)\n\n    # carry out DE selection\n    if fitness &lt;= self.de[fidelity].fitness[parent_id]:\n        self.de[fidelity].population[parent_id] = config\n        self.de[fidelity].population_ids[parent_id] = config_id\n        self.de[fidelity].fitness[parent_id] = fitness\n    # updating incumbents\n    if self.de[fidelity].fitness[parent_id] &lt; self.inc_score:\n        self._update_incumbents(\n            config=self.de[fidelity].population[parent_id],\n            score=self.de[fidelity].fitness[parent_id],\n            info=info\n        )\n    # book-keeping\n    self._update_trackers(\n        traj=self.inc_score, runtime=cost, history=(\n            config.tolist(), float(fitness), float(cost), float(fidelity), info\n        )\n    )\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.run","title":"<code>run(fevals=None, brackets=None, total_cost=None, single_node_with_gpus=False, verbose=False, debug=False, save_intermediate=True, save_history=True, name=None, **kwargs)</code>","text":"<p>Main interface to run optimization by DEHB</p> <p>This function waits on workers and if a worker is free, asks for a configuration and a fidelity to evaluate on and submits it to the worker. In each loop, it checks if a job is complete, fetches the results, carries the necessary processing of it asynchronously to the worker computations.</p> <p>The duration of the DEHB run can be controlled by specifying one of 3 parameters. If more than one are specified, DEHB selects only one in the priority order (high to low): 1) Number of function evaluations (fevals) 2) Number of Successive Halving brackets run under Hyperband (brackets) 3) Total computational cost (in seconds) aggregated by all function evaluations (total_cost)</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>@logger.catch\ndef run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus=False,\n        verbose=False, debug=False, save_intermediate=True, save_history=True, name=None, **kwargs):\n    \"\"\" Main interface to run optimization by DEHB\n\n    This function waits on workers and if a worker is free, asks for a configuration and a\n    fidelity to evaluate on and submits it to the worker. In each loop, it checks if a job\n    is complete, fetches the results, carries the necessary processing of it asynchronously\n    to the worker computations.\n\n    The duration of the DEHB run can be controlled by specifying one of 3 parameters. If more\n    than one are specified, DEHB selects only one in the priority order (high to low):\n    1) Number of function evaluations (fevals)\n    2) Number of Successive Halving brackets run under Hyperband (brackets)\n    3) Total computational cost (in seconds) aggregated by all function evaluations (total_cost)\n    \"\"\"\n    # check if run has already been called before\n    if self.start is not None:\n        logger.warning(\"DEHB has already been run. Calling 'run' twice could lead to unintended\"\n                       + \" behavior. Please restart DEHB with an increased compute budget\"\n                       + \" instead of calling 'run' twice.\")\n\n    # checks if a Dask client exists\n    if len(kwargs) &gt; 0 and self.n_workers &gt; 1 and isinstance(self.client, Client):\n        # broadcasts all additional data passed as **kwargs to all client workers\n        # this reduces overload in the client-worker communication by not having to\n        # serialize the redundant data used by all workers for every job\n        self.shared_data = self.client.scatter(kwargs, broadcast=True)\n\n    # allows each worker to be mapped to a different GPU when running on a single node\n    # where all available GPUs are accessible\n    self.single_node_with_gpus = single_node_with_gpus\n    if self.single_node_with_gpus:\n        self.distribute_gpus()\n\n    self.start = self.start = time.time()\n    fevals, brackets = self._adjust_budgets(fevals, brackets)\n    if verbose:\n        print(\"\\nLogging at {} for optimization starting at {}\\n\".format(\n            os.path.join(os.getcwd(), self.log_filename),\n            time.strftime(\"%x %X %Z\", time.localtime(self.start))\n        ))\n    if debug:\n        logger.configure(handlers=[{\"sink\": sys.stdout}])\n    while True:\n        if self._is_run_budget_exhausted(fevals, brackets, total_cost):\n            break\n        if self.is_worker_available():\n            job_info = self.ask()\n            if brackets is not None and job_info[\"bracket_id\"] &gt;= brackets:\n                # ignore submission and only collect results\n                # when brackets are chosen as run budget, an extra bracket is created\n                # since iteration_counter is incremented in ask() and then checked\n                # in _is_run_budget_exhausted(), therefore, need to skip suggestions\n                # coming from the extra allocated bracket\n                # _is_run_budget_exhausted() will not return True until all the lower brackets\n                # have finished computation and returned its results\n                pass\n            else:\n                if self.n_workers &gt; 1 or isinstance(self.client, Client):\n                    self.logger.debug(\"{}/{} worker(s) available.\".format(\n                        self._get_worker_count() - len(self.futures), self._get_worker_count()\n                    ))\n                # submits job_info to a worker for execution\n                self.submit_job(job_info, **kwargs)\n                if verbose:\n                    fidelity = job_info[\"fidelity\"]\n                    config_id = job_info[\"config_id\"]\n                    self._verbosity_runtime(fevals, brackets, total_cost)\n                    self.logger.info(\n                        \"Evaluating configuration {} with fidelity {} under \"\n                        \"bracket ID {}\".format(config_id, fidelity, job_info[\"bracket_id\"])\n                    )\n                    self.logger.info(\n                        \"Best score seen/Incumbent score: {}\".format(self.inc_score)\n                    )\n                self._verbosity_debug()\n        self._fetch_results_from_workers()\n        if save_intermediate and self.inc_config is not None:\n            self._save_incumbent(name)\n        if save_history and self.history is not None:\n            self._save_history(name)\n        self.clean_inactive_brackets()\n    # end of while\n\n    if verbose and len(self.futures) &gt; 0:\n        self.logger.info(\n            \"DEHB optimisation over! Waiting to collect results from workers running...\"\n        )\n    while len(self.futures) &gt; 0:\n        self._fetch_results_from_workers()\n        if save_intermediate and self.inc_config is not None:\n            self._save_incumbent(name)\n        if save_history and self.history is not None:\n            self._save_history(name)\n        time.sleep(0.05)  # waiting 50ms\n\n    if verbose:\n        time_taken = time.time() - self.start\n        self.logger.info(\"End of optimisation! Total duration: {}; Total fevals: {}\\n\".format(\n            time_taken, len(self.traj)\n        ))\n        self.logger.info(\"Incumbent score: {}\".format(self.inc_score))\n        self.logger.info(\"Incumbent config: \")\n        if self.configspace:\n            config = self.vector_to_configspace(self.inc_config)\n            for k, v in config.get_dictionary().items():\n                self.logger.info(\"{}: {}\".format(k, v))\n        else:\n            self.logger.info(\"{}\".format(self.inc_config))\n    self._save_incumbent(name)\n    self._save_history(name)\n    # reset waiting jobs of active bracket to allow for continuation\n    if len(self.active_brackets) &gt; 0:\n        for active_bracket in self.active_brackets:\n            active_bracket.reset_waiting_jobs()\n    return np.array(self.traj), np.array(self.runtime), np.array(self.history, dtype=object)\n</code></pre>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Welcome to DEHB's documentation!","text":""},{"location":"#introduction","title":"Introduction","text":"<p><code>dehb</code> is a python package implementing the DEHB algorithm. It offers an intuitive interface to optimize user-defined problems using DEHB.</p> <p>This documentation explains how to use <code>dehb</code> and demonstrates its features. In the following section you will be guided how to install the <code>dehb</code> package and how to use it in your own projects. Examples with more hands-on material can be found in the examples folder.</p>"},{"location":"#installation","title":"Installation","text":"<p>To start using the <code>dehb</code> package, you can install it via pip. You can either install the package right from git or install it as an editable package to modify the code and rerun your experiments:</p> <pre><code># Install from pypi\npip install dehb\n</code></pre> <p>From Source</p> <p>To install directly from from source</p> <pre><code>git clone https://github.com/automl/DEHB.git\npip install -e DEHB  # -e stands for editable, lets you modify the code and rerun things\n</code></pre>"},{"location":"#contributing","title":"Contributing","text":"<p>Please have a look at our contributing guidelines.</p>"},{"location":"#to-cite-the-paper-or-code","title":"To cite the paper or code","text":"<p>If you use DEHB in one of your research projects, please cite our paper(s): <pre><code>@inproceedings{awad-ijcai21,\n  author    = {N. Awad and N. Mallik and F. Hutter},\n  title     = {{DEHB}: Evolutionary Hyberband for Scalable, Robust and Efficient Hyperparameter Optimization},\n  pages     = {2147--2153},\n  booktitle = {Proceedings of the Thirtieth International Joint Conference on\n               Artificial Intelligence, {IJCAI-21}},\n  publisher = {ijcai.org},\n  editor    = {Z. Zhou},\n  year      = {2021}\n}\n</code></pre></p>"},{"location":"examples/00_interfacing_DEHB/","title":"Interfacing DEHB","text":"In\u00a0[1]: Copied! <pre>import time\nimport warnings\nimport numpy as np\nimport ConfigSpace\nfrom typing import Dict, Union, List\n\nwarnings.filterwarnings('ignore')\n</pre> import time import warnings import numpy as np import ConfigSpace from typing import Dict, Union, List  warnings.filterwarnings('ignore') <p>DEHB was designed to be an algorithm for Hyper Parameter Optimization (HPO). DEHB uses Differential Evolution (DE) under-the-hood as an Evolutionary Algorithm to power the black-box optimization that HPO problems pose. DE is a black-box optimization algorithm that generates candidate configurations $x$, to the black-box function $f(x)$, that is being optimized. The $x$ is evaluated by the black-box and the corresponding response $y$ is made available to the DE algorithm, which can then use this observation ($x$, $y$), along with previous such observations, to suggest a new candidate $x$ for the next evaluation.</p> <p>DEHB also uses Hyperband along with DE, to allow for cheaper approximations of the actual evaluations of $x$. Let $f(x)$ be the validation error of training a multilayer perceptron (MLP) on the complete training set. Multi-fidelity algorithms such as Hyperband, allow for cheaper approximations along a possible fidelity. For the MLP, a subset of the dataset maybe a cheaper approximation to the full data set evaluation. Whereas the fidelity can be quantifies as the fraction of the dataset used to evaluate the configuration $x$, instead of the full dataset. Such approximations can allow sneak-peek into the black-box, potentially revealing certain landscape feature of f(x), thus rendering it a gray-box and not completely opaque and black!</p> <p>The $z$ parameter is the fidelity parameter to the black-box function. If $z \\in [fidelity_{min}, fidelity_{max}]$, then $f(x, fidelity_{max})$ would be equivalent to the black-box case of $f(x)$.</p> <p></p> <p>HPO algorithms optimize such black/gray box by wrapping around this target function an interface, by which the algorithms can suggest new $x$ and also consume the result of the corresponding evaluation to store a collection of such ($x$, $y$) pairs. Therefore, to run DEHB, the most essential component required as input is the target function to optimize. Since DEHB can leverage a Hyperband, the target function interface should account for possible input of fidelity too.</p> In\u00a0[2]: Copied! <pre>def target_function(\n    x: Union[ConfigSpace.Configuration, List, np.array], \n    fidelity: Union[int, float] = None,\n    **kwargs\n) -&gt; Dict:\n    \"\"\" Target/objective function to optimize\n    \n    Parameters\n    ----------\n    x : configuration that DEHB wants to evaluate\n    fidelity : parameter determining cheaper evaluations\n    \n    Returns\n    -------\n    dict\n    \"\"\"\n    # ...\n    # write your code here\n    # ...\n    \n    # remove the code snippet below\n    start = time.time()\n    y = np.random.uniform()  # placeholder response of evaluation\n    time.sleep(fidelity)       # simulates runtime (mostly proportional to fidelity)\n    cost = time.time() - start\n    \n    # result dict passed to DE/DEHB as function evaluation output\n    result = {\n        \"fitness\": y,  # must-have key that DE/DEHB minimizes\n        \"cost\": cost,  # must-have key that associates cost/runtime \n        \"info\": dict() # optional key containing a dictionary of additional info\n    }\n    return result\n</pre> def target_function(     x: Union[ConfigSpace.Configuration, List, np.array],      fidelity: Union[int, float] = None,     **kwargs ) -&gt; Dict:     \"\"\" Target/objective function to optimize          Parameters     ----------     x : configuration that DEHB wants to evaluate     fidelity : parameter determining cheaper evaluations          Returns     -------     dict     \"\"\"     # ...     # write your code here     # ...          # remove the code snippet below     start = time.time()     y = np.random.uniform()  # placeholder response of evaluation     time.sleep(fidelity)       # simulates runtime (mostly proportional to fidelity)     cost = time.time() - start          # result dict passed to DE/DEHB as function evaluation output     result = {         \"fitness\": y,  # must-have key that DE/DEHB minimizes         \"cost\": cost,  # must-have key that associates cost/runtime          \"info\": dict() # optional key containing a dictionary of additional info     }     return result     <p>This <code>target_function</code> is the problem that needs to be solved, or the function to be optimized. The other prerequisite for this function is therefore the domain for its input $x$. In other words, the definition and constraints of the search space for DEHB.</p> <p>The DE component inside DEHB, assumes that the input domain is scaled to a unit hypercube. This is essential for effective search. If the ConfigSpace library is used to define the domain of $x$, or the parameters of the search space, DEHB can internally handle the scaling to and from the unit hypercube required for search. If ConfigSpace is not used, one needs to additionally handle the scaling of the parameters as an extra interface between DEHB and the target function (or encode it within the target function).</p> <p>For this template notebook, we will illustrate how a ConfigSpace parameter space can be created.</p> In\u00a0[3]: Copied! <pre>import ConfigSpace\n\n\ndef create_search_space():\n    # Creating a one-dimensional search space of real numbers in [3, 10]\n    cs = ConfigSpace.ConfigurationSpace()\n    cs.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter(\"x0\", lower=3, upper=10, log=False))\n    return cs\n\n\ncs = create_search_space()\nprint(cs)\n</pre> import ConfigSpace   def create_search_space():     # Creating a one-dimensional search space of real numbers in [3, 10]     cs = ConfigSpace.ConfigurationSpace()     cs.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter(\"x0\", lower=3, upper=10, log=False))     return cs   cs = create_search_space() print(cs) <pre>Configuration space object:\n  Hyperparameters:\n    x0, Type: UniformFloat, Range: [3.0, 10.0], Default: 6.5\n\n</pre> In\u00a0[4]: Copied! <pre># Finding dimensionality of search space\ndimensions = len(cs.get_hyperparameters())\nprint(dimensions)\n</pre> # Finding dimensionality of search space dimensions = len(cs.get_hyperparameters()) print(dimensions) <pre>1\n</pre> In\u00a0[5]: Copied! <pre># Sampling a random configuration\ncs.sample_configuration()\n</pre> # Sampling a random configuration cs.sample_configuration() Out[5]: <pre>Configuration(values={\n  'x0': 7.785477780551136,\n})</pre> <p>The ConfigSpace documentation can be referred to for more complicated search space creation.</p> <p>In a similar vein, for a complete gray-box definition, the fidelity domain needs to be defined too. For the earlier example of dataset fractions, the fidelity upper limit cannot clearly exceed 1, and therefore $[0.3, 1]$ is a legitimate definition for such a fidelity. In this template example, we shall simply define the lower and upper range of the fidelity as two parameters that can be input to DEHB. Given that fidelity is being used to simulate cost of runtime in our sample <code>target_function</code>, we shall use a reasonable time range as a placeholder for the fidelity in this case.</p> In\u00a0[6]: Copied! <pre>min_fidelity, max_fidelity = (0.1, 3)\n</pre> min_fidelity, max_fidelity = (0.1, 3)  <p>The above definitions are all the information that DEHB needs about a problem. We are now in a position to call upon DEHB and start running it, to tune $x$.</p> In\u00a0[7]: Copied! <pre>from dehb import DEHB\n\n\ndehb = DEHB(\n    f=target_function,\n    dimensions=dimensions,\n    cs=cs,\n    min_fidelity=min_fidelity,\n    max_fidelity=max_fidelity,\n    output_path=\"./temp\",\n    n_workers=1        # set to &gt;1 to utilize parallel workers\n)\n\n# NOTE: the other hyperparameters to DEHB have been set to certain defaults that were \n# empirically determined through related literature, ablation analysis and other experiments,\n# but can be tuned as desired\n</pre> from dehb import DEHB   dehb = DEHB(     f=target_function,     dimensions=dimensions,     cs=cs,     min_fidelity=min_fidelity,     max_fidelity=max_fidelity,     output_path=\"./temp\",     n_workers=1        # set to &gt;1 to utilize parallel workers )  # NOTE: the other hyperparameters to DEHB have been set to certain defaults that were  # empirically determined through related literature, ablation analysis and other experiments, # but can be tuned as desired In\u00a0[8]: Copied! <pre>?dehb.run\n</pre> ?dehb.run <p>DEHB allows the option of 3 different resources for its runtime budget:</p> In\u00a0[9]: Copied! <pre>_, _, _ = dehb.run(brackets=1, verbose=False, save_intermediate=True)\nprint(dehb.vector_to_configspace(dehb.inc_config))\n</pre> _, _, _ = dehb.run(brackets=1, verbose=False, save_intermediate=True) print(dehb.vector_to_configspace(dehb.inc_config)) <pre>Configuration(values={\n  'x0': 6.460117941570684,\n})\n</pre> In\u00a0[10]: Copied! <pre># allows optimization to restart from the beginning by forgetting all observations\ndehb.reset()  \n\n_, _, _ = dehb.run(fevals=20, verbose=False, save_intermediate=True)\nprint(dehb.get_incumbents())\n</pre> # allows optimization to restart from the beginning by forgetting all observations dehb.reset()    _, _, _ = dehb.run(fevals=20, verbose=False, save_intermediate=True) print(dehb.get_incumbents()) <pre>2024-02-17 18:38:08.099 | INFO     | dehb.optimizers.dehb:reset:121 - \n\nRESET at 02/17/24 18:38:08 UTC\n\n\n</pre> <pre>(Configuration(values={\n  'x0': 7.74688038121089,\n}), 0.1103633789699231)\n</pre> In\u00a0[11]: Copied! <pre># allows optimization to restart from the beginning by forgetting all observations\ndehb.reset()  \n\n_, _, _ = dehb.run(total_cost=10, verbose=False, save_intermediate=True)  # run for 10s\nprint(dehb.get_incumbents())\n</pre> # allows optimization to restart from the beginning by forgetting all observations dehb.reset()    _, _, _ = dehb.run(total_cost=10, verbose=False, save_intermediate=True)  # run for 10s print(dehb.get_incumbents()) <pre>2024-02-17 18:38:10.370 | INFO     | dehb.optimizers.dehb:reset:121 - \n\nRESET at 02/17/24 18:38:10 UTC\n\n\n</pre> <pre>(Configuration(values={\n  'x0': 4.290333531372179,\n}), 0.004440353907169903)\n</pre> <p>Each <code>dehb</code> object initialized maintains a <code>log</code> file in the <code>output_path</code> specified, where the progress and other debugging information is updated. While every alternative DEHB evaluation (and after full optimization), an <code>incumbent.json</code> file is written to disk <code>output_path</code>, with the incumbent (best seen so far) configuration and its corresponding score.</p> <p>We now rerun DEHB in parallel with 2 workers, and show that the incumbents can be retrieved in any of the following manner:</p> In\u00a0[12]: Copied! <pre>dehb = DEHB(\n    f=target_function,\n    dimensions=dimensions,\n    cs=cs,\n    min_fidelity=min_fidelity,\n    max_fidelity=max_fidelity,\n    output_path=\"./temp\",\n    n_workers=2\n)\ntrajectory, runtime, history = dehb.run(\n    total_cost=20, verbose=False,\n)\n\nprint(dehb.get_incumbents())\n</pre> dehb = DEHB(     f=target_function,     dimensions=dimensions,     cs=cs,     min_fidelity=min_fidelity,     max_fidelity=max_fidelity,     output_path=\"./temp\",     n_workers=2 ) trajectory, runtime, history = dehb.run(     total_cost=20, verbose=False, )  print(dehb.get_incumbents()) <pre>(Configuration(values={\n  'x0': 3.9501812728673467,\n}), 0.0031614215460380457)\n</pre> In\u00a0[13]: Copied! <pre>print(dehb.vector_to_configspace(dehb.inc_config))  # config as ConfigSpace configuration\n</pre> print(dehb.vector_to_configspace(dehb.inc_config))  # config as ConfigSpace configuration <pre>Configuration(values={\n  'x0': 3.9501812728673467,\n})\n</pre> In\u00a0[14]: Copied! <pre>print(trajectory[-1], dehb.inc_score)\nprint(dehb.vector_to_configspace(dehb.inc_config))\n</pre> print(trajectory[-1], dehb.inc_score) print(dehb.vector_to_configspace(dehb.inc_config)) <pre>0.0031614215460380457 0.0031614215460380457\nConfiguration(values={\n  'x0': 3.9501812728673467,\n})\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/00_interfacing_DEHB/#interfacing-dehb","title":"Interfacing DEHB\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#how-to-read-this-notebook","title":"How to read this notebook\u00b6","text":"<p>This notebook is designed to serve as a high-level, highly abstracted view of DEHB and how it can be used. The examples here are mere placeholders and only offer an interface to run DEHB on toy or actual problems.</p>"},{"location":"examples/00_interfacing_DEHB/#getting-started-with-dehb","title":"Getting started with DEHB\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#sample-interface-for-target-function-that-dehb-optimizes","title":"Sample interface for target function that DEHB optimizes\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#defining-a-sample-search-space","title":"Defining a sample search space\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#defining-fidelity-range-for-the-target-function","title":"Defining fidelity range for the target function\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#instantiating-and-running-dehb","title":"Instantiating and running DEHB\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#1-running-dehb-for-a-certain-number-of-successive-halving-brackets","title":"1) Running DEHB for a certain number of (successive halving) brackets\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#2-running-dehb-for-total-number-of-function-evaluations","title":"2) Running DEHB for total number of function evaluations\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#3-running-dehb-for-total-amount-of-wallclock-time","title":"3) Running DEHB for total amount of wallclock time\u00b6","text":""},{"location":"examples/00_interfacing_DEHB/#conclusion","title":"Conclusion\u00b6","text":"<p>As detailed above, the problem definition needs to be input to DEHB as the following information:</p> <ul> <li>the target_function (<code>f</code>) that is the primary black-box function to optimize</li> <li>the fidelity range of <code>min_fidelity</code> and <code>max_fidelity</code> that allows the cheaper, faster gray-box optimization of <code>f</code></li> <li>the search space or the input domain of the function <code>f</code>, that can be represented as a <code>ConfigSpace</code> object and passed to DEHB at initialization</li> </ul> <p>Following which, DEHB can be run for any amount of practical real-world budget. It can be run for either:</p> <ul> <li>a total amount of actual wallclock time, example one day (~86400 seconds), or</li> <li>a total number of function evaluations, or the number of times we want the black-box to be accessed for evaluation, across all fidelities</li> <li>the total number of brackets we want to run the DEHB algorithm for</li> </ul> <p>DEHB will terminate once its chosen runtime budget is exhausted, and report the incumbent found. DEHB, as an anytime algorithm, constantly writes to disk a lightweight <code>json</code> file with the best found configuration and its score seen till that point.</p>"},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/","title":"Optimizing RandomForest using DEHB","text":"In\u00a0[1]: Copied! <pre>import time\nimport numpy as np\nimport warnings\n\n\nseed = 123\nnp.random.seed(seed)\nwarnings.filterwarnings('ignore')\n</pre> import time import numpy as np import warnings   seed = 123 np.random.seed(seed) warnings.filterwarnings('ignore') <p>The problem defined here is to optimize a Random Forest model, on any given dataset, using DEHB. The hyperparameters chosen to be optimized are:</p> <ul> <li><code>max_depth</code></li> <li><code>min_samples_split</code></li> <li><code>max_features</code></li> <li><code>min_samples_leaf</code> while the <code>n_estimators</code> hyperparameter to the Random Forest is chosen to be a fidelity parameter instead. Lesser number of trees ($&lt;10$) in the Random Forest may not allow adequate ensembling for the grouped prediction to be significantly better than the individual tree predictions. Whereas a large number of trees (~$100$) often give accurate predictions but is naturally slower to train and predict on account of more trees to train. Therefore, a smaller <code>n_estimators</code> can be used as a cheaper approximation of the actual fidelity of <code>n_estimators=100</code>.</li> </ul> In\u00a0[2]: Copied! <pre>min_fidelity, max_fidelity = 2, 50\n</pre> min_fidelity, max_fidelity = 2, 50 <p>For the remaining $4$ hyperparameters, the search space can be created as a <code>ConfigSpace</code> object, with the domain of individual parameters defined.</p> In\u00a0[3]: Copied! <pre>import ConfigSpace as CS\n\n\ndef create_search_space(seed=123):\n    \"\"\"Parameter space to be optimized --- contains the hyperparameters\n    \"\"\"\n    cs = CS.ConfigurationSpace(seed=seed)\n\n    cs.add_hyperparameters([\n        CS.UniformIntegerHyperparameter(\n            'max_depth', lower=1, upper=15, default_value=2, log=False\n        ),\n        CS.UniformIntegerHyperparameter(\n            'min_samples_split', lower=2, upper=128, default_value=2, log=True\n        ),\n        CS.UniformFloatHyperparameter(\n            'max_features', lower=0.1, upper=0.9, default_value=0.5, log=False\n        ),\n        CS.UniformIntegerHyperparameter(\n            'min_samples_leaf', lower=1, upper=64, default_value=1, log=True\n        ),\n    ])\n    return cs\n</pre> import ConfigSpace as CS   def create_search_space(seed=123):     \"\"\"Parameter space to be optimized --- contains the hyperparameters     \"\"\"     cs = CS.ConfigurationSpace(seed=seed)      cs.add_hyperparameters([         CS.UniformIntegerHyperparameter(             'max_depth', lower=1, upper=15, default_value=2, log=False         ),         CS.UniformIntegerHyperparameter(             'min_samples_split', lower=2, upper=128, default_value=2, log=True         ),         CS.UniformFloatHyperparameter(             'max_features', lower=0.1, upper=0.9, default_value=0.5, log=False         ),         CS.UniformIntegerHyperparameter(             'min_samples_leaf', lower=1, upper=64, default_value=1, log=True         ),     ])     return cs In\u00a0[4]: Copied! <pre>cs = create_search_space(seed)\nprint(cs)\n</pre> cs = create_search_space(seed) print(cs) <pre>Configuration space object:\n  Hyperparameters:\n    max_depth, Type: UniformInteger, Range: [1, 15], Default: 2\n    max_features, Type: UniformFloat, Range: [0.1, 0.9], Default: 0.5\n    min_samples_leaf, Type: UniformInteger, Range: [1, 64], Default: 1, on log-scale\n    min_samples_split, Type: UniformInteger, Range: [2, 128], Default: 2, on log-scale\n\n</pre> In\u00a0[5]: Copied! <pre>dimensions = len(cs.get_hyperparameters())\nprint(\"Dimensionality of search space: {}\".format(dimensions))\n</pre> dimensions = len(cs.get_hyperparameters()) print(\"Dimensionality of search space: {}\".format(dimensions)) <pre>Dimensionality of search space: 4\n</pre> <p>Now the primary black/gray-box interface to the Random Forest model needs to be built for DEHB to query. As given in the <code>00_interfacing_DEHB</code> notebook, this function will have a signature akin to: <code>target_function(config, fidelity)</code>, and return a two-element tuple of the <code>score</code> and <code>cost</code>. It must be noted that DEHB minimizes and therefore the <code>score</code> being returned by this <code>target_function</code> should account for it.</p> <p>In this example, the target function trains a Random Forest model on a dataset. We load a dataset here and maintain a fixed, train-validation-test split for one complete run. Multiple DEHB runs can therefore optimize on the same validation split, and evaluate final performance on the same test set.</p> In\u00a0[6]: Copied! <pre>from sklearn.datasets import load_iris, load_digits, load_wine\n\n\nclassification = {\"iris\": load_iris, \"digits\": load_digits, \"wine\": load_wine}\n</pre> from sklearn.datasets import load_iris, load_digits, load_wine   classification = {\"iris\": load_iris, \"digits\": load_digits, \"wine\": load_wine} In\u00a0[7]: Copied! <pre>from sklearn.model_selection import train_test_split\n\n\ndef prepare_dataset(model_type=\"classification\", dataset=None):\n\n    if model_type == \"classification\":\n        if dataset is None:\n            dataset = np.random.choice(list(classification.keys())) \n        _data = classification[dataset]()\n    else:\n        if dataset is None:\n            dataset = np.random.choice(list(regression.keys()))\n        _data = regression[dataset]()\n\n    train_X, rest_X, train_y, rest_y = train_test_split(\n      _data.get(\"data\"), \n      _data.get(\"target\"), \n      train_size=0.7, \n      shuffle=True, \n      random_state=seed\n    )\n    \n    # 10% test and 20% validation data\n    valid_X, test_X, valid_y, test_y = train_test_split(\n      rest_X, rest_y,\n      test_size=0.3333, \n      shuffle=True, \n      random_state=seed\n    )\n    return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset\n</pre> from sklearn.model_selection import train_test_split   def prepare_dataset(model_type=\"classification\", dataset=None):      if model_type == \"classification\":         if dataset is None:             dataset = np.random.choice(list(classification.keys()))          _data = classification[dataset]()     else:         if dataset is None:             dataset = np.random.choice(list(regression.keys()))         _data = regression[dataset]()      train_X, rest_X, train_y, rest_y = train_test_split(       _data.get(\"data\"),        _data.get(\"target\"),        train_size=0.7,        shuffle=True,        random_state=seed     )          # 10% test and 20% validation data     valid_X, test_X, valid_y, test_y = train_test_split(       rest_X, rest_y,       test_size=0.3333,        shuffle=True,        random_state=seed     )     return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset In\u00a0[8]: Copied! <pre>train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\\n    prepare_dataset(model_type=\"classification\")\n\nprint(dataset)\nprint(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(\n    train_X.shape, valid_X.shape, test_X.shape\n))\n</pre> train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\     prepare_dataset(model_type=\"classification\")  print(dataset) print(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(     train_X.shape, valid_X.shape, test_X.shape )) <pre>wine\nTrain size: (124, 13)\nValid size: (36, 13)\nTest size: (18, 13)\n</pre> In\u00a0[9]: Copied! <pre>from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, make_scorer\n\n\naccuracy_scorer = make_scorer(accuracy_score)\n</pre> from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score, make_scorer   accuracy_scorer = make_scorer(accuracy_score) In\u00a0[10]: Copied! <pre>def target_function(config, fidelity, **kwargs):\n    # Extracting support information\n    seed = kwargs[\"seed\"]\n    train_X = kwargs[\"train_X\"]\n    train_y = kwargs[\"train_y\"]\n    valid_X = kwargs[\"valid_X\"]\n    valid_y = kwargs[\"valid_y\"]\n    max_fidelity = kwargs[\"max_fidelity\"]\n    \n    if fidelity is None:\n        fidelity = max_fidelity\n    \n    start = time.time()\n    # Building model \n    model = RandomForestClassifier(\n        **config.get_dictionary(),\n        n_estimators=int(fidelity),\n        bootstrap=True,\n        random_state=seed,\n    )\n    # Training the model on the complete training set\n    model.fit(train_X, train_y)\n    \n    # Evaluating the model on the validation set\n    valid_accuracy = accuracy_scorer(model, valid_X, valid_y)\n    cost = time.time() - start\n    \n    # Evaluating the model on the test set as additional info\n    test_accuracy = accuracy_scorer(model, test_X, test_y)\n    \n    result = {\n        \"fitness\": -valid_accuracy,  # DE/DEHB minimizes\n        \"cost\": cost,\n        \"info\": {\n            \"test_score\": test_accuracy,\n            \"fidelity\": fidelity\n        }\n    }\n    return result\n</pre> def target_function(config, fidelity, **kwargs):     # Extracting support information     seed = kwargs[\"seed\"]     train_X = kwargs[\"train_X\"]     train_y = kwargs[\"train_y\"]     valid_X = kwargs[\"valid_X\"]     valid_y = kwargs[\"valid_y\"]     max_fidelity = kwargs[\"max_fidelity\"]          if fidelity is None:         fidelity = max_fidelity          start = time.time()     # Building model      model = RandomForestClassifier(         **config.get_dictionary(),         n_estimators=int(fidelity),         bootstrap=True,         random_state=seed,     )     # Training the model on the complete training set     model.fit(train_X, train_y)          # Evaluating the model on the validation set     valid_accuracy = accuracy_scorer(model, valid_X, valid_y)     cost = time.time() - start          # Evaluating the model on the test set as additional info     test_accuracy = accuracy_scorer(model, test_X, test_y)          result = {         \"fitness\": -valid_accuracy,  # DE/DEHB minimizes         \"cost\": cost,         \"info\": {             \"test_score\": test_accuracy,             \"fidelity\": fidelity         }     }     return result <p>We now have all components to define the problem to be optimized. DEHB can be initialized using all these information.</p> In\u00a0[11]: Copied! <pre>from dehb import DEHB\n\ndehb = DEHB(\n    f=target_function, \n    cs=cs, \n    dimensions=dimensions, \n    min_fidelity=min_fidelity, \n    max_fidelity=max_fidelity,\n    n_workers=1,\n    output_path=\"./temp\"\n)\n</pre> from dehb import DEHB  dehb = DEHB(     f=target_function,      cs=cs,      dimensions=dimensions,      min_fidelity=min_fidelity,      max_fidelity=max_fidelity,     n_workers=1,     output_path=\"./temp\" ) In\u00a0[12]: Copied! <pre>trajectory, runtime, history = dehb.run(\n    total_cost=10,\n    verbose=False,\n    save_intermediate=False,\n    # parameters expected as **kwargs in target_function is passed here\n    seed=123,\n    train_X=train_X,\n    train_y=train_y,\n    valid_X=valid_X,\n    valid_y=valid_y,\n    max_fidelity=dehb.max_fidelity\n)\n</pre> trajectory, runtime, history = dehb.run(     total_cost=10,     verbose=False,     save_intermediate=False,     # parameters expected as **kwargs in target_function is passed here     seed=123,     train_X=train_X,     train_y=train_y,     valid_X=valid_X,     valid_y=valid_y,     max_fidelity=dehb.max_fidelity ) In\u00a0[13]: Copied! <pre>print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")\n\n# Last recorded function evaluation\nlast_eval = history[-1]\nconfig, score, cost, fidelity, _info = last_eval\n\nprint(\"Last evaluated configuration, \")\nprint(dehb.vector_to_configspace(config), end=\"\")\nprint(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"\n      \"took {:.3f} seconds to run.\".format(score, fidelity, cost))\nprint(\"The additional info attached: {}\".format(_info))\n</pre> print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")  # Last recorded function evaluation last_eval = history[-1] config, score, cost, fidelity, _info = last_eval  print(\"Last evaluated configuration, \") print(dehb.vector_to_configspace(config), end=\"\") print(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"       \"took {:.3f} seconds to run.\".format(score, fidelity, cost)) print(\"The additional info attached: {}\".format(_info)) <pre>416 416 416\n\nLast evaluated configuration, \nConfiguration(values={\n  'max_depth': 6,\n  'max_features': 0.5908227041967594,\n  'min_samples_leaf': 2,\n  'min_samples_split': 5,\n})got a score of -1.0, was evaluated at a fidelity of 16.67 and took 0.018 seconds to run.\nThe additional info attached: {'test_score': 1.0, 'fidelity': 16.666666666666664}\n</pre> <p>Below, we let DEHB optimize for $5$ different runs. The <code>reset()</code> allows DEHB to begin optimization from the beginning by cleaning all history and starting with random samples. Each run of DEHB optimization is for just $10$ seconds as set by <code>total_cost=10</code>. We then report the mean and the standard deviation of the best score seen across these $5$ runs.</p> In\u00a0[14]: Copied! <pre>runs = 5\n\nbest_config_list = []\n\nfor i in range(runs):\n    # Resetting to begin optimization again\n    dehb.reset()\n    # Executing a run of DEHB optimization lasting for 10s\n    trajectory, runtime, history = dehb.run(\n        total_cost=10,\n        verbose=False,\n        save_intermediate=False,\n        seed=123,\n        train_X=train_X,\n        train_y=train_y,\n        valid_X=valid_X,\n        valid_y=valid_y,\n        max_fidelity=dehb.max_fidelity\n    )\n    best_config = dehb.vector_to_configspace(dehb.inc_config)\n    \n    # Creating a model using the best configuration found\n    model = RandomForestClassifier(\n        **best_config.get_dictionary(),\n        n_estimators=int(max_fidelity),\n        bootstrap=True,\n        random_state=seed,\n    )\n    # Training the model on the complete training set\n    model.fit(\n        np.concatenate((train_X, valid_X)), \n        np.concatenate((train_y, valid_y))\n    )\n    # Evaluating the model on the held-out test set\n    test_accuracy = accuracy_scorer(model, test_X, test_y)\n    best_config_list.append((best_config, test_accuracy))\n</pre> runs = 5  best_config_list = []  for i in range(runs):     # Resetting to begin optimization again     dehb.reset()     # Executing a run of DEHB optimization lasting for 10s     trajectory, runtime, history = dehb.run(         total_cost=10,         verbose=False,         save_intermediate=False,         seed=123,         train_X=train_X,         train_y=train_y,         valid_X=valid_X,         valid_y=valid_y,         max_fidelity=dehb.max_fidelity     )     best_config = dehb.vector_to_configspace(dehb.inc_config)          # Creating a model using the best configuration found     model = RandomForestClassifier(         **best_config.get_dictionary(),         n_estimators=int(max_fidelity),         bootstrap=True,         random_state=seed,     )     # Training the model on the complete training set     model.fit(         np.concatenate((train_X, valid_X)),          np.concatenate((train_y, valid_y))     )     # Evaluating the model on the held-out test set     test_accuracy = accuracy_scorer(model, test_X, test_y)     best_config_list.append((best_config, test_accuracy)) <pre>2024-02-17 18:38:59.841 | INFO     | dehb.optimizers.dehb:reset:121 - \n\nRESET at 02/17/24 18:38:59 UTC\n\n\n</pre> <pre>2024-02-17 18:39:09.921 | INFO     | dehb.optimizers.dehb:reset:121 - \n\nRESET at 02/17/24 18:39:09 UTC\n\n\n</pre> <pre>2024-02-17 18:39:19.994 | INFO     | dehb.optimizers.dehb:reset:121 - \n\nRESET at 02/17/24 18:39:19 UTC\n\n\n</pre> <pre>2024-02-17 18:39:30.078 | INFO     | dehb.optimizers.dehb:reset:121 - \n\nRESET at 02/17/24 18:39:30 UTC\n\n\n</pre> <pre>2024-02-17 18:39:40.139 | INFO     | dehb.optimizers.dehb:reset:121 - \n\nRESET at 02/17/24 18:39:40 UTC\n\n\n</pre> In\u00a0[15]: Copied! <pre>print(\"Mean score across trials: \", np.mean([score for _, score in best_config_list]))\nprint(\"Std. dev. of score across trials: \", np.std([score for _, score in best_config_list]))\n</pre> print(\"Mean score across trials: \", np.mean([score for _, score in best_config_list])) print(\"Std. dev. of score across trials: \", np.std([score for _, score in best_config_list])) <pre>Mean score across trials:  1.0\nStd. dev. of score across trials:  0.0\n</pre> In\u00a0[16]: Copied! <pre>for config, score in best_config_list:\n    print(\"{} got an accuracy of {} on the test set.\".format(config, score))\n    print()\n</pre> for config, score in best_config_list:     print(\"{} got an accuracy of {} on the test set.\".format(config, score))     print() <pre>Configuration(values={\n  'max_depth': 10,\n  'max_features': 0.6478776855196833,\n  'min_samples_leaf': 1,\n  'min_samples_split': 23,\n}) got an accuracy of 1.0 on the test set.\n\nConfiguration(values={\n  'max_depth': 4,\n  'max_features': 0.7985430014261752,\n  'min_samples_leaf': 1,\n  'min_samples_split': 14,\n}) got an accuracy of 1.0 on the test set.\n\nConfiguration(values={\n  'max_depth': 12,\n  'max_features': 0.5994266493364799,\n  'min_samples_leaf': 3,\n  'min_samples_split': 3,\n}) got an accuracy of 1.0 on the test set.\n\nConfiguration(values={\n  'max_depth': 11,\n  'max_features': 0.3430438209075988,\n  'min_samples_leaf': 2,\n  'min_samples_split': 17,\n}) got an accuracy of 1.0 on the test set.\n\nConfiguration(values={\n  'max_depth': 6,\n  'max_features': 0.5899184020756805,\n  'min_samples_leaf': 11,\n  'min_samples_split': 8,\n}) got an accuracy of 1.0 on the test set.\n\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#optimizing-randomforest-using-dehb","title":"Optimizing RandomForest using DEHB\u00b6","text":"<p>This notebook aims to build on the template from <code>00_interfacing_DEHB</code> and use it on an actual problem, to optimize the hyperparameters of a Random Forest model, for a dataset.</p> <p>Additional requirements:</p> <ul> <li>scikit-learn&gt;=0.24</li> </ul>"},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#defining-fidelity-range","title":"Defining fidelity range\u00b6","text":""},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#creating-search-space","title":"Creating search space\u00b6","text":""},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#creating-target-function-to-optimize-2-parts","title":"Creating target function to optimize (2 parts)\u00b6","text":""},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#1-preparing-dataset-and-splits","title":"1 ) Preparing dataset and splits\u00b6","text":""},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#2-function-interface-with-dehb","title":"2 ) Function interface with DEHB\u00b6","text":""},{"location":"examples/01.1_Optimizing_RandomForest_using_DEHB/#running-dehb","title":"Running DEHB\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/","title":"Using the Ask &amp; Tell interface","text":"In\u00a0[1]: Copied! <pre>import time\nimport numpy as np\nimport warnings\n\n\nseed = 123\nnp.random.seed(seed)\nwarnings.filterwarnings('ignore')\n</pre> import time import numpy as np import warnings   seed = 123 np.random.seed(seed) warnings.filterwarnings('ignore') <p>The problem defined here is to optimize a Random Forest model, on any given dataset, using DEHB. The hyperparameters chosen to be optimized are:</p> <ul> <li><code>max_depth</code></li> <li><code>min_samples_split</code></li> <li><code>max_features</code></li> <li><code>min_samples_leaf</code> while the <code>n_estimators</code> hyperparameter to the Random Forest is chosen to be a fidelity parameter instead. Lesser number of trees ($&lt;10$) in the Random Forest may not allow adequate ensembling for the grouped prediction to be significantly better than the individual tree predictions. Whereas a large number of trees (~$100$) often give accurate predictions but is naturally slower to train and predict on account of more trees to train. Therefore, a smaller <code>n_estimators</code> can be used as a cheaper approximation of the actual fidelity of <code>n_estimators=100</code>.</li> </ul> In\u00a0[2]: Copied! <pre>min_fidelity, max_fidelity = 2, 50\n</pre> min_fidelity, max_fidelity = 2, 50 <p>For the remaining $4$ hyperparameters, the search space can be created as a <code>ConfigSpace</code> object, with the domain of individual parameters defined.</p> In\u00a0[3]: Copied! <pre>import ConfigSpace as CS\n\n\ndef create_search_space(seed=123):\n    \"\"\"Parameter space to be optimized --- contains the hyperparameters\n    \"\"\"\n    cs = CS.ConfigurationSpace(seed=seed)\n\n    cs.add_hyperparameters([\n        CS.UniformIntegerHyperparameter(\n            'max_depth', lower=1, upper=15, default_value=2, log=False\n        ),\n        CS.UniformIntegerHyperparameter(\n            'min_samples_split', lower=2, upper=128, default_value=2, log=True\n        ),\n        CS.UniformFloatHyperparameter(\n            'max_features', lower=0.1, upper=0.9, default_value=0.5, log=False\n        ),\n        CS.UniformIntegerHyperparameter(\n            'min_samples_leaf', lower=1, upper=64, default_value=1, log=True\n        ),\n    ])\n    return cs\n</pre> import ConfigSpace as CS   def create_search_space(seed=123):     \"\"\"Parameter space to be optimized --- contains the hyperparameters     \"\"\"     cs = CS.ConfigurationSpace(seed=seed)      cs.add_hyperparameters([         CS.UniformIntegerHyperparameter(             'max_depth', lower=1, upper=15, default_value=2, log=False         ),         CS.UniformIntegerHyperparameter(             'min_samples_split', lower=2, upper=128, default_value=2, log=True         ),         CS.UniformFloatHyperparameter(             'max_features', lower=0.1, upper=0.9, default_value=0.5, log=False         ),         CS.UniformIntegerHyperparameter(             'min_samples_leaf', lower=1, upper=64, default_value=1, log=True         ),     ])     return cs In\u00a0[4]: Copied! <pre>cs = create_search_space(seed)\nprint(cs)\n</pre> cs = create_search_space(seed) print(cs) <pre>Configuration space object:\n  Hyperparameters:\n    max_depth, Type: UniformInteger, Range: [1, 15], Default: 2\n    max_features, Type: UniformFloat, Range: [0.1, 0.9], Default: 0.5\n    min_samples_leaf, Type: UniformInteger, Range: [1, 64], Default: 1, on log-scale\n    min_samples_split, Type: UniformInteger, Range: [2, 128], Default: 2, on log-scale\n\n</pre> In\u00a0[5]: Copied! <pre>dimensions = len(cs.get_hyperparameters())\nprint(\"Dimensionality of search space: {}\".format(dimensions))\n</pre> dimensions = len(cs.get_hyperparameters()) print(\"Dimensionality of search space: {}\".format(dimensions)) <pre>Dimensionality of search space: 4\n</pre> <p>Now the primary black/gray-box interface to the Random Forest model needs to be built for DEHB to query. As given in the <code>00_interfacing_DEHB</code> notebook, this function will have a signature akin to: <code>target_function(config, fidelity)</code>, and return a two-element tuple of the <code>score</code> and <code>cost</code>. It must be noted that DEHB minimizes and therefore the <code>score</code> being returned by this <code>target_function</code> should account for it.</p> <p>In this example, the target function trains a Random Forest model on a dataset. We load a dataset here and maintain a fixed, train-validation-test split for one complete run. Multiple DEHB runs can therefore optimize on the same validation split, and evaluate final performance on the same test set.</p> In\u00a0[6]: Copied! <pre>from sklearn.datasets import load_iris, load_digits, load_wine\n\n\nclassification = {\"iris\": load_iris, \"digits\": load_digits, \"wine\": load_wine}\n</pre> from sklearn.datasets import load_iris, load_digits, load_wine   classification = {\"iris\": load_iris, \"digits\": load_digits, \"wine\": load_wine} In\u00a0[7]: Copied! <pre>from sklearn.model_selection import train_test_split\n\n\ndef prepare_dataset(model_type=\"classification\", dataset=None):\n\n    if model_type == \"classification\":\n        if dataset is None:\n            dataset = np.random.choice(list(classification.keys())) \n        _data = classification[dataset]()\n    else:\n        if dataset is None:\n            dataset = np.random.choice(list(regression.keys()))\n        _data = regression[dataset]()\n\n    train_X, rest_X, train_y, rest_y = train_test_split(\n      _data.get(\"data\"), \n      _data.get(\"target\"), \n      train_size=0.7, \n      shuffle=True, \n      random_state=seed\n    )\n    \n    # 10% test and 20% validation data\n    valid_X, test_X, valid_y, test_y = train_test_split(\n      rest_X, rest_y,\n      test_size=0.3333, \n      shuffle=True, \n      random_state=seed\n    )\n    return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset\n</pre> from sklearn.model_selection import train_test_split   def prepare_dataset(model_type=\"classification\", dataset=None):      if model_type == \"classification\":         if dataset is None:             dataset = np.random.choice(list(classification.keys()))          _data = classification[dataset]()     else:         if dataset is None:             dataset = np.random.choice(list(regression.keys()))         _data = regression[dataset]()      train_X, rest_X, train_y, rest_y = train_test_split(       _data.get(\"data\"),        _data.get(\"target\"),        train_size=0.7,        shuffle=True,        random_state=seed     )          # 10% test and 20% validation data     valid_X, test_X, valid_y, test_y = train_test_split(       rest_X, rest_y,       test_size=0.3333,        shuffle=True,        random_state=seed     )     return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset In\u00a0[8]: Copied! <pre>train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\\n    prepare_dataset(model_type=\"classification\")\n\nprint(dataset)\nprint(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(\n    train_X.shape, valid_X.shape, test_X.shape\n))\n</pre> train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\     prepare_dataset(model_type=\"classification\")  print(dataset) print(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(     train_X.shape, valid_X.shape, test_X.shape )) <pre>wine\nTrain size: (124, 13)\nValid size: (36, 13)\nTest size: (18, 13)\n</pre> In\u00a0[9]: Copied! <pre>from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, make_scorer\n\n\naccuracy_scorer = make_scorer(accuracy_score)\n</pre> from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score, make_scorer   accuracy_scorer = make_scorer(accuracy_score) In\u00a0[10]: Copied! <pre>def target_function(config, fidelity, **kwargs):\n    # Extracting support information\n    seed = kwargs[\"seed\"]\n    train_X = kwargs[\"train_X\"]\n    train_y = kwargs[\"train_y\"]\n    valid_X = kwargs[\"valid_X\"]\n    valid_y = kwargs[\"valid_y\"]\n    max_fidelity = kwargs[\"max_fidelity\"]\n    \n    if fidelity is None:\n        fidelity = max_fidelity\n    \n    start = time.time()\n    # Building model \n    model = RandomForestClassifier(\n        **config.get_dictionary(),\n        n_estimators=int(fidelity),\n        bootstrap=True,\n        random_state=seed,\n    )\n    # Training the model on the complete training set\n    model.fit(train_X, train_y)\n    \n    # Evaluating the model on the validation set\n    valid_accuracy = accuracy_scorer(model, valid_X, valid_y)\n    cost = time.time() - start\n    \n    # Evaluating the model on the test set as additional info\n    test_accuracy = accuracy_scorer(model, test_X, test_y)\n    \n    result = {\n        \"fitness\": -valid_accuracy,  # DE/DEHB minimizes\n        \"cost\": cost,\n        \"info\": {\n            \"test_score\": test_accuracy,\n            \"fidelity\": fidelity\n        }\n    }\n    return result\n</pre> def target_function(config, fidelity, **kwargs):     # Extracting support information     seed = kwargs[\"seed\"]     train_X = kwargs[\"train_X\"]     train_y = kwargs[\"train_y\"]     valid_X = kwargs[\"valid_X\"]     valid_y = kwargs[\"valid_y\"]     max_fidelity = kwargs[\"max_fidelity\"]          if fidelity is None:         fidelity = max_fidelity          start = time.time()     # Building model      model = RandomForestClassifier(         **config.get_dictionary(),         n_estimators=int(fidelity),         bootstrap=True,         random_state=seed,     )     # Training the model on the complete training set     model.fit(train_X, train_y)          # Evaluating the model on the validation set     valid_accuracy = accuracy_scorer(model, valid_X, valid_y)     cost = time.time() - start          # Evaluating the model on the test set as additional info     test_accuracy = accuracy_scorer(model, test_X, test_y)          result = {         \"fitness\": -valid_accuracy,  # DE/DEHB minimizes         \"cost\": cost,         \"info\": {             \"test_score\": test_accuracy,             \"fidelity\": fidelity         }     }     return result <p>We now have all components to define the problem to be optimized. DEHB can be initialized using all these information.</p> In\u00a0[11]: Copied! <pre>from dehb import DEHB\n\ndehb = DEHB(\n    f=target_function, # Here we do not need to necessarily specify the target function, but it can still be useful to call 'run' later.\n    cs=cs, \n    dimensions=dimensions, \n    min_fidelity=min_fidelity, \n    max_fidelity=max_fidelity,\n    n_workers=1,\n    output_path=\"./temp\"\n)\n</pre> from dehb import DEHB  dehb = DEHB(     f=target_function, # Here we do not need to necessarily specify the target function, but it can still be useful to call 'run' later.     cs=cs,      dimensions=dimensions,      min_fidelity=min_fidelity,      max_fidelity=max_fidelity,     n_workers=1,     output_path=\"./temp\" ) In\u00a0[12]: Copied! <pre>n_function_evals = 50\n\nfor _ in range(n_function_evals):\n    # Ask for the job_info, including the configuration to run and the fidelity\n    job_info = dehb.ask()\n\n    # Evaluate the configuration on the given fidelity. Here you are free to use\n    # any technique to compute the result. This job could e.g. be forwarded to\n    # a worker on your cluster (Which is not required to use Dask).\n    # The results dict has to contain the keys \"cost\" and \"fitness\" with an additional \"info\"\n    # dict for additional, user-specific data.\n    res = target_function(job_info[\"config\"], job_info[\"fidelity\"],\n                          # parameters as **kwargs in target_function\n                          seed=123,\n                          train_X=train_X,\n                          train_y=train_y,\n                          valid_X=valid_X,\n                          valid_y=valid_y,\n                          max_fidelity=dehb.max_fidelity)\n    \n    # When the evaluation is done, report the results back to the DEHB controller.\n    dehb.tell(job_info, res)\n\ntrajectory = dehb.traj\nruntime = dehb.runtime\nhistory = dehb.history\n</pre> n_function_evals = 50  for _ in range(n_function_evals):     # Ask for the job_info, including the configuration to run and the fidelity     job_info = dehb.ask()      # Evaluate the configuration on the given fidelity. Here you are free to use     # any technique to compute the result. This job could e.g. be forwarded to     # a worker on your cluster (Which is not required to use Dask).     # The results dict has to contain the keys \"cost\" and \"fitness\" with an additional \"info\"     # dict for additional, user-specific data.     res = target_function(job_info[\"config\"], job_info[\"fidelity\"],                           # parameters as **kwargs in target_function                           seed=123,                           train_X=train_X,                           train_y=train_y,                           valid_X=valid_X,                           valid_y=valid_y,                           max_fidelity=dehb.max_fidelity)          # When the evaluation is done, report the results back to the DEHB controller.     dehb.tell(job_info, res)  trajectory = dehb.traj runtime = dehb.runtime history = dehb.history In\u00a0[13]: Copied! <pre>print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")\n\n# Last recorded function evaluation\nlast_eval = history[-1]\nconfig, score, cost, fidelity, _info = last_eval\n\nprint(\"Last evaluated configuration, \")\nprint(dehb.vector_to_configspace(config), end=\"\")\nprint(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"\n      \"took {:.3f} seconds to run.\".format(score, fidelity, cost))\nprint(\"The additional info attached: {}\".format(_info))\n\nprint()\nprint(\"Best evaluated configuration, \")\n\nbest_config = dehb.vector_to_configspace(dehb.inc_config)\n\n# Creating a model using the best configuration found\nmodel = RandomForestClassifier(\n      **best_config.get_dictionary(),\n      n_estimators=int(max_fidelity),\n      bootstrap=True,\n      random_state=seed,\n)\n# Training the model on the complete training set\nmodel.fit(\n      np.concatenate((train_X, valid_X)), \n      np.concatenate((train_y, valid_y))\n)\n# Evaluating the model on the held-out test set\ntest_accuracy = accuracy_scorer(model, test_X, test_y)\n\nprint(f\"{best_config} got an accuracy of {test_accuracy} on the test set.\")\n</pre> print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")  # Last recorded function evaluation last_eval = history[-1] config, score, cost, fidelity, _info = last_eval  print(\"Last evaluated configuration, \") print(dehb.vector_to_configspace(config), end=\"\") print(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"       \"took {:.3f} seconds to run.\".format(score, fidelity, cost)) print(\"The additional info attached: {}\".format(_info))  print() print(\"Best evaluated configuration, \")  best_config = dehb.vector_to_configspace(dehb.inc_config)  # Creating a model using the best configuration found model = RandomForestClassifier(       **best_config.get_dictionary(),       n_estimators=int(max_fidelity),       bootstrap=True,       random_state=seed, ) # Training the model on the complete training set model.fit(       np.concatenate((train_X, valid_X)),        np.concatenate((train_y, valid_y)) ) # Evaluating the model on the held-out test set test_accuracy = accuracy_scorer(model, test_X, test_y)  print(f\"{best_config} got an accuracy of {test_accuracy} on the test set.\") <pre>50 50 50\n\nLast evaluated configuration, \nConfiguration(values={\n  'max_depth': 12,\n  'max_features': 0.6422911372304849,\n  'min_samples_leaf': 9,\n  'min_samples_split': 8,\n})got a score of -0.9722222222222222, was evaluated at a fidelity of 16.67 and took 0.018 seconds to run.\nThe additional info attached: {'test_score': 1.0, 'fidelity': 16.666666666666664}\n\nBest evaluated configuration, \nConfiguration(values={\n  'max_depth': 12,\n  'max_features': 0.6422911372304849,\n  'min_samples_leaf': 8,\n  'min_samples_split': 8,\n}) got an accuracy of 1.0 on the test set.\n</pre> <p>After running DEHB for 50 function evaluations using the ask and tell interface, we can still call the <code>run</code> function in order keep optimizing without specifically using ask and tell.</p> In\u00a0[14]: Copied! <pre># Continuing the ask/tell run of DEHB optimization for another 10s\ntrajectory, runtime, history = dehb.run(\n    total_cost=10,\n    verbose=False,\n    save_intermediate=False,\n    seed=123,\n    train_X=train_X,\n    train_y=train_y,\n    valid_X=valid_X,\n    valid_y=valid_y,\n    max_fidelity=dehb.max_fidelity\n)\nbest_config = dehb.vector_to_configspace(dehb.inc_config)\n\n# Creating a model using the best configuration found\nmodel = RandomForestClassifier(\n    **best_config.get_dictionary(),\n    n_estimators=int(max_fidelity),\n    bootstrap=True,\n    random_state=seed,\n)\n# Training the model on the complete training set\nmodel.fit(\n    np.concatenate((train_X, valid_X)), \n    np.concatenate((train_y, valid_y))\n)\n# Evaluating the model on the held-out test set\ntest_accuracy = accuracy_scorer(model, test_X, test_y)\n\n\nprint(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")\n\n# Last recorded function evaluation\nlast_eval = history[-1]\nconfig, score, cost, fidelity, _info = last_eval\n\nprint(\"Last evaluated configuration, \")\nprint(dehb.vector_to_configspace(config), end=\"\")\nprint(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"\n      \"took {:.3f} seconds to run.\".format(score, fidelity, cost))\nprint(\"The additional info attached: {}\".format(_info))\n\nprint()\nprint(\"Best evaluated configuration, \")\nprint(f\"{best_config} got an accuracy of {test_accuracy} on the test set.\")\n</pre> # Continuing the ask/tell run of DEHB optimization for another 10s trajectory, runtime, history = dehb.run(     total_cost=10,     verbose=False,     save_intermediate=False,     seed=123,     train_X=train_X,     train_y=train_y,     valid_X=valid_X,     valid_y=valid_y,     max_fidelity=dehb.max_fidelity ) best_config = dehb.vector_to_configspace(dehb.inc_config)  # Creating a model using the best configuration found model = RandomForestClassifier(     **best_config.get_dictionary(),     n_estimators=int(max_fidelity),     bootstrap=True,     random_state=seed, ) # Training the model on the complete training set model.fit(     np.concatenate((train_X, valid_X)),      np.concatenate((train_y, valid_y)) ) # Evaluating the model on the held-out test set test_accuracy = accuracy_scorer(model, test_X, test_y)   print(len(trajectory), len(runtime), len(history), end=\"\\n\\n\")  # Last recorded function evaluation last_eval = history[-1] config, score, cost, fidelity, _info = last_eval  print(\"Last evaluated configuration, \") print(dehb.vector_to_configspace(config), end=\"\") print(\"got a score of {}, was evaluated at a fidelity of {:.2f} and \"       \"took {:.3f} seconds to run.\".format(score, fidelity, cost)) print(\"The additional info attached: {}\".format(_info))  print() print(\"Best evaluated configuration, \") print(f\"{best_config} got an accuracy of {test_accuracy} on the test set.\") <pre>460 460 460\n\nLast evaluated configuration, \nConfiguration(values={\n  'max_depth': 8,\n  'max_features': 0.3567601906530009,\n  'min_samples_leaf': 5,\n  'min_samples_split': 2,\n})got a score of -1.0, was evaluated at a fidelity of 50.00 and took 0.050 seconds to run.\nThe additional info attached: {'test_score': 1.0, 'fidelity': 50.0}\n\nBest evaluated configuration, \nConfiguration(values={\n  'max_depth': 10,\n  'max_features': 0.6012257129543677,\n  'min_samples_leaf': 2,\n  'min_samples_split': 3,\n}) got an accuracy of 1.0 on the test set.\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#using-the-ask-tell-interface","title":"Using the Ask &amp; Tell interface\u00b6","text":"<p>This notebook aims to build on the template from <code>00_interfacing_DEHB</code> and use it on an actual problem, to optimize the hyperparameters of a Random Forest model, for a dataset. Here we use DEHB with the built-in ask and tell functionality.</p> <p>Additional requirements:</p> <ul> <li>scikit-learn&gt;=0.24</li> </ul>"},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#defining-fidelity-range","title":"Defining fidelity range\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#creating-search-space","title":"Creating search space\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#creating-target-function-to-optimize-2-parts","title":"Creating target function to optimize (2 parts)\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#1-preparing-dataset-and-splits","title":"1 ) Preparing dataset and splits\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#2-function-interface-with-dehb","title":"2 ) Function interface with DEHB\u00b6","text":""},{"location":"examples/01.2_Optimizing_RandomForest_using_Ask_Tell/#running-dehb","title":"Running DEHB\u00b6","text":""},{"location":"examples/02_using_DEHB_without_ConfigSpace/","title":"Using DEHB without ConfigSpace","text":"In\u00a0[1]: Copied! <pre>import time\nimport numpy as np\nimport warnings\n\n\nseed = 123\nnp.random.seed(seed)\nwarnings.filterwarnings('ignore')\n</pre> import time import numpy as np import warnings   seed = 123 np.random.seed(seed) warnings.filterwarnings('ignore') <p>The hyperparameters chosen, along with their type, and ranges:</p> <ul> <li><code>max_depth</code> $-$ integer $-$ [1, 15]</li> <li><code>min_samples_split</code> $-$ integer $-$ [2, 128] $-$ log-spacing</li> <li><code>max_features</code> $-$ float $-$ [0.1, 0.9]</li> <li><code>min_samples_leaf</code> $-$ integer $-$ [1, 64] $-$ log-spacing</li> </ul> <p>DE, and therefore DEHB, work in the unit hypercube space. The random individuals sampled at the beginning of DEHB, performs a uniform random sampling in the [0, 1] range for each parameter/dimension. Hence, each configuration suggested by DEHB also is in the [0, 1] range. The <code>vector_to_configspace</code> included in the DEHB source code, can reliably handle the transformation of the [0, 1] space of DEHB configurations to the original parameter space required. In the absence of ConfigSpace usage, such a conversion needs to included as part of the objective/target function being passed.</p> In\u00a0[2]: Copied! <pre># Declaring the search space\nparam_space = {\n    \"max_depth\": [1, 15, int, False],\n    \"min_samples_split\": [2, 128, int, True],\n    \"max_features\": [0.1, 0.9, float, False],\n    \"min_samples_leaf\": [1, 64, int, True],\n}\ndimensions = len(param_space)\n\n# Declaring the fidelity range\nmin_fidelity, max_fidelity = 2, 50\n\n\ndef transform_space(param_space, configuration):\n    \"\"\" Scales the [0, 1]-ranged parameter linearly to [lower, upper]\n    \n    Parameters\n    ----------\n    param_space : a dict containing the parameters and their meta-info\n    configuration : a vector with each dimension in [0, 1] (from DEHB)\n    \n    Results\n    -------\n    a dict which can be passed to the model as named hyperparameters\n    \"\"\"\n    assert len(configuration) == len(param_space)\n    config_dict = dict()\n    for i, (k, v) in enumerate(param_space.items()):\n        value = configuration[i]\n        lower, upper = v[0], v[1]\n        is_log = v[3]\n        if is_log:\n            # performs linear scaling in the log-space\n            log_range = np.log(upper) - np.log(lower)\n            value = np.exp(np.log(lower) + log_range * value)\n        else:\n            # linear scaling within the range of the parameter\n            value = lower + (upper - lower) * value\n        if v[2] == int:\n            value = np.round(value).astype(int)\n        config_dict[k] = value\n    return config_dict\n</pre> # Declaring the search space param_space = {     \"max_depth\": [1, 15, int, False],     \"min_samples_split\": [2, 128, int, True],     \"max_features\": [0.1, 0.9, float, False],     \"min_samples_leaf\": [1, 64, int, True], } dimensions = len(param_space)  # Declaring the fidelity range min_fidelity, max_fidelity = 2, 50   def transform_space(param_space, configuration):     \"\"\" Scales the [0, 1]-ranged parameter linearly to [lower, upper]          Parameters     ----------     param_space : a dict containing the parameters and their meta-info     configuration : a vector with each dimension in [0, 1] (from DEHB)          Results     -------     a dict which can be passed to the model as named hyperparameters     \"\"\"     assert len(configuration) == len(param_space)     config_dict = dict()     for i, (k, v) in enumerate(param_space.items()):         value = configuration[i]         lower, upper = v[0], v[1]         is_log = v[3]         if is_log:             # performs linear scaling in the log-space             log_range = np.log(upper) - np.log(lower)             value = np.exp(np.log(lower) + log_range * value)         else:             # linear scaling within the range of the parameter             value = lower + (upper - lower) * value         if v[2] == int:             value = np.round(value).astype(int)         config_dict[k] = value     return config_dict     <p>NOTE: To handle categorical parameters would require custom representations for such cases. Categorical parameters don't have a lower or upper range but rather a possible list of discrete choices or values. Moreoever, categorical parameters can be string categories, boolean or even ordinal in nature.</p> <p>Given this <code>transform_space</code> function, everything else from <code>01_Optimizing_RandomForest_using_DEHB</code> can be largely reused. Only the <code>target_function</code> needs to be modified to include the <code>transform_space</code> function. Also, the <code>configspace</code> parameter needs to be set tp <code>False</code> while initializing DEHB.</p> In\u00a0[3]: Copied! <pre>from sklearn.datasets import load_digits, load_wine\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, make_scorer\n\n\naccuracy_scorer = make_scorer(accuracy_score)\n\n\ndef prepare_dataset(model_type=\"classification\"):\n\n    if model_type == \"classification\":\n        dataset = np.random.choice(list(classification.keys()))\n        _data = classification[dataset]()\n    else:\n        dataset = np.random.choice(list(regression.keys()))\n        _data = regression[dataset]()\n\n    train_X, test_X, train_y, test_y = train_test_split(\n        _data.get(\"data\"), \n        _data.get(\"target\"), \n        test_size=0.1, \n        shuffle=True, \n        random_state=seed\n    )\n    train_X, valid_X, train_y, valid_y = train_test_split(\n        _data.get(\"data\"), \n        _data.get(\"target\"), \n        test_size=0.3, \n        shuffle=True, \n        random_state=seed\n    )\n    return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset\n\n\ndef target_function(config, fidelity, **kwargs):\n    # Extracting support information\n    seed = kwargs[\"seed\"]\n    train_X = kwargs[\"train_X\"]\n    train_y = kwargs[\"train_y\"]\n    valid_X = kwargs[\"valid_X\"]\n    valid_y = kwargs[\"valid_y\"]\n    max_fidelity = kwargs[\"max_fidelity\"]\n    \n    # Mapping [0, 1]-vector to Sklearn parameters\n    param_space = kwargs[\"param_space\"]\n    config = transform_space(param_space, config)\n    \n    if fidelity is None:\n        fidelity = max_fidelity\n    \n    start = time.time()\n    # Building model \n    model = RandomForestClassifier(\n        **config,\n        n_estimators=int(fidelity),\n        bootstrap=True,\n        random_state=seed,\n    )\n    # Training the model on the complete training set\n    model.fit(train_X, train_y)\n    \n    # Evaluating the model on the validation set\n    valid_accuracy = accuracy_scorer(model, valid_X, valid_y)\n    cost = time.time() - start\n    \n    # Evaluating the model on the test set as additional info\n    test_accuracy = accuracy_scorer(model, test_X, test_y)\n    \n    result = {\n        \"fitness\": -valid_accuracy,  # DE/DEHB minimizes\n        \"cost\": cost,\n        \"info\": {\n            \"test_score\": test_accuracy,\n            \"fidelity\": fidelity\n        }\n    }\n    return result\n\n\nclassification = {\"digits\": load_digits, \"wine\": load_wine}\ntrain_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\\n    prepare_dataset(model_type=\"classification\")\n\nprint(dataset)\nprint(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(\n    train_X.shape, valid_X.shape, test_X.shape\n))\n</pre> from sklearn.datasets import load_digits, load_wine from sklearn.model_selection import train_test_split from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score, make_scorer   accuracy_scorer = make_scorer(accuracy_score)   def prepare_dataset(model_type=\"classification\"):      if model_type == \"classification\":         dataset = np.random.choice(list(classification.keys()))         _data = classification[dataset]()     else:         dataset = np.random.choice(list(regression.keys()))         _data = regression[dataset]()      train_X, test_X, train_y, test_y = train_test_split(         _data.get(\"data\"),          _data.get(\"target\"),          test_size=0.1,          shuffle=True,          random_state=seed     )     train_X, valid_X, train_y, valid_y = train_test_split(         _data.get(\"data\"),          _data.get(\"target\"),          test_size=0.3,          shuffle=True,          random_state=seed     )     return train_X, train_y, valid_X, valid_y, test_X, test_y, dataset   def target_function(config, fidelity, **kwargs):     # Extracting support information     seed = kwargs[\"seed\"]     train_X = kwargs[\"train_X\"]     train_y = kwargs[\"train_y\"]     valid_X = kwargs[\"valid_X\"]     valid_y = kwargs[\"valid_y\"]     max_fidelity = kwargs[\"max_fidelity\"]          # Mapping [0, 1]-vector to Sklearn parameters     param_space = kwargs[\"param_space\"]     config = transform_space(param_space, config)          if fidelity is None:         fidelity = max_fidelity          start = time.time()     # Building model      model = RandomForestClassifier(         **config,         n_estimators=int(fidelity),         bootstrap=True,         random_state=seed,     )     # Training the model on the complete training set     model.fit(train_X, train_y)          # Evaluating the model on the validation set     valid_accuracy = accuracy_scorer(model, valid_X, valid_y)     cost = time.time() - start          # Evaluating the model on the test set as additional info     test_accuracy = accuracy_scorer(model, test_X, test_y)          result = {         \"fitness\": -valid_accuracy,  # DE/DEHB minimizes         \"cost\": cost,         \"info\": {             \"test_score\": test_accuracy,             \"fidelity\": fidelity         }     }     return result   classification = {\"digits\": load_digits, \"wine\": load_wine} train_X, train_y, valid_X, valid_y, test_X, test_y, dataset = \\     prepare_dataset(model_type=\"classification\")  print(dataset) print(\"Train size: {}\\nValid size: {}\\nTest size: {}\".format(     train_X.shape, valid_X.shape, test_X.shape )) <pre>digits\nTrain size: (1257, 64)\nValid size: (540, 64)\nTest size: (180, 64)\n</pre> In\u00a0[4]: Copied! <pre>from dehb import DEHB\n\n\ndehb = DEHB(\n    f=target_function, \n    dimensions=dimensions, \n    min_fidelity=min_fidelity, \n    max_fidelity=max_fidelity,\n    n_workers=1,\n    output_path=\"./temp\"\n)\n</pre> from dehb import DEHB   dehb = DEHB(     f=target_function,      dimensions=dimensions,      min_fidelity=min_fidelity,      max_fidelity=max_fidelity,     n_workers=1,     output_path=\"./temp\" ) In\u00a0[5]: Copied! <pre>trajectory, runtime, history = dehb.run(\n    total_cost=10, \n    verbose=False,\n    save_intermediate=False,\n    seed=123,\n    train_X=train_X,\n    train_y=train_y,\n    valid_X=valid_X,\n    valid_y=valid_y,\n    max_fidelity=dehb.max_fidelity,\n    param_space=param_space\n)\n</pre> trajectory, runtime, history = dehb.run(     total_cost=10,      verbose=False,     save_intermediate=False,     seed=123,     train_X=train_X,     train_y=train_y,     valid_X=valid_X,     valid_y=valid_y,     max_fidelity=dehb.max_fidelity,     param_space=param_space ) In\u00a0[6]: Copied! <pre>print(\"Incumbent score: {}\".format(dehb.inc_score))\nprint(\"Incumbent configuration:\\n{}\".format(transform_space(param_space, dehb.inc_config)))\n</pre> print(\"Incumbent score: {}\".format(dehb.inc_score)) print(\"Incumbent configuration:\\n{}\".format(transform_space(param_space, dehb.inc_config))) <pre>Incumbent score: -0.9611111111111111\nIncumbent configuration:\n{'max_depth': 9, 'min_samples_split': 3, 'max_features': 0.3990411414400532, 'min_samples_leaf': 1}\n</pre> In\u00a0[7]: Copied! <pre>model = RandomForestClassifier(\n    **transform_space(param_space, dehb.inc_config),\n    n_estimators=int(max_fidelity),\n    bootstrap=True,\n    random_state=seed,\n)\nmodel.fit(\n    np.concatenate((train_X, valid_X)),\n    np.concatenate((train_y, valid_y))\n)\ntest_accuracy = accuracy_scorer(model, test_X, test_y)\nprint(\"Test accuracy: {}\".format(test_accuracy))\n</pre> model = RandomForestClassifier(     **transform_space(param_space, dehb.inc_config),     n_estimators=int(max_fidelity),     bootstrap=True,     random_state=seed, ) model.fit(     np.concatenate((train_X, valid_X)),     np.concatenate((train_y, valid_y)) ) test_accuracy = accuracy_scorer(model, test_X, test_y) print(\"Test accuracy: {}\".format(test_accuracy)) <pre>Test accuracy: 0.9944444444444445\n</pre> In\u00a0[8]: Copied! <pre>from matplotlib import pyplot as plt\n\nplt.plot(np.cumsum(runtime), np.array(trajectory) + 1)\nplt.xlabel(\"Wallclock time in seconds\")\nplt.ylabel(\"Regret (1 - accuracy)\");\n</pre> from matplotlib import pyplot as plt  plt.plot(np.cumsum(runtime), np.array(trajectory) + 1) plt.xlabel(\"Wallclock time in seconds\") plt.ylabel(\"Regret (1 - accuracy)\"); <pre>\n---------------------------------------------------------------------------\nModuleNotFoundError                       Traceback (most recent call last)\nCell In[8], line 1\n----&gt; 1 from matplotlib import pyplot as plt\n      3 plt.plot(np.cumsum(runtime), np.array(trajectory) + 1)\n      4 plt.xlabel(\"Wallclock time in seconds\")\n\nModuleNotFoundError: No module named 'matplotlib'</pre>"},{"location":"examples/02_using_DEHB_without_ConfigSpace/#using-dehb-without-configspace","title":"Using DEHB without ConfigSpace\u00b6","text":"<p>This notebook repeats the example from <code>01.1_Optimizing_RandomForest_using_DEHB</code> but without using <code>ConfigSpace</code> for the parameter space, or the search space definition.</p>"},{"location":"examples/02_using_DEHB_without_ConfigSpace/#defining-transformation-from-dehb-01-space-to-original-parameter-space","title":"Defining transformation from DEHB [0,1]-space to original parameter space\u00b6","text":""},{"location":"examples/02_using_DEHB_without_ConfigSpace/#defining-the-target_function","title":"Defining the target_function\u00b6","text":""},{"location":"examples/02_using_DEHB_without_ConfigSpace/#running-dehb","title":"Running DEHB\u00b6","text":""},{"location":"examples/02_using_DEHB_without_ConfigSpace/#evaluating-the-incumbent","title":"Evaluating the incumbent\u00b6","text":""},{"location":"examples/02_using_DEHB_without_ConfigSpace/#plotting-the-optimization-trace-with-the-update-of-incumbents-over-time","title":"Plotting the optimization trace with the update of incumbents over time\u00b6","text":""},{"location":"getting_started/parallel/","title":"Parallel","text":""},{"location":"getting_started/parallel/#running-dehb-in-a-parallel-setting","title":"Running DEHB in a parallel setting","text":"<p>DEHB has been designed to interface a Dask client. DEHB can either create a Dask client during instantiation and close/kill the client during garbage collection.  Or a client can be passed as an argument during instantiation.</p> <ul> <li>Setting <code>n_workers</code> during instantiation \\     If set to <code>1</code> (default) then the entire process is a sequential run without invoking Dask. \\     If set to <code>&gt;1</code> then a Dask Client is initialized with as many workers as <code>n_workers</code>. \\     This parameter is ignored if <code>client</code> is not None.</li> <li>Setting <code>client</code> during instantiation \\     When <code>None</code> (default), a Dask client is created using <code>n_workers</code> specified. \\     Else, any custom-configured Dask Client can be created and passed as the <code>client</code> argument to DEHB.</li> </ul>"},{"location":"getting_started/parallel/#using-gpus-in-a-parallel-run","title":"Using GPUs in a parallel run","text":"<p>Certain target function evaluations (especially for Deep Learning) require computations to be  carried out on GPUs. The GPU devices are often ordered by device ID and if not configured, all  spawned worker processes access these devices in the same order and can either run out of memory or not exhibit parallelism.</p> <p>For <code>n_workers&gt;1</code> and when running on a single node (or local), the <code>single_node_with_gpus</code> can be  passed to the <code>run()</code> call to DEHB. Setting it to <code>False</code> (default) has no effect on the default setup  of the machine. Setting it to <code>True</code> will reorder the GPU device IDs dynamically by setting the environment  variable <code>CUDA_VISIBLE_DEVICES</code> for each worker process executing a target function evaluation. The re-ordering  is done in a manner that the first priority device is the one with the least number of active jobs assigned  to it by that DEHB run.</p> <p>To run the PyTorch MNIST example on a single node using 2 workers: <pre><code>python examples/03_pytorch_mnist_hpo.py \\\n    --min_budget 1 \\\n    --max_budget 3 \\\n    --runtime 60 \\\n    --n_workers 2 \\\n    --single_node_with_gpus \\\n    --verbose\n</code></pre></p>"},{"location":"getting_started/parallel/#multi-node-runs","title":"Multi-node runs","text":"<p>Multi-node parallelism is often contingent on the cluster setup to be deployed on. Dask provides useful  frameworks to interface various cluster designs. As long as the <code>client</code> passed to DEHB during  instantiation is of type <code>dask.distributed.Client</code>, DEHB can interact with this client and  distribute its optimization process in a parallel manner. </p> <p>For instance, <code>Dask-CLI</code> can be used to create a <code>dask-scheduler</code> which can dump its connection  details to a file on a cluster node accessible to all processes. Multiple <code>dask-worker</code> can then be created to interface the <code>dask-scheduler</code> by connecting to the details read from the file dumped. Each dask-worker can be triggered on any remote machine. Each worker can be configured as required,  including mapping to specific GPU devices. </p> <p>Some helper scripts can be found here, that can be used as a reference to run DEHB in a multi-node  manner on clusters managed by SLURM. (not expected to work off-the-shelf)</p> <p>To run the PyTorch MNIST example on a multi-node setup using 4 workers: <pre><code>bash utils/run_dask_setup.sh \\\n    -n 4 \\\n    -f dask_dump/scheduler.json \\   # This is how the workers will be discovered by DEHB\n    -e env_name\n\n# Make sure to sleep to allow the workers to setup properly\nsleep 5\n\npython examples/03_pytorch_mnist_hpo.py \\\n    --min_fidelity 1 \\\n    --max_fidelity 3 \\\n    --runtime 60 \\\n    --scheduler_file dask_dump/scheduler.json \\\n    --verbose\n</code></pre></p>"},{"location":"getting_started/single_worker/","title":"Single Worker","text":""},{"location":"getting_started/single_worker/#basic-single-worker-setup","title":"Basic single worker setup","text":"<p>A basic setup for optimizing can be done as follows. Please note, that this is example should solely show a simple setup of <code>dehb</code>. More in-depth examples can be found in the examples folder. First we need to setup a <code>ConfigurationSpace</code>, from which Configurations will be sampled:</p> Configuration Space<pre><code>from ConfigSpace import ConfigurationSpace, Configuration\n\ncs = ConfigurationSpace({\"x0\": (3.0, 10.0), \"x1\": [\"red\", \"green\"]})\nprint(cs)\n</code></pre> <pre><code>Configuration space object:\n  Hyperparameters:\n    x0, Type: UniformFloat, Range: [3.0, 10.0], Default: 6.5\n    x1, Type: Categorical, Choices: {red, green}, Default: red\n</code></pre> <p>Next, we need an <code>object_function</code>, which we are aiming to optimize: Configuration Space<pre><code>import numpy as np\n\ndef objective_function(x: Configuration, fidelity: float, **kwargs):\n    # Replace this with your actual objective value (y) and cost.\n    cost = (10 if x[\"x1\"] == \"red\" else 100) + fidelity\n    y = x[\"x0\"] + np.random.uniform()\n    return {\"fitness\": y, \"cost\": x[\"x0\"]}\n\nsample_config = cs.sample_configuration()\nprint(sample_config)\n\nresult = objective_function(sample_config, fidelity=10)\nprint(result)\n</code></pre> <pre><code>Configuration(values={\n  'x0': 8.081442413593486,\n  'x1': 'green',\n})\n{'fitness': 8.426257816719632, 'cost': 8.081442413593486}\n</code></pre> </p> <p>Finally, we can setup our optimizer and run DEHB:</p> Configuration Space<pre><code>from dehb import DEHB\n\ndim = len(cs.get_hyperparameters())\noptimizer = DEHB(\n    f=objective_function,\n    cs=cs,\n    dimensions=dim,\n    min_fidelity=3,\n    max_fidelity=27,\n    eta=3,\n    n_workers=1,\n    output_path=\"./logs\",\n)\n\n# Run optimization for 1 bracket. Output files will be saved to ./logs\ntraj, runtime, history = optimizer.run(brackets=1, verbose=True)\nconfig, fitness, runtime, fidelity, _ = history[0]\nprint(\"config\", config)\nprint(\"fitness\", fitness)\nprint(\"runtime\", runtime)\nprint(\"fidelity\", fidelity)\n</code></pre> <pre><code>config [0.4899593969425501, 0.0]\nfitness 6.898280767484525\nruntime 6.429715778597851\nfidelity 3.0\n</code></pre>"},{"location":"references/bracket_manager/","title":"Bracket manager","text":""},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager","title":"<code>SHBracketManager(n_configs, fidelities, bracket_id=None)</code>","text":"<p>             Bases: <code>object</code></p> <p>Synchronous Successive Halving utilities</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def __init__(self, n_configs, fidelities, bracket_id=None):\n    assert len(n_configs) == len(fidelities)\n    self.n_configs = n_configs\n    self.fidelities = fidelities\n    self.bracket_id = bracket_id\n    self.sh_bracket = {}\n    self._sh_bracket = {}\n    self._config_map = {}\n    for i, fidelity in enumerate(fidelities):\n        # sh_bracket keeps track of jobs/configs that are still to be scheduled/allocatted\n        # _sh_bracket keeps track of jobs/configs that have been run and results retrieved for\n        # (sh_bracket[i] + _sh_bracket[i]) == n_configs[i] is when no jobs have been scheduled\n        #   or all jobs for that fidelity/rung are over\n        # (sh_bracket[i] + _sh_bracket[i]) &lt; n_configs[i] indicates a job has been scheduled\n        #   and is queued/running and the bracket needs to be paused till results are retrieved\n        self.sh_bracket[fidelity] = n_configs[i]  # each scheduled job does -= 1\n        self._sh_bracket[fidelity] = 0  # each retrieved job does +=1\n    self.n_rungs = len(fidelities)\n    self.current_rung = 0\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.get_fidelity","title":"<code>get_fidelity(rung=None)</code>","text":"<p>Returns the exact fidelity that rung is pointing to.</p> <p>Returns current rung's fidelity if no rung is passed.</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def get_fidelity(self, rung=None):\n    \"\"\" Returns the exact fidelity that rung is pointing to.\n\n    Returns current rung's fidelity if no rung is passed.\n    \"\"\"\n    if rung is not None:\n        return self.fidelities[rung]\n    return self.fidelities[self.current_rung]\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.get_lower_fidelity_promotions","title":"<code>get_lower_fidelity_promotions(fidelity)</code>","text":"<p>Returns the immediate lower fidelity and the number of configs to be promoted from there</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def get_lower_fidelity_promotions(self, fidelity):\n    \"\"\" Returns the immediate lower fidelity and the number of configs to be promoted from there\n    \"\"\"\n    assert fidelity in self.fidelities\n    rung = np.where(fidelity == self.fidelities)[0][0]\n    prev_rung = np.clip(rung - 1, a_min=0, a_max=self.n_rungs-1)\n    lower_fidelity = self.fidelities[prev_rung]\n    num_promote_configs = self.n_configs[rung]\n    return lower_fidelity, num_promote_configs\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.get_next_job_fidelity","title":"<code>get_next_job_fidelity()</code>","text":"<p>Returns the fidelity that will be selected if current_rung is incremented by 1</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def get_next_job_fidelity(self):\n    \"\"\" Returns the fidelity that will be selected if current_rung is incremented by 1\n    \"\"\"\n    if self.sh_bracket[self.get_fidelity()] &gt; 0:\n        # the current rung still has unallocated jobs (&gt;0)\n        return self.get_fidelity()\n    else:\n        # the current rung has no more jobs to allocate, increment it\n        rung = (self.current_rung + 1) % self.n_rungs\n        if self.sh_bracket[self.get_fidelity(rung)] &gt; 0:\n            # the incremented rung has unallocated jobs (&gt;0)\n            return self.get_fidelity(rung)\n        else:\n            # all jobs for this bracket has been allocated/bracket is complete\n            # no more fidelities to evaluate and can return None\n            pass\n        return None\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.register_job","title":"<code>register_job(fidelity)</code>","text":"<p>Registers the allocation of a configuration for the fidelity and updates current rung</p> <p>This function must be called when scheduling a job in order to allow the bracket manager to continue job and fidelity allocation without waiting for jobs to finish and return results necessarily. This feature can be leveraged to run brackets asynchronously.</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def register_job(self, fidelity):\n    \"\"\" Registers the allocation of a configuration for the fidelity and updates current rung\n\n    This function must be called when scheduling a job in order to allow the bracket manager\n    to continue job and fidelity allocation without waiting for jobs to finish and return\n    results necessarily. This feature can be leveraged to run brackets asynchronously.\n    \"\"\"\n    assert fidelity in self.fidelities\n    assert self.sh_bracket[fidelity] &gt; 0\n    self.sh_bracket[fidelity] -= 1\n    if not self._is_rung_pending(self.current_rung):\n        # increment current rung if no jobs left in the rung\n        self.current_rung = (self.current_rung + 1) % self.n_rungs\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.complete_job","title":"<code>complete_job(fidelity)</code>","text":"<p>Notifies the bracket that a job for a fidelity has been completed</p> <p>This function must be called when a config for a fidelity has finished evaluation to inform the Bracket Manager that no job needs to be waited for and the next rung can begin for the synchronous Successive Halving case.</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def complete_job(self, fidelity):\n    \"\"\" Notifies the bracket that a job for a fidelity has been completed\n\n    This function must be called when a config for a fidelity has finished evaluation to inform\n    the Bracket Manager that no job needs to be waited for and the next rung can begin for the\n    synchronous Successive Halving case.\n    \"\"\"\n    assert fidelity in self.fidelities\n    _max_configs = self.n_configs[list(self.fidelities).index(fidelity)]\n    assert self._sh_bracket[fidelity] &lt; _max_configs\n    self._sh_bracket[fidelity] += 1\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.previous_rung_waits","title":"<code>previous_rung_waits()</code>","text":"<p>Returns True if none of the rungs &lt; current rung is waiting for results</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def previous_rung_waits(self):\n    \"\"\" Returns True if none of the rungs &lt; current rung is waiting for results\n    \"\"\"\n    for rung in range(self.current_rung):\n        if self._is_rung_waiting(rung) and not self._is_rung_pending(rung):\n            return True\n    return False\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.is_bracket_done","title":"<code>is_bracket_done()</code>","text":"<p>Returns True if all configs in all rungs in the bracket have been allocated</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def is_bracket_done(self):\n    \"\"\" Returns True if all configs in all rungs in the bracket have been allocated\n    \"\"\"\n    return ~self.is_pending() and ~self.is_waiting()\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.is_pending","title":"<code>is_pending()</code>","text":"<p>Returns True if any of the rungs/fidelities have still a configuration to submit</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def is_pending(self):\n    \"\"\" Returns True if any of the rungs/fidelities have still a configuration to submit\n    \"\"\"\n    return np.any([self._is_rung_pending(i) &gt; 0 for i, _ in enumerate(self.fidelities)])\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.is_waiting","title":"<code>is_waiting()</code>","text":"<p>Returns True if any of the rungs/fidelities have a configuration pending/running</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def is_waiting(self):\n    \"\"\" Returns True if any of the rungs/fidelities have a configuration pending/running\n    \"\"\"\n    return np.any([self._is_rung_waiting(i) &gt; 0 for i, _ in enumerate(self.fidelities)])\n</code></pre>"},{"location":"references/bracket_manager/#dehb.utils.bracket_manager.SHBracketManager.reset_waiting_jobs","title":"<code>reset_waiting_jobs()</code>","text":"<p>Resets all waiting jobs and updates the current_rung pointer accordingly.</p> Source code in <code>src/dehb/utils/bracket_manager.py</code> <pre><code>def reset_waiting_jobs(self):\n    \"\"\"Resets all waiting jobs and updates the current_rung pointer accordingly.\"\"\"\n    for i, fidelity in enumerate(self.fidelities):\n        pending = self.sh_bracket[fidelity]\n        done = self._sh_bracket[fidelity]\n        waiting = np.abs(self.n_configs[i] - pending - done)\n\n        # update current_rung pointer to the lowest rung with waiting jobs\n        if waiting &gt; 0 and self.current_rung &gt; i:\n            self.current_rung = i\n        # reset waiting jobs\n        self.sh_bracket[fidelity] += waiting\n</code></pre>"},{"location":"references/de/","title":"DE","text":""},{"location":"references/de/#dehb.optimizers.de.DEBase","title":"<code>DEBase(cs=None, f=None, dimensions=None, pop_size=None, max_age=None, mutation_factor=None, crossover_prob=None, strategy=None, boundary_fix_type='random', config_repository=None, **kwargs)</code>","text":"<p>Base class for Differential Evolution</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=None,\n             mutation_factor=None, crossover_prob=None, strategy=None,\n             boundary_fix_type='random', config_repository=None, **kwargs):\n    # Benchmark related variables\n    self.cs = cs\n    self.f = f\n    if dimensions is None and self.cs is not None:\n        self.dimensions = len(self.cs.get_hyperparameters())\n    else:\n        self.dimensions = dimensions\n\n    # DE related variables\n    self.pop_size = pop_size\n    self.max_age = max_age\n    self.mutation_factor = mutation_factor\n    self.crossover_prob = crossover_prob\n    self.strategy = strategy\n    self.fix_type = boundary_fix_type\n\n    # Miscellaneous\n    self.configspace = True if isinstance(self.cs, ConfigSpace.ConfigurationSpace) else False\n    self.hps = dict()\n    if self.configspace:\n        for i, hp in enumerate(cs.get_hyperparameters()):\n            # maps hyperparameter name to positional index in vector form\n            self.hps[hp.name] = i\n    self.output_path = kwargs['output_path'] if 'output_path' in kwargs else './'\n    os.makedirs(self.output_path, exist_ok=True)\n\n    if config_repository:\n        self.config_repository = config_repository\n    else:\n        self.config_repository = ConfigRepository()\n\n    # Global trackers\n    self.inc_score : float\n    self.inc_config : np.ndarray[float]\n    self.inc_id : int\n    self.population : np.ndarray[np.ndarray[float]]\n    self.population_ids :np.ndarray[int]\n    self.fitness : np.ndarray[float]\n    self.age : int\n    self.history : list[object]\n    self.reset()\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DEBase.sample_population","title":"<code>sample_population(size=3, alt_pop=None)</code>","text":"<p>Samples 'size' individuals</p> <p>If alt_pop is None or a list/array of None, sample from own population Else sample from the specified alternate population (alt_pop)</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def sample_population(self, size: int = 3, alt_pop: List = None) -&gt; List:\n    '''Samples 'size' individuals\n\n    If alt_pop is None or a list/array of None, sample from own population\n    Else sample from the specified alternate population (alt_pop)\n    '''\n    if isinstance(alt_pop, list) or isinstance(alt_pop, np.ndarray):\n        idx = [indv is None for indv in alt_pop]\n        if any(idx):\n            selection = np.random.choice(np.arange(len(self.population)), size, replace=False)\n            return self.population[selection]\n        else:\n            if len(alt_pop) &lt; 3:\n                alt_pop = np.vstack((alt_pop, self.population))\n            selection = np.random.choice(np.arange(len(alt_pop)), size, replace=False)\n            alt_pop = np.stack(alt_pop)\n            return alt_pop[selection]\n    else:\n        selection = np.random.choice(np.arange(len(self.population)), size, replace=False)\n        return self.population[selection]\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DEBase.boundary_check","title":"<code>boundary_check(vector)</code>","text":"<p>Checks whether each of the dimensions of the input vector are within [0, 1]. If not, values of those dimensions are replaced with the type of fix selected.</p> <p>if fix_type == 'random', the values are replaced with a random sampling from (0,1) if fix_type == 'clip', the values are clipped to the closest limit from {0, 1}</p>"},{"location":"references/de/#dehb.optimizers.de.DEBase.boundary_check--parameters","title":"Parameters","text":"<p>vector : array</p>"},{"location":"references/de/#dehb.optimizers.de.DEBase.boundary_check--returns","title":"Returns","text":"<p>array</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def boundary_check(self, vector: np.ndarray) -&gt; np.ndarray:\n    '''\n    Checks whether each of the dimensions of the input vector are within [0, 1].\n    If not, values of those dimensions are replaced with the type of fix selected.\n\n    if fix_type == 'random', the values are replaced with a random sampling from (0,1)\n    if fix_type == 'clip', the values are clipped to the closest limit from {0, 1}\n\n    Parameters\n    ----------\n    vector : array\n\n    Returns\n    -------\n    array\n    '''\n    violations = np.where((vector &gt; 1) | (vector &lt; 0))[0]\n    if len(violations) == 0:\n        return vector\n    if self.fix_type == 'random':\n        vector[violations] = np.random.uniform(low=0.0, high=1.0, size=len(violations))\n    else:\n        vector[violations] = np.clip(vector[violations], a_min=0, a_max=1)\n    return vector\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DEBase.vector_to_configspace","title":"<code>vector_to_configspace(vector)</code>","text":"<p>Converts numpy array to ConfigSpace object</p> <p>Works when self.cs is a ConfigSpace object and the input vector is in the domain [0, 1].</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def vector_to_configspace(self, vector: np.ndarray) -&gt; ConfigSpace.Configuration:\n    '''Converts numpy array to ConfigSpace object\n\n    Works when self.cs is a ConfigSpace object and the input vector is in the domain [0, 1].\n    '''\n    # creates a ConfigSpace object dict with all hyperparameters present, the inactive too\n    new_config = ConfigSpace.util.impute_inactive_values(\n        self.cs.get_default_configuration()\n    ).get_dictionary()\n    # iterates over all hyperparameters and normalizes each based on its type\n    for i, hyper in enumerate(self.cs.get_hyperparameters()):\n        if type(hyper) == ConfigSpace.OrdinalHyperparameter:\n            ranges = np.arange(start=0, stop=1, step=1/len(hyper.sequence))\n            param_value = hyper.sequence[np.where((vector[i] &lt; ranges) == False)[0][-1]]\n        elif type(hyper) == ConfigSpace.CategoricalHyperparameter:\n            ranges = np.arange(start=0, stop=1, step=1/len(hyper.choices))\n            param_value = hyper.choices[np.where((vector[i] &lt; ranges) == False)[0][-1]]\n        elif type(hyper) == ConfigSpace.Constant:\n            param_value = hyper.default_value\n        else:  # handles UniformFloatHyperparameter &amp; UniformIntegerHyperparameter\n            # rescaling continuous values\n            if hyper.log:\n                log_range = np.log(hyper.upper) - np.log(hyper.lower)\n                param_value = np.exp(np.log(hyper.lower) + vector[i] * log_range)\n            else:\n                param_value = hyper.lower + (hyper.upper - hyper.lower) * vector[i]\n            if type(hyper) == ConfigSpace.UniformIntegerHyperparameter:\n                param_value = int(np.round(param_value))  # converting to discrete (int)\n            else:\n                param_value = float(param_value)\n        new_config[hyper.name] = param_value\n    # the mapping from unit hypercube to the actual config space may lead to illegal\n    # configurations based on conditions defined, which need to be deactivated/removed\n    new_config = ConfigSpace.util.deactivate_inactive_hyperparameters(\n        configuration = new_config, configuration_space=self.cs\n    )\n    return new_config\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DEBase.configspace_to_vector","title":"<code>configspace_to_vector(config)</code>","text":"<p>Converts ConfigSpace object to numpy array scaled to [0,1]</p> <p>Works when self.cs is a ConfigSpace object and the input config is a ConfigSpace object. Handles conditional spaces implicitly by replacing illegal parameters with default values to maintain the dimensionality of the vector.</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def configspace_to_vector(self, config: ConfigSpace.Configuration) -&gt; np.ndarray:\n    '''Converts ConfigSpace object to numpy array scaled to [0,1]\n\n    Works when self.cs is a ConfigSpace object and the input config is a ConfigSpace object.\n    Handles conditional spaces implicitly by replacing illegal parameters with default values\n    to maintain the dimensionality of the vector.\n    '''\n    # the imputation replaces illegal parameter values with their default\n    config = ConfigSpace.util.impute_inactive_values(config)\n    dimensions = len(self.cs.get_hyperparameters())\n    vector = [np.nan for i in range(dimensions)]\n    for name in config:\n        i = self.hps[name]\n        hyper = self.cs.get_hyperparameter(name)\n        if type(hyper) == ConfigSpace.OrdinalHyperparameter:\n            nlevels = len(hyper.sequence)\n            vector[i] = hyper.sequence.index(config[name]) / nlevels\n        elif type(hyper) == ConfigSpace.CategoricalHyperparameter:\n            nlevels = len(hyper.choices)\n            vector[i] = hyper.choices.index(config[name]) / nlevels\n        elif type(hyper) == ConfigSpace.Constant:\n            vector[i] = 0 # set constant to 0, so that it wont be affected by mutation\n        else:\n            bounds = (hyper.lower, hyper.upper)\n            param_value = config[name]\n            if hyper.log:\n                vector[i] = np.log(param_value / bounds[0]) / np.log(bounds[1] / bounds[0])\n            else:\n                vector[i] = (config[name] - bounds[0]) / (bounds[1] - bounds[0])\n    return np.array(vector)\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE","title":"<code>DE(cs=None, f=None, dimensions=None, pop_size=20, max_age=np.inf, mutation_factor=None, crossover_prob=None, strategy='rand1_bin', encoding=False, dim_map=None, config_repository=None, **kwargs)</code>","text":"<p>             Bases: <code>DEBase</code></p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def __init__(self, cs=None, f=None, dimensions=None, pop_size=20, max_age=np.inf,\n             mutation_factor=None, crossover_prob=None, strategy='rand1_bin',\n             encoding=False, dim_map=None, config_repository=None, **kwargs):\n    super().__init__(cs=cs, f=f, dimensions=dimensions, pop_size=pop_size, max_age=max_age,\n                     mutation_factor=mutation_factor, crossover_prob=crossover_prob,\n                     strategy=strategy, config_repository=config_repository,\n                     **kwargs)\n    if self.strategy is not None:\n        self.mutation_strategy = self.strategy.split('_')[0]\n        self.crossover_strategy = self.strategy.split('_')[1]\n    else:\n        self.mutation_strategy = self.crossover_strategy = None\n    self.encoding = encoding\n    self.dim_map = dim_map\n    self._set_min_pop_size()\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.__getstate__","title":"<code>__getstate__()</code>","text":"<p>Allows the object to picklable while having Dask client as a class attribute.</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def __getstate__(self):\n    \"\"\" Allows the object to picklable while having Dask client as a class attribute.\n    \"\"\"\n    d = dict(self.__dict__)\n    d[\"client\"] = None  # hack to allow Dask client to be a class attribute\n    d[\"logger\"] = None  # hack to allow logger object to be a class attribute\n    return d\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.__del__","title":"<code>__del__()</code>","text":"<p>Ensures a clean kill of the Dask client and frees up a port.</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def __del__(self):\n    \"\"\" Ensures a clean kill of the Dask client and frees up a port.\n    \"\"\"\n    if hasattr(self, \"client\") and isinstance(self.client, Client):\n        self.client.close()\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.init_eval_pop","title":"<code>init_eval_pop(fidelity=None, eval=True, **kwargs)</code>","text":"<p>Creates new population of 'pop_size' and evaluates individuals.</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def init_eval_pop(self, fidelity=None, eval=True, **kwargs):\n    '''Creates new population of 'pop_size' and evaluates individuals.\n    '''\n    self.population = self.init_population(self.pop_size)\n    self.population_ids = self.config_repository.announce_population(self.population, fidelity)\n    self.fitness = np.array([np.inf for i in range(self.pop_size)])\n    self.age = np.array([self.max_age] * self.pop_size)\n\n    traj = []\n    runtime = []\n    history = []\n\n    if not eval:\n        return traj, runtime, history\n\n    for i in range(self.pop_size):\n        config = self.population[i]\n        config_id = self.population_ids[i]\n        res = self.f_objective(config, fidelity, **kwargs)\n        self.fitness[i], cost = res[\"fitness\"], res[\"cost\"]\n        info = res[\"info\"] if \"info\" in res else dict()\n        if self.fitness[i] &lt; self.inc_score:\n            self.inc_score = self.fitness[i]\n            self.inc_config = config\n            self.inc_id = config_id\n        self.config_repository.tell_result(config_id, float(fidelity or 0), res[\"fitness\"], res[\"cost\"], info)\n        traj.append(self.inc_score)\n        runtime.append(cost)\n        history.append((config.tolist(), float(self.fitness[i]), float(fidelity or 0), info))\n\n    return traj, runtime, history\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.eval_pop","title":"<code>eval_pop(population=None, population_ids=None, fidelity=None, **kwargs)</code>","text":"<p>Evaluates a population</p> <p>If population=None, the current population's fitness will be evaluated If population!=None, this population will be evaluated</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def eval_pop(self, population=None, population_ids=None, fidelity=None, **kwargs):\n    '''Evaluates a population\n\n    If population=None, the current population's fitness will be evaluated\n    If population!=None, this population will be evaluated\n    '''\n    pop = self.population if population is None else population\n    pop_ids = self.population_ids if population_ids is None else population_ids\n    pop_size = self.pop_size if population is None else len(pop)\n    traj = []\n    runtime = []\n    history = []\n    fitnesses = []\n    costs = []\n    ages = []\n    for i in range(pop_size):\n        res = self.f_objective(pop[i], fidelity, **kwargs)\n        fitness, cost = res[\"fitness\"], res[\"cost\"]\n        info = res[\"info\"] if \"info\" in res else dict()\n        if population is None:\n            self.fitness[i] = fitness\n        if fitness &lt;= self.inc_score:\n            self.inc_score = fitness\n            self.inc_config = pop[i]\n            self.inc_id = pop_ids[i]\n        self.config_repository.tell_result(pop_ids[i], float(fidelity or 0), info)\n        traj.append(self.inc_score)\n        runtime.append(cost)\n        history.append((pop[i].tolist(), float(fitness), float(fidelity or 0), info))\n        fitnesses.append(fitness)\n        costs.append(cost)\n        ages.append(self.max_age)\n    if population is None:\n        self.fitness = np.array(fitnesses)\n        return traj, runtime, history\n    else:\n        return traj, runtime, history, np.array(fitnesses), np.array(ages)\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.mutation_rand1","title":"<code>mutation_rand1(r1, r2, r3)</code>","text":"<p>Performs the 'rand1' type of DE mutation</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def mutation_rand1(self, r1, r2, r3):\n    '''Performs the 'rand1' type of DE mutation\n    '''\n    diff = r2 - r3\n    mutant = r1 + self.mutation_factor * diff\n    return mutant\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.mutation_rand2","title":"<code>mutation_rand2(r1, r2, r3, r4, r5)</code>","text":"<p>Performs the 'rand2' type of DE mutation</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def mutation_rand2(self, r1, r2, r3, r4, r5):\n    '''Performs the 'rand2' type of DE mutation\n    '''\n    diff1 = r2 - r3\n    diff2 = r4 - r5\n    mutant = r1 + self.mutation_factor * diff1 + self.mutation_factor * diff2\n    return mutant\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.mutation","title":"<code>mutation(current=None, best=None, alt_pop=None)</code>","text":"<p>Performs DE mutation</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def mutation(self, current=None, best=None, alt_pop=None):\n    '''Performs DE mutation\n    '''\n    if self.mutation_strategy == 'rand1':\n        r1, r2, r3 = self.sample_population(size=3, alt_pop=alt_pop)\n        mutant = self.mutation_rand1(r1, r2, r3)\n\n    elif self.mutation_strategy == 'rand2':\n        r1, r2, r3, r4, r5 = self.sample_population(size=5, alt_pop=alt_pop)\n        mutant = self.mutation_rand2(r1, r2, r3, r4, r5)\n\n    elif self.mutation_strategy == 'rand2dir':\n        r1, r2, r3 = self.sample_population(size=3, alt_pop=alt_pop)\n        mutant = self.mutation_rand2dir(r1, r2, r3)\n\n    elif self.mutation_strategy == 'best1':\n        r1, r2 = self.sample_population(size=2, alt_pop=alt_pop)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_rand1(best, r1, r2)\n\n    elif self.mutation_strategy == 'best2':\n        r1, r2, r3, r4 = self.sample_population(size=4, alt_pop=alt_pop)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_rand2(best, r1, r2, r3, r4)\n\n    elif self.mutation_strategy == 'currenttobest1':\n        r1, r2 = self.sample_population(size=2, alt_pop=alt_pop)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_currenttobest1(current, best, r1, r2)\n\n    elif self.mutation_strategy == 'randtobest1':\n        r1, r2, r3 = self.sample_population(size=3, alt_pop=alt_pop)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_currenttobest1(r1, best, r2, r3)\n\n    return mutant\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.crossover_bin","title":"<code>crossover_bin(target, mutant)</code>","text":"<p>Performs the binomial crossover of DE</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def crossover_bin(self, target, mutant):\n    '''Performs the binomial crossover of DE\n    '''\n    cross_points = np.random.rand(self.dimensions) &lt; self.crossover_prob\n    if not np.any(cross_points):\n        cross_points[np.random.randint(0, self.dimensions)] = True\n    offspring = np.where(cross_points, mutant, target)\n    return offspring\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.crossover_exp","title":"<code>crossover_exp(target, mutant)</code>","text":"<p>Performs the exponential crossover of DE</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def crossover_exp(self, target, mutant):\n    '''Performs the exponential crossover of DE\n    '''\n    n = np.random.randint(0, self.dimensions)\n    L = 0\n    while ((np.random.rand() &lt; self.crossover_prob) and L &lt; self.dimensions):\n        idx = (n+L) % self.dimensions\n        target[idx] = mutant[idx]\n        L = L + 1\n    return target\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.crossover","title":"<code>crossover(target, mutant)</code>","text":"<p>Performs DE crossover</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def crossover(self, target, mutant):\n    '''Performs DE crossover\n    '''\n    if self.crossover_strategy == 'bin':\n        offspring = self.crossover_bin(target, mutant)\n    elif self.crossover_strategy == 'exp':\n        offspring = self.crossover_exp(target, mutant)\n    return offspring\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.selection","title":"<code>selection(trials, trial_ids, fidelity=None, **kwargs)</code>","text":"<p>Carries out a parent-offspring competition given a set of trial population</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def selection(self, trials, trial_ids, fidelity=None, **kwargs):\n    '''Carries out a parent-offspring competition given a set of trial population\n    '''\n    traj = []\n    runtime = []\n    history = []\n    for i in range(len(trials)):\n        # evaluation of the newly created individuals\n        res = self.f_objective(trials[i], fidelity, **kwargs)\n        fitness, cost = res[\"fitness\"], res[\"cost\"]\n        info = res[\"info\"] if \"info\" in res else dict()\n        # log result to config repo\n        self.config_repository.tell_result(trial_ids[i], float(fidelity or 0), fitness, cost, info)\n        # selection -- competition between parent[i] -- child[i]\n        ## equality is important for landscape exploration\n        if fitness &lt;= self.fitness[i]:\n            self.population[i] = trials[i]\n            self.population_ids[i] = trial_ids[i]\n            self.fitness[i] = fitness\n            # resetting age since new individual in the population\n            self.age[i] = self.max_age\n        else:\n            # decreasing age by 1 of parent who is better than offspring/trial\n            self.age[i] -= 1\n        # updation of global incumbent for trajectory\n        if self.fitness[i] &lt; self.inc_score:\n            self.inc_score = self.fitness[i]\n            self.inc_config = self.population[i]\n            self.inc_id = self.population[i]\n        traj.append(self.inc_score)\n        runtime.append(cost)\n        history.append((trials[i].tolist(), float(fitness), float(fidelity or 0), info))\n    return traj, runtime, history\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.evolve_generation","title":"<code>evolve_generation(fidelity=None, best=None, alt_pop=None, **kwargs)</code>","text":"<p>Performs a complete DE evolution: mutation -&gt; crossover -&gt; selection</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs):\n    '''Performs a complete DE evolution: mutation -&gt; crossover -&gt; selection\n    '''\n    trials = []\n    trial_ids = []\n    for j in range(self.pop_size):\n        target = self.population[j]\n        donor = self.mutation(current=target, best=best, alt_pop=alt_pop)\n        trial = self.crossover(target, donor)\n        trial = self.boundary_check(trial)\n        trial_id = self.config_repository.announce_config(trial, float(fidelity or 0))\n        trials.append(trial)\n        trial_ids.append(trial_id)\n    trials = np.array(trials)\n    trial_ids = np.array(trial_ids)\n    traj, runtime, history = self.selection(trials, trial_ids, fidelity, **kwargs)\n    return traj, runtime, history\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.DE.sample_mutants","title":"<code>sample_mutants(size, population=None)</code>","text":"<p>Generates 'size' mutants from the population using rand1</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def sample_mutants(self, size, population=None):\n    '''Generates 'size' mutants from the population using rand1\n    '''\n    if population is None:\n        population = self.population\n    elif len(population) &lt; 3:\n        population = np.vstack((self.population, population))\n\n    old_strategy = self.mutation_strategy\n    self.mutation_strategy = 'rand1'\n    mutants = np.random.uniform(low=0.0, high=1.0, size=(size, self.dimensions))\n    for i in range(size):\n        mutant = self.mutation(current=None, best=None, alt_pop=population)\n        mutants[i] = self.boundary_check(mutant)\n    self.mutation_strategy = old_strategy\n\n    return mutants\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.AsyncDE","title":"<code>AsyncDE(cs=None, f=None, dimensions=None, pop_size=None, max_age=np.inf, mutation_factor=None, crossover_prob=None, strategy='rand1_bin', async_strategy='immediate', config_repository=None, **kwargs)</code>","text":"<p>             Bases: <code>DE</code></p> <p>Extends DE to be Asynchronous with variations</p>"},{"location":"references/de/#dehb.optimizers.de.AsyncDE--parameters","title":"Parameters","text":"<p>async_strategy : str     'deferred' - target will be chosen sequentially from the population         the winner of the selection step will be included in the population only after         the entire population has had a selection step in that generation     'immediate' - target will be chosen sequentially from the population         the winner of the selection step is included in the population right away     'random' - target will be chosen randomly from the population for mutation-crossover         the winner of the selection step is included in the population right away     'worst' - the worst individual will be chosen as the target         the winner of the selection step is included in the population right away     {immediate, worst, random} implement Asynchronous-DE</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def __init__(self, cs=None, f=None, dimensions=None, pop_size=None, max_age=np.inf,\n             mutation_factor=None, crossover_prob=None, strategy='rand1_bin',\n             async_strategy='immediate', config_repository=None, **kwargs):\n    '''Extends DE to be Asynchronous with variations\n\n    Parameters\n    ----------\n    async_strategy : str\n        'deferred' - target will be chosen sequentially from the population\n            the winner of the selection step will be included in the population only after\n            the entire population has had a selection step in that generation\n        'immediate' - target will be chosen sequentially from the population\n            the winner of the selection step is included in the population right away\n        'random' - target will be chosen randomly from the population for mutation-crossover\n            the winner of the selection step is included in the population right away\n        'worst' - the worst individual will be chosen as the target\n            the winner of the selection step is included in the population right away\n        {immediate, worst, random} implement Asynchronous-DE\n    '''\n    super().__init__(cs=cs, f=f, dimensions=dimensions, pop_size=pop_size, max_age=max_age,\n                     mutation_factor=mutation_factor, crossover_prob=crossover_prob,\n                     strategy=strategy, config_repository=config_repository,\n                     **kwargs)\n    if self.strategy is not None:\n        self.mutation_strategy = self.strategy.split('_')[0]\n        self.crossover_strategy = self.strategy.split('_')[1]\n    else:\n        self.mutation_strategy = self.crossover_strategy = None\n    self.async_strategy = async_strategy\n    assert self.async_strategy in ['immediate', 'random', 'worst', 'deferred'], \\\n            \"{} is not a valid choice for type of DE\".format(self.async_strategy)\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.AsyncDE.mutation","title":"<code>mutation(current=None, best=None, alt_pop=None)</code>","text":"<p>Performs DE mutation</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def mutation(self, current=None, best=None, alt_pop=None):\n    '''Performs DE mutation\n    '''\n    if self.mutation_strategy == 'rand1':\n        r1, r2, r3 = self._sample_population(size=3, alt_pop=alt_pop, target=current)\n        mutant = self.mutation_rand1(r1, r2, r3)\n\n    elif self.mutation_strategy == 'rand2':\n        r1, r2, r3, r4, r5 = self._sample_population(size=5, alt_pop=alt_pop, target=current)\n        mutant = self.mutation_rand2(r1, r2, r3, r4, r5)\n\n    elif self.mutation_strategy == 'rand2dir':\n        r1, r2, r3 = self._sample_population(size=3, alt_pop=alt_pop, target=current)\n        mutant = self.mutation_rand2dir(r1, r2, r3)\n\n    elif self.mutation_strategy == 'best1':\n        r1, r2 = self._sample_population(size=2, alt_pop=alt_pop, target=current)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_rand1(best, r1, r2)\n\n    elif self.mutation_strategy == 'best2':\n        r1, r2, r3, r4 = self._sample_population(size=4, alt_pop=alt_pop, target=current)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_rand2(best, r1, r2, r3, r4)\n\n    elif self.mutation_strategy == 'currenttobest1':\n        r1, r2 = self._sample_population(size=2, alt_pop=alt_pop, target=current)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_currenttobest1(current, best, r1, r2)\n\n    elif self.mutation_strategy == 'randtobest1':\n        r1, r2, r3 = self._sample_population(size=3, alt_pop=alt_pop, target=current)\n        if best is None:\n            best = self.population[np.argmin(self.fitness)]\n        mutant = self.mutation_currenttobest1(r1, best, r2, r3)\n\n    return mutant\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.AsyncDE.sample_mutants","title":"<code>sample_mutants(size, population=None)</code>","text":"<p>Samples 'size' mutants from the population</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def sample_mutants(self, size, population=None):\n    '''Samples 'size' mutants from the population\n    '''\n    if population is None:\n        population = self.population\n\n    mutants = np.random.uniform(low=0.0, high=1.0, size=(size, self.dimensions))\n    for i in range(size):\n        j = np.random.choice(np.arange(len(population)))\n        mutant = self.mutation(current=population[j], best=self.inc_config, alt_pop=population)\n        mutants[i] = self.boundary_check(mutant)\n\n    return mutants\n</code></pre>"},{"location":"references/de/#dehb.optimizers.de.AsyncDE.evolve_generation","title":"<code>evolve_generation(fidelity=None, best=None, alt_pop=None, **kwargs)</code>","text":"<p>Performs a complete DE evolution, mutation -&gt; crossover -&gt; selection</p> Source code in <code>src/dehb/optimizers/de.py</code> <pre><code>def evolve_generation(self, fidelity=None, best=None, alt_pop=None, **kwargs):\n    '''Performs a complete DE evolution, mutation -&gt; crossover -&gt; selection\n    '''\n    traj = []\n    runtime = []\n    history = []\n\n    if self.async_strategy == \"deferred\":\n        trials = []\n        trial_ids = []\n        for j in range(self.pop_size):\n            target = self.population[j]\n            donor = self.mutation(current=target, best=best, alt_pop=alt_pop)\n            trial = self.crossover(target, donor)\n            trial = self.boundary_check(trial)\n            trial_id = self.config_repository.announce_config(trial, float(fidelity or 0))\n            trials.append(trial)\n            trial_ids.append(trial_id)\n        # selection takes place on a separate trial population only after\n        # one iteration through the population has taken place\n        trials = np.array(trials)\n        traj, runtime, history = self.selection(trials, trial_ids, fidelity, **kwargs)\n        return traj, runtime, history\n\n    elif self.async_strategy == \"immediate\":\n        for i in range(self.pop_size):\n            target = self.population[i]\n            donor = self.mutation(current=target, best=best, alt_pop=alt_pop)\n            trial = self.crossover(target, donor)\n            trial = self.boundary_check(trial)\n            trial_id = self.config_repository.announce_config(trial, float(fidelity or 0))\n            # evaluating a single trial population for the i-th individual\n            de_traj, de_runtime, de_history, fitnesses, costs = \\\n                self.eval_pop(trial.reshape(1, self.dimensions),\n                              np.array([trial_id]), fidelity=fidelity, **kwargs)\n            # one-vs-one selection\n            ## can replace the i-the population despite not completing one iteration\n            if fitnesses[0] &lt;= self.fitness[i]:\n                self.population[i] = trial\n                self.population_ids[i] = trial_id\n                self.fitness[i] = fitnesses[0]\n            traj.extend(de_traj)\n            runtime.extend(de_runtime)\n            history.extend(de_history)\n        return traj, runtime, history\n\n    else:  # async_strategy == 'random' or async_strategy == 'worst':\n        for count in range(self.pop_size):\n            # choosing target individual\n            if self.async_strategy == \"random\":\n                i = np.random.choice(np.arange(self.pop_size))\n            else:  # async_strategy == 'worst'\n                i = np.argsort(-self.fitness)[0]\n            target = self.population[i]\n            mutant = self.mutation(current=target, best=best, alt_pop=alt_pop)\n            trial = self.crossover(target, mutant)\n            trial = self.boundary_check(trial)\n            trial_id = self.config_repository.announce_config(trial, float(fidelity or 0))\n            # evaluating a single trial population for the i-th individual\n            de_traj, de_runtime, de_history, fitnesses, costs = \\\n                self.eval_pop(trial.reshape(1, self.dimensions), np.array([trial_id]),\n                               fidelity=fidelity, **kwargs)\n            # one-vs-one selection\n            ## can replace the i-the population despite not completing one iteration\n            if fitnesses[0] &lt;= self.fitness[i]:\n                self.population[i] = trial\n                self.fitness[i] = fitnesses[0]\n            traj.extend(de_traj)\n            runtime.extend(de_runtime)\n            history.extend(de_history)\n\n    return traj, runtime, history\n</code></pre>"},{"location":"references/dehb/","title":"DEHB","text":""},{"location":"references/dehb/#dehb.optimizers.dehb.DEHBBase","title":"<code>DEHBBase(cs=None, f=None, dimensions=None, mutation_factor=None, crossover_prob=None, strategy=None, min_fidelity=None, max_fidelity=None, eta=None, min_clip=None, max_clip=None, boundary_fix_type='random', max_age=np.inf, **kwargs)</code>","text":"Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None,\n             crossover_prob=None, strategy=None, min_fidelity=None,\n             max_fidelity=None, eta=None, min_clip=None, max_clip=None,\n             boundary_fix_type='random', max_age=np.inf, **kwargs):\n    # Miscellaneous\n    self._setup_logger(kwargs)\n    self.config_repository = ConfigRepository()\n\n    # Benchmark related variables\n    self.cs = cs\n    self.configspace = True if isinstance(self.cs, ConfigSpace.ConfigurationSpace) else False\n    if self.configspace:\n        self.dimensions = len(self.cs.get_hyperparameters())\n    elif dimensions is None or not isinstance(dimensions, (int, np.integer)):\n        assert \"Need to specify `dimensions` as an int when `cs` is not available/specified!\"\n    else:\n        self.dimensions = dimensions\n    self.f = f\n\n    # DE related variables\n    self.mutation_factor = mutation_factor\n    self.crossover_prob = crossover_prob\n    self.strategy = strategy\n    self.fix_type = boundary_fix_type\n    self.max_age = max_age\n    self.de_params = {\n        \"mutation_factor\": self.mutation_factor,\n        \"crossover_prob\": self.crossover_prob,\n        \"strategy\": self.strategy,\n        \"configspace\": self.configspace,\n        \"boundary_fix_type\": self.fix_type,\n        \"max_age\": self.max_age,\n        \"cs\": self.cs,\n        \"dimensions\": self.dimensions,\n        \"f\": f\n    }\n\n    # Hyperband related variables\n    self.min_fidelity = min_fidelity\n    self.max_fidelity = max_fidelity\n    if self.max_fidelity &lt;= self.min_fidelity:\n        self.logger.error(\"Only (Max Fidelity &gt; Min Fidelity) is supported for DEHB.\")\n        if self.max_fidelity == self.min_fidelity:\n            self.logger.error(\n                \"If you have a fixed fidelity, \" \\\n                \"you can instead run DE. For more information checkout: \" \\\n                \"https://automl.github.io/DEHB/references/de\")\n        raise AssertionError()\n    self.eta = eta\n    self.min_clip = min_clip\n    self.max_clip = max_clip\n\n    # Precomputing fidelity spacing and number of configurations for HB iterations\n    self.max_SH_iter = None\n    self.fidelities = None\n    if self.min_fidelity is not None and \\\n       self.max_fidelity is not None and \\\n       self.eta is not None:\n        self.max_SH_iter = -int(np.log(self.min_fidelity / self.max_fidelity) / np.log(self.eta)) + 1\n        self.fidelities = self.max_fidelity * np.power(self.eta,\n                                                 -np.linspace(start=self.max_SH_iter - 1,\n                                                              stop=0, num=self.max_SH_iter))\n\n    # Updating DE parameter list\n    self.de_params.update({\"output_path\": self.output_path})\n\n    # Global trackers\n    self.population = None\n    self.fitness = None\n    self.inc_score = np.inf\n    self.inc_config = None\n    self.history = []\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHBBase.get_next_iteration","title":"<code>get_next_iteration(iteration)</code>","text":"<p>Computes the Successive Halving spacing</p> <p>Given the iteration index, computes the fidelity spacing to be used and the number of configurations to be used for the SH iterations.</p>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHBBase.get_next_iteration--parameters","title":"Parameters","text":"<p>iteration : int     Iteration index clip : int, {1, 2, 3, ..., None}     If not None, clips the minimum number of configurations to 'clip'</p>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHBBase.get_next_iteration--returns","title":"Returns","text":"<p>ns : array fidelities : array</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def get_next_iteration(self, iteration):\n    '''Computes the Successive Halving spacing\n\n    Given the iteration index, computes the fidelity spacing to be used and\n    the number of configurations to be used for the SH iterations.\n\n    Parameters\n    ----------\n    iteration : int\n        Iteration index\n    clip : int, {1, 2, 3, ..., None}\n        If not None, clips the minimum number of configurations to 'clip'\n\n    Returns\n    -------\n    ns : array\n    fidelities : array\n    '''\n    # number of 'SH runs'\n    s = self.max_SH_iter - 1 - (iteration % self.max_SH_iter)\n    # fidelity spacing for this iteration\n    fidelities = self.fidelities[(-s-1):]\n    # number of configurations in that bracket\n    n0 = int(np.floor((self.max_SH_iter)/(s+1)) * self.eta**s)\n    ns = [max(int(n0*(self.eta**(-i))), 1) for i in range(s+1)]\n    if self.min_clip is not None and self.max_clip is not None:\n        ns = np.clip(ns, a_min=self.min_clip, a_max=self.max_clip)\n    elif self.min_clip is not None:\n        ns = np.clip(ns, a_min=self.min_clip, a_max=np.max(ns))\n\n    return ns, fidelities\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHBBase.get_incumbents","title":"<code>get_incumbents()</code>","text":"<p>Returns a tuple of the (incumbent configuration, incumbent score/fitness).</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def get_incumbents(self):\n    \"\"\" Returns a tuple of the (incumbent configuration, incumbent score/fitness). \"\"\"\n    if self.configspace:\n        return self.vector_to_configspace(self.inc_config), self.inc_score\n    return self.inc_config, self.inc_score\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB","title":"<code>DEHB(cs=None, f=None, dimensions=None, mutation_factor=0.5, crossover_prob=0.5, strategy='rand1_bin', min_fidelity=None, max_fidelity=None, eta=3, min_clip=None, max_clip=None, configspace=True, boundary_fix_type='random', max_age=np.inf, n_workers=None, client=None, async_strategy='immediate', **kwargs)</code>","text":"<p>             Bases: <code>DEHBBase</code></p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=0.5,\n             crossover_prob=0.5, strategy='rand1_bin', min_fidelity=None,\n             max_fidelity=None, eta=3, min_clip=None, max_clip=None, configspace=True,\n             boundary_fix_type='random', max_age=np.inf, n_workers=None, client=None,\n             async_strategy=\"immediate\", **kwargs):\n    super().__init__(cs=cs, f=f, dimensions=dimensions, mutation_factor=mutation_factor,\n                     crossover_prob=crossover_prob, strategy=strategy, min_fidelity=min_fidelity,\n                     max_fidelity=max_fidelity, eta=eta, min_clip=min_clip, max_clip=max_clip,\n                     configspace=configspace, boundary_fix_type=boundary_fix_type,\n                     max_age=max_age, **kwargs)\n    self.de_params.update({\"async_strategy\": async_strategy})\n    self.iteration_counter = -1\n    self.de = {}\n    self._max_pop_size = None\n    self.active_brackets = []  # list of SHBracketManager objects\n    self.traj = []\n    self.runtime = []\n    self.history = []\n    self.start = None\n\n    # Dask variables\n    if n_workers is None and client is None:\n        raise ValueError(\"Need to specify either 'n_workers'(&gt;0) or 'client' (a Dask client)!\")\n    if client is not None and isinstance(client, Client):\n        self.client = client\n        self.n_workers = len(client.ncores())\n    else:\n        self.n_workers = n_workers\n        if self.n_workers &gt; 1:\n            self.client = Client(\n                n_workers=self.n_workers, processes=True, threads_per_worker=1, scheduler_port=0\n            )  # port 0 makes Dask select a random free port\n        else:\n            self.client = None\n    self.futures = []\n    self.shared_data = None\n\n    # Initializing DE subpopulations\n    self._get_pop_sizes()\n    self._init_subpop()\n\n    # Misc.\n    self.available_gpus = None\n    self.gpu_usage = None\n    self.single_node_with_gpus = None\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.__getstate__","title":"<code>__getstate__()</code>","text":"<p>Allows the object to picklable while having Dask client as a class attribute.</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def __getstate__(self):\n    \"\"\" Allows the object to picklable while having Dask client as a class attribute.\n    \"\"\"\n    d = dict(self.__dict__)\n    d[\"client\"] = None  # hack to allow Dask client to be a class attribute\n    d[\"logger\"] = None  # hack to allow logger object to be a class attribute\n    return d\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.__del__","title":"<code>__del__()</code>","text":"<p>Ensures a clean kill of the Dask client and frees up a port.</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def __del__(self):\n    \"\"\" Ensures a clean kill of the Dask client and frees up a port.\n    \"\"\"\n    if hasattr(self, \"client\") and isinstance(self, Client):\n        self.client.close()\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.distribute_gpus","title":"<code>distribute_gpus()</code>","text":"<p>Function to create a GPU usage tracker dict.</p> <p>The idea is to extract the exact GPU device IDs available. During job submission, each submitted job is given a preference of a GPU device ID based on the GPU device with the least number of active running jobs. On retrieval of the result, this gpu usage dict is updated for the device ID that the finished job was mapped to.</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def distribute_gpus(self):\n    \"\"\" Function to create a GPU usage tracker dict.\n\n    The idea is to extract the exact GPU device IDs available. During job submission, each\n    submitted job is given a preference of a GPU device ID based on the GPU device with the\n    least number of active running jobs. On retrieval of the result, this gpu usage dict is\n    updated for the device ID that the finished job was mapped to.\n    \"\"\"\n    try:\n        available_gpus = os.environ[\"CUDA_VISIBLE_DEVICES\"]\n        available_gpus = available_gpus.strip().split(\",\")\n        self.available_gpus = [int(_id) for _id in available_gpus]\n    except KeyError as e:\n        print(\"Unable to find valid GPU devices. \"\n              \"Environment variable {} not visible!\".format(str(e)))\n        self.available_gpus = []\n    self.gpu_usage = dict()\n    for _id in self.available_gpus:\n        self.gpu_usage[_id] = 0\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.clean_inactive_brackets","title":"<code>clean_inactive_brackets()</code>","text":"<p>Removes brackets from the active list if it is done as communicated by Bracket Manager</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def clean_inactive_brackets(self):\n    \"\"\" Removes brackets from the active list if it is done as communicated by Bracket Manager\n    \"\"\"\n    if len(self.active_brackets) == 0:\n        return\n    self.active_brackets = [\n        bracket for bracket in self.active_brackets if ~bracket.is_bracket_done()\n    ]\n    return\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.is_worker_available","title":"<code>is_worker_available(verbose=False)</code>","text":"<p>Checks if at least one worker is available to run a job</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def is_worker_available(self, verbose=False):\n    \"\"\" Checks if at least one worker is available to run a job\n    \"\"\"\n    if self.n_workers == 1 or self.client is None or not isinstance(self.client, Client):\n        # in the synchronous case, one worker is always available\n        return True\n    # checks the absolute number of workers mapped to the client scheduler\n    # client.ncores() should return a dict with the keys as unique addresses to these workers\n    # treating the number of available workers in this manner\n    workers = self._get_worker_count()  # len(self.client.ncores())\n    if len(self.futures) &gt;= workers:\n        # pause/wait if active worker count greater allocated workers\n        return False\n    return True\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.ask","title":"<code>ask(n_configs=1)</code>","text":"<p>Get the next configuration to run from the optimizer.</p> <p>The retrieved configuration can then be evaluated by the user. After evaluation use <code>tell</code> to report the results back to the optimizer. For more information, please refer to the description of <code>tell</code>.</p> PARAMETER  DESCRIPTION <code>n_configs</code> <p>Number of configs to ask for. Defaults to 1.</p> <p> TYPE: <code>int</code> DEFAULT: <code>1</code> </p> RETURNS DESCRIPTION <p>dict or list of dict: Job info(s) of next configuration to evaluate.</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def ask(self, n_configs: int=1):\n    \"\"\"Get the next configuration to run from the optimizer.\n\n    The retrieved configuration can then be evaluated by the user.\n    After evaluation use `tell` to report the results back to the optimizer.\n    For more information, please refer to the description of `tell`.\n\n    Args:\n        n_configs (int, optional): Number of configs to ask for. Defaults to 1.\n\n    Returns:\n        dict or list of dict: Job info(s) of next configuration to evaluate.\n    \"\"\"\n    if n_configs == 1:\n        return self._get_next_job()\n\n    jobs = []\n    for _ in range(n_configs):\n        jobs.append(self._get_next_job())\n    return jobs\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.submit_job","title":"<code>submit_job(job_info, **kwargs)</code>","text":"<p>Asks a free worker to run the objective function on config and fidelity</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def submit_job(self, job_info, **kwargs):\n    \"\"\" Asks a free worker to run the objective function on config and fidelity\n    \"\"\"\n    job_info[\"kwargs\"] = self.shared_data if self.shared_data is not None else kwargs\n    # submit to Dask client\n    if self.n_workers &gt; 1 or isinstance(self.client, Client):\n        if self.single_node_with_gpus:\n            # managing GPU allocation for the job to be submitted\n            job_info.update({\"gpu_devices\": self._get_gpu_id_with_low_load()})\n        self.futures.append(\n            self.client.submit(self._f_objective, job_info)\n        )\n    else:\n        # skipping scheduling to Dask worker to avoid added overheads in the synchronous case\n        self.futures.append(self._f_objective(job_info))\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.tell","title":"<code>tell(job_info, result)</code>","text":"<p>Feed a result back to the optimizer.</p> <p>In order to correctly interpret the results, the <code>job_info</code> dict, retrieved by <code>ask</code>, has to be given. Moreover, the <code>result</code> dict has to contain the keys <code>fitness</code> and <code>cost</code>. It is also possible to add the field <code>info</code> to the <code>result</code> in order to store additional, user-specific information.</p> PARAMETER  DESCRIPTION <code>job_info</code> <p>Job info returned by ask().</p> <p> TYPE: <code>dict</code> </p> <code>result</code> <p>Result dictionary with mandatory keys <code>fitness</code> and <code>cost</code>.</p> <p> TYPE: <code>dict</code> </p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>def tell(self, job_info: dict, result: dict):\n    \"\"\"Feed a result back to the optimizer.\n\n    In order to correctly interpret the results, the `job_info` dict, retrieved by `ask`,\n    has to be given. Moreover, the `result` dict has to contain the keys `fitness` and `cost`.\n    It is also possible to add the field `info` to the `result` in order to store additional,\n    user-specific information.\n\n    Args:\n        job_info (dict): Job info returned by ask().\n        result (dict): Result dictionary with mandatory keys `fitness` and `cost`.\n    \"\"\"\n    # update bracket information\n    fitness, cost = result[\"fitness\"], result[\"cost\"]\n    info = result[\"info\"] if \"info\" in result else dict()\n    fidelity, parent_id = job_info[\"fidelity\"], job_info[\"parent_id\"]\n    config, config_id = job_info[\"config\"], job_info[\"config_id\"]\n    bracket_id = job_info[\"bracket_id\"]\n    for bracket in self.active_brackets:\n        if bracket.bracket_id == bracket_id:\n            # bracket job complete\n            bracket.complete_job(fidelity)  # IMPORTANT to perform synchronous SH\n\n    self.config_repository.tell_result(config_id, fidelity, fitness, cost, info)\n\n    # get hypercube representation from config repo\n    if self.configspace:\n        config = self.config_repository.get(config_id)\n\n    # carry out DE selection\n    if fitness &lt;= self.de[fidelity].fitness[parent_id]:\n        self.de[fidelity].population[parent_id] = config\n        self.de[fidelity].population_ids[parent_id] = config_id\n        self.de[fidelity].fitness[parent_id] = fitness\n    # updating incumbents\n    if self.de[fidelity].fitness[parent_id] &lt; self.inc_score:\n        self._update_incumbents(\n            config=self.de[fidelity].population[parent_id],\n            score=self.de[fidelity].fitness[parent_id],\n            info=info\n        )\n    # book-keeping\n    self._update_trackers(\n        traj=self.inc_score, runtime=cost, history=(\n            config.tolist(), float(fitness), float(cost), float(fidelity), info\n        )\n    )\n</code></pre>"},{"location":"references/dehb/#dehb.optimizers.dehb.DEHB.run","title":"<code>run(fevals=None, brackets=None, total_cost=None, single_node_with_gpus=False, verbose=False, debug=False, save_intermediate=True, save_history=True, name=None, **kwargs)</code>","text":"<p>Main interface to run optimization by DEHB</p> <p>This function waits on workers and if a worker is free, asks for a configuration and a fidelity to evaluate on and submits it to the worker. In each loop, it checks if a job is complete, fetches the results, carries the necessary processing of it asynchronously to the worker computations.</p> <p>The duration of the DEHB run can be controlled by specifying one of 3 parameters. If more than one are specified, DEHB selects only one in the priority order (high to low): 1) Number of function evaluations (fevals) 2) Number of Successive Halving brackets run under Hyperband (brackets) 3) Total computational cost (in seconds) aggregated by all function evaluations (total_cost)</p> Source code in <code>src/dehb/optimizers/dehb.py</code> <pre><code>@logger.catch\ndef run(self, fevals=None, brackets=None, total_cost=None, single_node_with_gpus=False,\n        verbose=False, debug=False, save_intermediate=True, save_history=True, name=None, **kwargs):\n    \"\"\" Main interface to run optimization by DEHB\n\n    This function waits on workers and if a worker is free, asks for a configuration and a\n    fidelity to evaluate on and submits it to the worker. In each loop, it checks if a job\n    is complete, fetches the results, carries the necessary processing of it asynchronously\n    to the worker computations.\n\n    The duration of the DEHB run can be controlled by specifying one of 3 parameters. If more\n    than one are specified, DEHB selects only one in the priority order (high to low):\n    1) Number of function evaluations (fevals)\n    2) Number of Successive Halving brackets run under Hyperband (brackets)\n    3) Total computational cost (in seconds) aggregated by all function evaluations (total_cost)\n    \"\"\"\n    # check if run has already been called before\n    if self.start is not None:\n        logger.warning(\"DEHB has already been run. Calling 'run' twice could lead to unintended\"\n                       + \" behavior. Please restart DEHB with an increased compute budget\"\n                       + \" instead of calling 'run' twice.\")\n\n    # checks if a Dask client exists\n    if len(kwargs) &gt; 0 and self.n_workers &gt; 1 and isinstance(self.client, Client):\n        # broadcasts all additional data passed as **kwargs to all client workers\n        # this reduces overload in the client-worker communication by not having to\n        # serialize the redundant data used by all workers for every job\n        self.shared_data = self.client.scatter(kwargs, broadcast=True)\n\n    # allows each worker to be mapped to a different GPU when running on a single node\n    # where all available GPUs are accessible\n    self.single_node_with_gpus = single_node_with_gpus\n    if self.single_node_with_gpus:\n        self.distribute_gpus()\n\n    self.start = self.start = time.time()\n    fevals, brackets = self._adjust_budgets(fevals, brackets)\n    if verbose:\n        print(\"\\nLogging at {} for optimization starting at {}\\n\".format(\n            os.path.join(os.getcwd(), self.log_filename),\n            time.strftime(\"%x %X %Z\", time.localtime(self.start))\n        ))\n    if debug:\n        logger.configure(handlers=[{\"sink\": sys.stdout}])\n    while True:\n        if self._is_run_budget_exhausted(fevals, brackets, total_cost):\n            break\n        if self.is_worker_available():\n            job_info = self.ask()\n            if brackets is not None and job_info[\"bracket_id\"] &gt;= brackets:\n                # ignore submission and only collect results\n                # when brackets are chosen as run budget, an extra bracket is created\n                # since iteration_counter is incremented in ask() and then checked\n                # in _is_run_budget_exhausted(), therefore, need to skip suggestions\n                # coming from the extra allocated bracket\n                # _is_run_budget_exhausted() will not return True until all the lower brackets\n                # have finished computation and returned its results\n                pass\n            else:\n                if self.n_workers &gt; 1 or isinstance(self.client, Client):\n                    self.logger.debug(\"{}/{} worker(s) available.\".format(\n                        self._get_worker_count() - len(self.futures), self._get_worker_count()\n                    ))\n                # submits job_info to a worker for execution\n                self.submit_job(job_info, **kwargs)\n                if verbose:\n                    fidelity = job_info[\"fidelity\"]\n                    config_id = job_info[\"config_id\"]\n                    self._verbosity_runtime(fevals, brackets, total_cost)\n                    self.logger.info(\n                        \"Evaluating configuration {} with fidelity {} under \"\n                        \"bracket ID {}\".format(config_id, fidelity, job_info[\"bracket_id\"])\n                    )\n                    self.logger.info(\n                        \"Best score seen/Incumbent score: {}\".format(self.inc_score)\n                    )\n                self._verbosity_debug()\n        self._fetch_results_from_workers()\n        if save_intermediate and self.inc_config is not None:\n            self._save_incumbent(name)\n        if save_history and self.history is not None:\n            self._save_history(name)\n        self.clean_inactive_brackets()\n    # end of while\n\n    if verbose and len(self.futures) &gt; 0:\n        self.logger.info(\n            \"DEHB optimisation over! Waiting to collect results from workers running...\"\n        )\n    while len(self.futures) &gt; 0:\n        self._fetch_results_from_workers()\n        if save_intermediate and self.inc_config is not None:\n            self._save_incumbent(name)\n        if save_history and self.history is not None:\n            self._save_history(name)\n        time.sleep(0.05)  # waiting 50ms\n\n    if verbose:\n        time_taken = time.time() - self.start\n        self.logger.info(\"End of optimisation! Total duration: {}; Total fevals: {}\\n\".format(\n            time_taken, len(self.traj)\n        ))\n        self.logger.info(\"Incumbent score: {}\".format(self.inc_score))\n        self.logger.info(\"Incumbent config: \")\n        if self.configspace:\n            config = self.vector_to_configspace(self.inc_config)\n            for k, v in config.get_dictionary().items():\n                self.logger.info(\"{}: {}\".format(k, v))\n        else:\n            self.logger.info(\"{}\".format(self.inc_config))\n    self._save_incumbent(name)\n    self._save_history(name)\n    # reset waiting jobs of active bracket to allow for continuation\n    if len(self.active_brackets) &gt; 0:\n        for active_bracket in self.active_brackets:\n            active_bracket.reset_waiting_jobs()\n    return np.array(self.traj), np.array(self.runtime), np.array(self.history, dtype=object)\n</code></pre>"}]}
\ No newline at end of file
diff --git a/0.1.0/sitemap.xml.gz b/0.1.0/sitemap.xml.gz
index e2770f3fcfd58a870093f5881099aa9e6c81cbad..0d6ef0df2d9a2393bc4dc685a14feab88ee56619 100644
GIT binary patch
delta 12
Tcmb=gXOr*d;F$b(B3mT@8j}QH

delta 12
Tcmb=gXOr*d;P5;?k*yK{7)b-)