Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for sigma=0 in normally distributed hyperparameters #285

Closed
wants to merge 7 commits into from
Closed

Add support for sigma=0 in normally distributed hyperparameters #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0e41484
Add support for sigma=0 in normal distributions
nchristensen Jan 5, 2023
e07f87b
Update docs to clarify float now allowed
nchristensen Jan 9, 2023
d6a172a
Base normalized_default_value on the original unrounded default value…
nchristensen Jan 9, 2023
c89bc7a
Update NormalIntegerHyperparameter tests
nchristensen Jan 9, 2023
2ab524f
Don't convert normalized_default_value to int'
nchristensen Jan 9, 2023
a8b1d6d
Merge branch 'main' into enable-sigma-0
nchristensen Jan 13, 2023
d4f6bcc
Fix type conversion from boolean numpy array to float64 numpy array
nchristensen Oct 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 65 additions & 61 deletions ConfigSpace/hyperparameters.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ cdef class Hyperparameter(object):
def has_neighbors(self):
raise NotImplementedError()

def get_neighbors(self, value, rs, number, transform = False):
def get_neighbors(self, value, rs, number, transform=False):
raise NotImplementedError()

def get_num_neighbors(self, value):
Expand Down Expand Up @@ -314,7 +314,7 @@ cdef class Constant(Hyperparameter):
def has_neighbors(self) -> bool:
return False

def get_num_neighbors(self, value = None) -> int:
def get_num_neighbors(self, value=None) -> int:
return 0

def get_neighbors(self, value: Any, rs: np.random.RandomState, number: int,
Expand Down Expand Up @@ -385,7 +385,7 @@ cdef class NumericalHyperparameter(Hyperparameter):
def has_neighbors(self) -> bool:
return True

def get_num_neighbors(self, value = None) -> float:
def get_num_neighbors(self, value=None) -> float:

return np.inf

Expand Down Expand Up @@ -539,7 +539,6 @@ cdef class FloatHyperparameter(NumericalHyperparameter):
raise NotImplementedError()



cdef class IntegerHyperparameter(NumericalHyperparameter):
def __init__(self, name: str, default_value: int, meta: Optional[Dict] = None) -> None:
super(IntegerHyperparameter, self).__init__(name, default_value, meta)
Expand All @@ -555,7 +554,7 @@ cdef class IntegerHyperparameter(NumericalHyperparameter):

def check_int(self, parameter: int, name: str) -> int:
if abs(int(parameter) - parameter) > 0.00000001 and \
type(parameter) is not int:
type(parameter) is not int:
raise ValueError("For the Integer parameter %s, the value must be "
"an Integer, too. Right now it is a %s with value"
" %s." % (name, type(parameter), str(parameter)))
Expand Down Expand Up @@ -912,20 +911,21 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
self.normalized_default_value = self._inverse_transform(self.default_value)

if (lower is not None) ^ (upper is not None):
raise ValueError("Only one bound was provided when both lower and upper bounds must be provided.")
raise ValueError(
"Only one bound was provided when both lower and upper bounds must be provided.")

if lower is not None and upper is not None:
self.lower = float(lower)
self.upper = float(upper)

if self.lower >= self.upper:
raise ValueError("Upper bound %f must be larger than lower bound "
"%f for hyperparameter %s" %
(self.upper, self.lower, name))
"%f for hyperparameter %s" %
(self.upper, self.lower, name))
elif log and self.lower <= 0:
raise ValueError("Negative lower bound (%f) for log-scale "
"hyperparameter %s is forbidden." %
(self.lower, name))
"hyperparameter %s is forbidden." %
(self.lower, name))

self.default_value = self.check_default(default_value)

Expand Down Expand Up @@ -959,9 +959,11 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
repr_str = io.StringIO()

if self.lower is None or self.upper is None:
repr_str.write("%s, Type: NormalFloat, Mu: %s Sigma: %s, Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.default_value)))
repr_str.write("%s, Type: NormalFloat, Mu: %s Sigma: %s, Default: %s" %
(self.name, repr(self.mu), repr(self.sigma), repr(self.default_value)))
else:
repr_str.write("%s, Type: NormalFloat, Mu: %s Sigma: %s, Range: [%s, %s], Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.lower), repr(self.upper), repr(self.default_value)))
repr_str.write("%s, Type: NormalFloat, Mu: %s Sigma: %s, Range: [%s, %s], Default: %s" % (
self.name, repr(self.mu), repr(self.sigma), repr(self.lower), repr(self.upper), repr(self.default_value)))

if self.log:
repr_str.write(", on log-scale")
Expand Down Expand Up @@ -1048,8 +1050,8 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
lower = None
upper = None
else:
lower=np.ceil(self.lower)
upper=np.floor(self.upper)
lower = np.ceil(self.lower)
upper = np.floor(self.upper)

return NormalIntegerHyperparameter(self.name, int(np.rint(self.mu)), self.sigma,
lower=lower, upper=upper,
Expand All @@ -1065,13 +1067,14 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
def _sample(self, rs: np.random.RandomState, size: Optional[int] = None
) -> Union[np.ndarray, float]:

if self.lower == None:
sigma = self.sigma
if sigma == 0:
return self.mu
elif self.lower == None:
mu = self.mu
sigma = self.sigma
return rs.normal(mu, sigma, size=size)
else:
mu = self.mu
sigma = self.sigma
lower = self._lower
upper = self._upper
a = (lower - mu) / sigma
Expand Down Expand Up @@ -1112,7 +1115,7 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
new_value = rs.normal(value, self.sigma)

if self.lower is not None and self.upper is not None:
new_value = min(max(new_value, self.lower), self.upper)
new_value = min(max(new_value, self.lower), self.upper)

neighbors.append(new_value)
return neighbors
Expand Down Expand Up @@ -1146,7 +1149,9 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
"""
mu = self.mu
sigma = self.sigma
if self.lower == None:
if sigma == 0:
return (vector == mu).astype(np.float64)
elif self.lower == None:
return norm(loc=mu, scale=sigma).pdf(vector)
else:
mu = self.mu
Expand Down Expand Up @@ -1240,7 +1245,8 @@ cdef class BetaFloatHyperparameter(UniformFloatHyperparameter):

def __repr__(self) -> str:
repr_str = io.StringIO()
repr_str.write("%s, Type: BetaFloat, Alpha: %s Beta: %s, Range: [%s, %s], Default: %s" % (self.name, repr(self.alpha), repr(self.beta), repr(self.lower), repr(self.upper), repr(self.default_value)))
repr_str.write("%s, Type: BetaFloat, Alpha: %s Beta: %s, Range: [%s, %s], Default: %s" % (
self.name, repr(self.alpha), repr(self.beta), repr(self.lower), repr(self.upper), repr(self.default_value)))

if self.log:
repr_str.write(", on log-scale")
Expand Down Expand Up @@ -1331,15 +1337,14 @@ cdef class BetaFloatHyperparameter(UniformFloatHyperparameter):
upper = int(np.floor(self.upper))
default_value = int(np.rint(self.default_value))
return BetaIntegerHyperparameter(self.name, lower=lower, upper=upper, alpha=self.alpha, beta=self.beta,
default_value=int(np.rint(self.default_value)),
q=q_int, log=self.log)
default_value=int(np.rint(self.default_value)),
q=q_int, log=self.log)

def is_legal(self, value: Union[float]) -> bool:
if isinstance(value, (float, int)):
return self.upper >= value >= self.lower
return False


cpdef bint is_legal_vector(self, DTYPE_t value):
return self._upper >= value >= self._lower

Expand Down Expand Up @@ -1373,7 +1378,7 @@ cdef class BetaFloatHyperparameter(UniformFloatHyperparameter):
alpha = self.alpha
beta = self.beta
return spbeta(alpha, beta, loc=lb, scale=ub-lb).pdf(vector) \
* (ub-lb) / (self._upper - self._lower)
* (ub-lb) / (self._upper - self._lower)

def get_max_density(self) -> float:
if (self.alpha > 1) or (self.beta > 1):
Expand Down Expand Up @@ -1554,13 +1559,8 @@ cdef class UniformIntegerHyperparameter(IntegerHyperparameter):
else:
return False

def get_num_neighbors(self, value = None) -> int:
# If there is a value in the range, then that value is not a neighbor of itself
# so we need to remove one
if value is not None and self.lower <= value <= self.upper:
return self.upper - self.lower - 1
else:
return self.upper - self.lower
def get_num_neighbors(self, value=None) -> int:
return self.upper - self.lower

def get_neighbors(
self,
Expand Down Expand Up @@ -1726,8 +1726,7 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
cdef public nfhp
cdef normalization_constant


def __init__(self, name: str, mu: int, sigma: Union[int, float],
def __init__(self, name: str, mu: Union[int, float], sigma: Union[int, float],
nchristensen marked this conversation as resolved.
Show resolved Hide resolved
default_value: Union[int, None] = None, q: Union[None, int] = None,
log: bool = False,
lower: Optional[int] = None,
Expand All @@ -1748,7 +1747,7 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
----------
name : str
Name of the hyperparameter with which it can be accessed
mu : int
mu : int, float
Mean of the distribution, from which hyperparameter is sampled
sigma : int, float
Standard deviation of the distribution, from which
Expand All @@ -1773,6 +1772,8 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):

self.mu = mu
self.sigma = sigma
if self.sigma == 0:
assert int(self.mu) == self.mu

if default_value is not None:
default_value = self.check_int(default_value, self.name)
Expand All @@ -1790,19 +1791,20 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
self.log = bool(log)

if (lower is not None) ^ (upper is not None):
raise ValueError("Only one bound was provided when both lower and upper bounds must be provided.")
raise ValueError(
"Only one bound was provided when both lower and upper bounds must be provided.")

if lower is not None and upper is not None:
self.upper = self.check_int(upper, "upper")
self.lower = self.check_int(lower, "lower")
if self.lower >= self.upper:
raise ValueError("Upper bound %d must be larger than lower bound "
"%d for hyperparameter %s" %
(self.lower, self.upper, name))
"%d for hyperparameter %s" %
(self.lower, self.upper, name))
elif log and self.lower <= 0:
raise ValueError("Negative lower bound (%d) for log-scale "
"hyperparameter %s is forbidden." %
(self.lower, name))
"hyperparameter %s is forbidden." %
(self.lower, name))
self.lower = lower
self.upper = upper

Expand All @@ -1828,9 +1830,11 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
repr_str = io.StringIO()

if self.lower is None or self.upper is None:
repr_str.write("%s, Type: NormalInteger, Mu: %s Sigma: %s, Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.default_value)))
repr_str.write("%s, Type: NormalInteger, Mu: %s Sigma: %s, Default: %s" %
(self.name, repr(self.mu), repr(self.sigma), repr(self.default_value)))
else:
repr_str.write("%s, Type: NormalInteger, Mu: %s Sigma: %s, Range: [%s, %s], Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.lower), repr(self.upper), repr(self.default_value)))
repr_str.write("%s, Type: NormalInteger, Mu: %s Sigma: %s, Range: [%s, %s], Default: %s" % (
self.name, repr(self.mu), repr(self.sigma), repr(self.lower), repr(self.upper), repr(self.default_value)))

if self.log:
repr_str.write(", on log-scale")
Expand Down Expand Up @@ -1907,7 +1911,7 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
if self.log:
return self._transform_scalar(self.mu)
else:
return self.mu
return int(np.round(self.mu))

elif self.is_legal(default_value):
return default_value
Expand Down Expand Up @@ -2008,7 +2012,7 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
def _compute_normalization(self):
if self.lower is None:
warnings.warn('Cannot normalize the pdf exactly for a NormalIntegerHyperparameter'
f' {self.name} without bounds. Skipping normalization for that hyperparameter.')
f' {self.name} without bounds. Skipping normalization for that hyperparameter.')
return 1

else:
Expand Down Expand Up @@ -2060,7 +2064,6 @@ cdef class BetaIntegerHyperparameter(UniformIntegerHyperparameter):
cdef public bfhp
cdef normalization_constant


def __init__(self, name: str, alpha: Union[int, float], beta: Union[int, float],
lower: Union[int, float],
upper: Union[int, float],
Expand Down Expand Up @@ -2118,21 +2121,22 @@ cdef class BetaIntegerHyperparameter(UniformIntegerHyperparameter):
else:
q = self.q
self.bfhp = BetaFloatHyperparameter(self.name,
self.alpha,
self.beta,
log=self.log,
q=q,
lower=self.lower,
upper=self.upper,
default_value=self.default_value)
self.alpha,
self.beta,
log=self.log,
q=q,
lower=self.lower,
upper=self.upper,
default_value=self.default_value)

self.default_value = self.check_default(default_value)
self.normalized_default_value = self._inverse_transform(self.default_value)
self.normalization_constant = self._compute_normalization()

def __repr__(self) -> str:
repr_str = io.StringIO()
repr_str.write("%s, Type: BetaInteger, Alpha: %s Beta: %s, Range: [%s, %s], Default: %s" % (self.name, repr(self.alpha), repr(self.beta), repr(self.lower), repr(self.upper), repr(self.default_value)))
repr_str.write("%s, Type: BetaInteger, Alpha: %s Beta: %s, Range: [%s, %s], Default: %s" % (
self.name, repr(self.alpha), repr(self.beta), repr(self.lower), repr(self.upper), repr(self.default_value)))

if self.log:
repr_str.write(", on log-scale")
Expand Down Expand Up @@ -2190,7 +2194,6 @@ cdef class BetaIntegerHyperparameter(UniformIntegerHyperparameter):
default_value=self.default_value,
q=self.q, log=self.log, meta=self.meta)


def check_default(self, default_value: Union[int, float, None]) -> int:
if default_value is None:
# Here, we just let the BetaFloat take care of the default value
Expand Down Expand Up @@ -2389,7 +2392,7 @@ cdef class CategoricalHyperparameter(Hyperparameter):
ordered_probabilities_other is None
and len(np.unique(list(ordered_probabilities_self.values()))) == 1
)
)
)
)

def __hash__(self):
Expand Down Expand Up @@ -2514,7 +2517,7 @@ cdef class CategoricalHyperparameter(Hyperparameter):
def has_neighbors(self) -> bool:
return len(self.choices) > 1

def get_num_neighbors(self, value = None) -> int:
def get_num_neighbors(self, value=None) -> int:
return len(self.choices) - 1

def get_neighbors(self, value: int, rs: np.random.RandomState,
Expand Down Expand Up @@ -2724,11 +2727,11 @@ cdef class OrdinalHyperparameter(Hyperparameter):

def __copy__(self):
return OrdinalHyperparameter(
name=self.name,
sequence=copy.deepcopy(self.sequence),
default_value=self.default_value,
meta=self.meta
)
name=self.name,
sequence=copy.deepcopy(self.sequence),
default_value=self.default_value,
meta=self.meta
)

cpdef int compare(self, value: Union[int, float, str], value2: Union[int, float, str]):
if self.value_dict[value] < self.value_dict[value2]:
Expand Down Expand Up @@ -2948,7 +2951,8 @@ cdef class OrdinalHyperparameter(Hyperparameter):
Probability density values of the input vector
"""
if not np.all(np.isin(vector, self.sequence)):
raise ValueError(f'Some element in the vector {vector} is not in the sequence {self.sequence}.')
raise ValueError(
f'Some element in the vector {vector} is not in the sequence {self.sequence}.')
return np.ones_like(vector, dtype=np.float64) / self.num_elements

def get_max_density(self) -> float:
Expand Down
7 changes: 4 additions & 3 deletions test/test_hyperparameters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1193,8 +1193,9 @@ def test_normalint(self):
f1 = NormalIntegerHyperparameter("param", 0.5, 5.5)
f1_ = NormalIntegerHyperparameter("param", 0.5, 5.5)
self.assertEqual(f1, f1_)
default = np.int32(np.round(0.5))
self.assertEqual(
"param, Type: NormalInteger, Mu: 0.5 Sigma: 5.5, Default: 0.5",
f"param, Type: NormalInteger, Mu: 0.5 Sigma: 5.5, Default: {default}",
str(f1))

# Test attributes are accessible
Expand All @@ -1203,8 +1204,8 @@ def test_normalint(self):
self.assertEqual(f1.sigma, 5.5)
self.assertEqual(f1.q, None)
self.assertEqual(f1.log, False)
self.assertAlmostEqual(f1.default_value, 0.5)
self.assertAlmostEqual(f1.normalized_default_value, 0.5)
self.assertAlmostEqual(f1.default_value, default)
self.assertAlmostEqual(f1.normalized_default_value, 0.0)

with pytest.warns(UserWarning, match="Setting quantization < 1 for Integer "
"Hyperparameter 'param' has no effect"):
Expand Down