Skip to content

Commit

Permalink
Fides 0.6.3 (#41)
Browse files Browse the repository at this point in the history 
* make 2D default subspace method

* version bump

* add additional hybrid methods

* version bump

* fix flake

* increase tolerances for fletcher test

* fixup

* fix broyden methods

* fixups

* doc update

* fixup

* fixup

* add more tests

* fix flake

* fixup

* add/fix pragmas

* refactor broyden/bfgs/dfp, add references

* fix error

* fix nan values in hessian

* version bump

* add reference, fixes #36

* fix flake

* update for 3.10

* fix 3.10

* update readme

* wait with 3.10 until scipy gets it shit together

* don't bump requirement just yet

* better nan guard in bfgs

* fix initialization

* fix test

* fix structured methods

* fix tssm

* fix GNSBFGS

* fix SSM, TSSM, GNSBFGS

* fix hybridfixed approach

* fixups

* fix hybrid schemes

* use citeall zenodo doi

* Update hessian_approximation.py

* fix and simplify structured methods

* fixups

* Update test_hessian_approximation.py

* Update test_minimize.py

* remove PSB

* Update test_hessian_approximation.py

* Update test_hessian_approximation.py

* simplify
  • Loading branch information
@FFroehlich
FFroehlich authored Oct 27, 2021
1 parent 099164f commit f481387
Show file tree
Hide file tree
Showing 7 changed files with 48 additions and 130 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ Fides can be installed via `pip install fides`. Further documentation is
* Boundary constrained and unconstrained interior trust-region optimization
* Reflective, truncated and optimization based boundary heuristics
* Exact, 2D and CG subproblem solvers
* BFGS, DFP, SR1, PSB, Broyden (good and bad) and Broyden class iterative
* BFGS, DFP, SR1, Broyden (good and bad) and Broyden class iterative
Hessian Approximation schemes
* SSM, TSSM, FX, GNSBFGS and custom hybrid Hessian Approximations schemes

2 changes: 1 addition & 1 deletion fides/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# flake8: noqa
from .minimize import Optimizer
from .hessian_approximation import (
SR1, BFGS, DFP, FX, HybridFixed, GNSBFGS, BB, BG, Broyden, PSB, SSM, TSSM
SR1, BFGS, DFP, FX, HybridFixed, GNSBFGS, BB, BG, Broyden, SSM, TSSM
)
from .logging import create_logger
from .version import __version__
Expand Down
113 changes: 26 additions & 87 deletions fides/hessian_approximation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,19 +133,6 @@ def __init__(self, init_with_hess: Optional[bool] = False):
super(DFP, self).__init__(phi=1.0, init_with_hess=init_with_hess)


class PSB(IterativeHessianApproximation):
"""
Powell-symmetric-Broyden update strategy as introduced in
[Powell 1970](https://doi.org/10.1016/B978-0-12-597050-1.50006-3).
This is a rank 2 update strategy that preserves symmetry and
positive-semidefiniteness.
This scheme only works with a function that returns (fval, grad)
"""
def update(self, s, y):
self._hess += broyden_class_update(y, s, self._hess, v=s)


class SR1(IterativeHessianApproximation):
"""
Symmetric Rank 1 update strategy as described in
Expand Down Expand Up @@ -281,45 +268,24 @@ def requires_resfun(self):
return True # pragma: no cover


def _bfgs_vector(s, y, mat):
u = mat.dot(s)
c = u.T.dot(s)
b = y.T.dot(s)
rho = np.sqrt(b / c)
if not np.isfinite(rho):
rho = 0
return y + np.sqrt(rho)*u


def _psb_vector(s, y, mat):
return s


def _dfp_vector(s, y, mat):
return y


class StructuredApproximation(HessianApproximation):
vector_routines = {
'BFGS': _bfgs_vector,
'PSB': _psb_vector,
'DFP': _dfp_vector,
}

def __init__(self,
update_method: Optional[str] = 'BFGS'):
def __init__(self, phi: Optional[float] = 0.0):
"""
This is the base class for structured secant methods (SSM). SSMs
approximate the hessian by combining the Gauss-Newton component C(x)
and an iteratively updated component that approximates the
difference S to the true Hessian.
:parameter phi:
convex combination parameter interpolating between BFGS (phi==0)
and DFP (phi==1) update schemes.
"""
self.A: np.ndarray = np.empty(0)
if update_method not in self.vector_routines:
raise ValueError(f'Unknown update method {update_method}. Known '
f'methods are {self.vector_routines.keys()}.')

self.vector_routine = self.vector_routines[update_method]
self.phi = phi
if phi < 0 or phi > 1:
warnings.warn('Setting phi to values outside the interval [0, 1]'
'will not guarantee that positive definiteness is '
'preserved during updating.')
super(StructuredApproximation, self).__init__(init_with_hess=True)

def init_mat(self, dim: int, hess: Optional[np.ndarray] = None):
Expand All @@ -341,7 +307,7 @@ class SSM(StructuredApproximation):
"""
Structured Secant Method as introduced by
[Dennis et al 1989](https://doi.org/10.1007/BF00962795), which is
compatible with BFGS, DFP and PSB update schemes.
compatible with BFGS, DFP update schemes.
This scheme only works with a function that returns (res, sres)
"""
Expand All @@ -352,10 +318,8 @@ def update(self, s: np.ndarray, y: np.ndarray, r: np.ndarray,
Bs = hess + self.A
# y^S = y^# + C(x_+)*s
ys = yb + hess.dot(s)
# Equation (8-10) + Equation (13)
v = self.vector_routine(s, ys, Bs)
# Equation (13)
self.A += broyden_class_update(yb, s, self.A, v=v)
self.A += broyden_class_update(ys, s, Bs, phi=self.phi)
# B_+ = C(x_+) + A + BFGS update A (=A_+)
self._hess = hess + self.A

Expand All @@ -374,11 +338,9 @@ def update(self, s: np.ndarray, y: np.ndarray, r: np.ndarray,
# Equation (2.7)
Bs = hess + norm(r) * self.A
# Equation (2.6)
ys = hess.dot(s) + norm(r) * yb
# Equation (2.8)
v = self.vector_routine(s, ys, Bs)
# Equation (2.8)
self.A += broyden_class_update(yb, s, self.A, v=v)
ys = hess.dot(s) + yb
# Equation (2.10)
self.A += broyden_class_update(ys, s, Bs, phi=self.phi)/norm(r)
# Equation (2.9)
self._hess = hess + norm(r) * self.A

Expand All @@ -397,29 +359,25 @@ def __init__(self, hybrid_tol: float = 1e-6):
switching tolerance that controls switching between update methods
"""
self.hybrid_tol: float = hybrid_tol
super(GNSBFGS, self).__init__('BFGS')
super(GNSBFGS, self).__init__(phi=0.0)

def update(self, s: np.ndarray, y: np.ndarray, r: np.ndarray,
hess: np.ndarray, yb: np.ndarray):
# Equation (2.1)
ys = yb * norm(r)
ratio = ys.T.dot(s)/s.dot(s)
ratio = yb.T.dot(s)/s.dot(s)
if ratio > self.hybrid_tol:
# Equation (2.3)
self.A += broyden_class_update(ys, s, self.A, phi=1.0)
self.A += broyden_class_update(yb, s, self.A, phi=self.phi)
# Equation (2.2)
self._hess = hess + self.A
else:
# Equation (2.2)
self._hess = hess + norm(r) * np.eye(len(y))


def broyden_class_update(y, s, mat, phi=None, v=None):
def broyden_class_update(y, s, mat, phi=0.0):
"""
Scale free implementation of the broyden class update scheme. This can
either be called by using a phi parameter that interpolates between BFGS
(phi=0) and DFP (phi=1) or by using the weighting vector v that allows
implementation of PSB (v=s), DFP (v=y) and BFGS (V=y+rho*B*s).
Scale free implementation of the broyden class update scheme.
:param y:
difference in gradient
Expand All @@ -428,10 +386,8 @@ def broyden_class_update(y, s, mat, phi=None, v=None):
:param mat:
current hessian approximation
:param phi:
convex combination parameter. Must not pass this parameter at the same
time as v.
:param v:
weighting vector. Must not pass this parameter at the same time as phi.
convex combination parameter, interpolates between BFGS (phi=0) and
DFP (phi=1).
"""
u = mat.dot(s)
c = u.T.dot(s)
Expand All @@ -440,27 +396,10 @@ def broyden_class_update(y, s, mat, phi=None, v=None):
if b <= 0:
return np.zeros(mat.shape)

if v is None and phi is not None:
bfgs = phi == 0
elif v is not None and phi is None:
bfgs = False
else:
raise ValueError('Exactly one of the values of phi and v must be '
'provided.')

if bfgs: # BFGS
return np.outer(y, y.T) / b - np.outer(u, u.T) / c

if v is None:
rho = np.sqrt(b / c)
if not np.isfinite(rho):
rho = 0
v = y + (1-phi) * rho * u

z = y - mat.dot(s)
d = v.T.dot(s)
a = z.T.dot(s) / (d ** 2)
update = np.outer(y, y.T) / b - np.outer(u, u.T) / c

update = (np.outer(z, v.T) + np.outer(v, z.T)) / d - a * np.outer(v, v.T)
if phi != 0.0:
v = y / b - u / c
update += phi * c * np.outer(v, v.T)

return update
2 changes: 1 addition & 1 deletion fides/minimize.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,7 @@ def update(self,
if isinstance(self.hessian_update, (TSSM, GNSBFGS)):
# TSSM: Equation (2.5) in [Huschens 1994]
# GNSBFGS: Equation (2.1) in [Zhou & Chen 2010]
yb /= norm(funout.res)
yb *= norm(funout_new.res)/norm(funout.res)
self.hessian_update.update(s=s, y=y, yb=yb, r=funout_new.res,
hess=funout_new.hess)
else:
Expand Down
2 changes: 1 addition & 1 deletion fides/version.py
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.6.2"
__version__ = "0.6.3"
22 changes: 1 addition & 21 deletions tests/test_hessian_approximation.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
from fides import BFGS, Broyden, SSM
from fides.hessian_approximation import broyden_class_update
from fides import BFGS, Broyden

import pytest
import numpy as np


def test_wrong_dim():

with pytest.raises(ValueError):
h = BFGS(init_with_hess=True)
h.init_mat(dim=3, hess=np.ones((2, 2)))
Expand All @@ -25,21 +23,3 @@ def test_broyden():
h = Broyden(phi=-1)
h.init_mat(dim=2)
h.update(np.random.random((2,)), np.random.random((2,)))


def test_ssm_bad_method():
with pytest.raises(ValueError):
SSM('SR1')


def test_broyden_class_update():
with pytest.raises(ValueError):
broyden_class_update(np.random.random((2,)),
np.random.random((2,)),
np.random.random((2, 2)),
phi=1, v=np.random.random((2,)))

with pytest.raises(ValueError):
broyden_class_update(np.random.random((2,)),
np.random.random((2,)),
np.random.random((2, 2)))
35 changes: 17 additions & 18 deletions tests/test_minimize.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from fides import (
Optimizer, BFGS, SR1, DFP, BB, BG, PSB, Broyden, GNSBFGS, HybridFixed,
Optimizer, BFGS, SR1, DFP, BB, BG, Broyden, GNSBFGS, HybridFixed,
FX, SSM, TSSM, SubSpaceDim, StepBackStrategy
)
import numpy as np
Expand Down Expand Up @@ -142,23 +142,22 @@ def unbounded_and_init():
(rosengrad, DFP()), # 3
(rosengrad, BG()), # 4
(rosengrad, BB()), # 5
(rosengrad, PSB()), # 6
(rosengrad, Broyden(0.5)), # 7
(rosenboth, HybridFixed(BFGS())), # 8
(rosenboth, HybridFixed(SR1())), # 9
(rosenboth, HybridFixed(BFGS(init_with_hess=True))), # 10
(rosenboth, HybridFixed(SR1(init_with_hess=True))), # 11
(fletcher, FX(BFGS())), # 12
(fletcher, FX(SR1())), # 13
(fletcher, FX(BFGS(init_with_hess=True))), # 14
(fletcher, FX(SR1(init_with_hess=True))), # 15
(fletcher, SSM('BFGS')), # 16
(fletcher, SSM('DFP')), # 17
(fletcher, SSM('PSB')), # 18
(fletcher, TSSM('BFGS')), # 19
(fletcher, TSSM('DFP')), # 20
(fletcher, TSSM('PSB')), # 21
(fletcher, GNSBFGS()), # 22
(rosengrad, Broyden(0.5)), # 6
(rosenboth, HybridFixed(BFGS())), # 7
(rosenboth, HybridFixed(SR1())), # 8
(rosenboth, HybridFixed(BFGS(init_with_hess=True))), # 9
(rosenboth, HybridFixed(SR1(init_with_hess=True))), # 10
(fletcher, FX(BFGS())), # 11
(fletcher, FX(SR1())), # 12
(fletcher, FX(BFGS(init_with_hess=True))), # 13
(fletcher, FX(SR1(init_with_hess=True))), # 14
(fletcher, SSM(0.0)), # 15
(fletcher, SSM(0.5)), # 16
(fletcher, SSM(1.0)), # 17
(fletcher, TSSM(0.0)), # 18
(fletcher, TSSM(0.5)), # 19
(fletcher, TSSM(1.0)), # 20
(fletcher, GNSBFGS()), # 21
])
def test_minimize_hess_approx(bounds_and_init, fun, happ, subspace_dim,
stepback, refine, sgradient):
Expand Down

0 comments on commit f481387

Please sign in to comment.