From 86bf9b49ad8ef211c6abd864c236e8f8d073a98c Mon Sep 17 00:00:00 2001 From: Evgeniy Belousov Date: Fri, 28 Feb 2020 13:56:03 +0300 Subject: [PATCH 1/3] Custom checker for half-open state like ReadyToTrip --- gobreaker.go | 44 ++++++++++++++++++++++++++++---------------- gobreaker_test.go | 12 +++++++----- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/gobreaker.go b/gobreaker.go index f402217..5ad7c3c 100644 --- a/gobreaker.go +++ b/gobreaker.go @@ -20,8 +20,6 @@ const ( ) var ( - // ErrTooManyRequests is returned when the CB state is half open and the requests count is over the cb maxRequests - ErrTooManyRequests = errors.New("too many requests") // ErrOpenState is returned when the CB state is open ErrOpenState = errors.New("circuit breaker is open") ) @@ -52,6 +50,8 @@ type Counts struct { ConsecutiveFailures uint32 } +type readyToFn func(counts Counts) bool + func (c *Counts) onRequest() { c.Requests++ } @@ -80,9 +80,11 @@ func (c *Counts) clear() { // // Name is the name of the CircuitBreaker. // -// MaxRequests is the maximum number of requests allowed to pass through -// when the CircuitBreaker is half-open. -// If MaxRequests is 0, the CircuitBreaker allows only 1 request. +// ReadyToClose is called with a copy of Counts for each request in the half-open state. +// If ReadyToClose returns true, the CircuitBreaker will be placed into the close state. +// If ReadyToClose returns false, the CircuitBreaker will be placed into the open state. +// If ReadyToClose is nil, default ReadyToClose is used. +// Default ReadyToClose returns true when the number of consecutive successes is more than 1. // // Interval is the cyclic period of the closed state // for the CircuitBreaker to clear the internal Counts. @@ -100,20 +102,20 @@ func (c *Counts) clear() { // OnStateChange is called whenever the state of the CircuitBreaker changes. type Settings struct { Name string - MaxRequests uint32 + ReadyToClose readyToFn Interval time.Duration Timeout time.Duration - ReadyToTrip func(counts Counts) bool + ReadyToTrip readyToFn OnStateChange func(name string, from State, to State) } // CircuitBreaker is a state machine to prevent sending requests that are likely to fail. type CircuitBreaker struct { name string - maxRequests uint32 + readyToClose readyToFn interval time.Duration timeout time.Duration - readyToTrip func(counts Counts) bool + readyToTrip readyToFn onStateChange func(name string, from State, to State) mutex sync.Mutex @@ -137,10 +139,10 @@ func NewCircuitBreaker(st Settings) *CircuitBreaker { cb.name = st.Name cb.onStateChange = st.OnStateChange - if st.MaxRequests == 0 { - cb.maxRequests = 1 + if st.ReadyToClose == nil { + cb.readyToClose = defaultReadyToClose } else { - cb.maxRequests = st.MaxRequests + cb.readyToClose = st.ReadyToClose } if st.Interval <= 0 { @@ -180,6 +182,10 @@ func defaultReadyToTrip(counts Counts) bool { return counts.ConsecutiveFailures > 5 } +func defaultReadyToClose(counts Counts) bool { + return counts.ConsecutiveSuccesses >= 1 +} + // Name returns the name of the CircuitBreaker. func (cb *CircuitBreaker) Name() string { return cb.name @@ -252,8 +258,6 @@ func (cb *CircuitBreaker) beforeRequest() (uint64, error) { if state == StateOpen { return generation, ErrOpenState - } else if state == StateHalfOpen && cb.counts.Requests >= cb.maxRequests { - return generation, ErrTooManyRequests } cb.counts.onRequest() @@ -283,7 +287,7 @@ func (cb *CircuitBreaker) onSuccess(state State, now time.Time) { cb.counts.onSuccess() case StateHalfOpen: cb.counts.onSuccess() - if cb.counts.ConsecutiveSuccesses >= cb.maxRequests { + if cb.readyToClose(cb.counts) { cb.setState(StateClosed, now) } } @@ -296,8 +300,12 @@ func (cb *CircuitBreaker) onFailure(state State, now time.Time) { if cb.readyToTrip(cb.counts) { cb.setState(StateOpen, now) } + case StateHalfOpen: - cb.setState(StateOpen, now) + cb.counts.onFailure() + if false == cb.readyToClose(cb.counts) { + cb.setState(StateOpen, now) + } } } @@ -307,11 +315,13 @@ func (cb *CircuitBreaker) currentState(now time.Time) (State, uint64) { if !cb.expiry.IsZero() && cb.expiry.Before(now) { cb.toNewGeneration(now) } + case StateOpen: if cb.expiry.Before(now) { cb.setState(StateHalfOpen, now) } } + return cb.state, cb.generation } @@ -342,8 +352,10 @@ func (cb *CircuitBreaker) toNewGeneration(now time.Time) { } else { cb.expiry = now.Add(cb.interval) } + case StateOpen: cb.expiry = now.Add(cb.timeout) + default: // StateHalfOpen cb.expiry = zero } diff --git a/gobreaker_test.go b/gobreaker_test.go index eb8ecc5..6b383e9 100644 --- a/gobreaker_test.go +++ b/gobreaker_test.go @@ -81,7 +81,9 @@ func causePanic(cb *CircuitBreaker) error { func newCustom() *CircuitBreaker { var customSt Settings customSt.Name = "cb" - customSt.MaxRequests = 3 + customSt.ReadyToClose = func(counts Counts) bool { + return counts.ConsecutiveSuccesses >= 3 + } customSt.Interval = time.Duration(30) * time.Second customSt.Timeout = time.Duration(90) * time.Second customSt.ReadyToTrip = func(counts Counts) bool { @@ -128,7 +130,7 @@ func TestStateConstants(t *testing.T) { func TestNewCircuitBreaker(t *testing.T) { defaultCB := NewCircuitBreaker(Settings{}) assert.Equal(t, "", defaultCB.name) - assert.Equal(t, uint32(1), defaultCB.maxRequests) + assert.NotNil(t, defaultCB.readyToClose) assert.Equal(t, time.Duration(0), defaultCB.interval) assert.Equal(t, time.Duration(60)*time.Second, defaultCB.timeout) assert.NotNil(t, defaultCB.readyToTrip) @@ -139,7 +141,7 @@ func TestNewCircuitBreaker(t *testing.T) { customCB := newCustom() assert.Equal(t, "cb", customCB.name) - assert.Equal(t, uint32(3), customCB.maxRequests) + assert.NotNil(t, customCB.readyToClose) assert.Equal(t, time.Duration(30)*time.Second, customCB.interval) assert.Equal(t, time.Duration(90)*time.Second, customCB.timeout) assert.NotNil(t, customCB.readyToTrip) @@ -150,7 +152,7 @@ func TestNewCircuitBreaker(t *testing.T) { negativeDurationCB := newNegativeDurationCB() assert.Equal(t, "ncb", negativeDurationCB.name) - assert.Equal(t, uint32(1), negativeDurationCB.maxRequests) + assert.NotNil(t, negativeDurationCB.readyToClose) assert.Equal(t, time.Duration(0)*time.Second, negativeDurationCB.interval) assert.Equal(t, time.Duration(60)*time.Second, negativeDurationCB.timeout) assert.NotNil(t, negativeDurationCB.readyToTrip) @@ -258,7 +260,7 @@ func TestCustomCircuitBreaker(t *testing.T) { ch := succeedLater(customCB, time.Duration(100)*time.Millisecond) // 3 consecutive successes time.Sleep(time.Duration(50) * time.Millisecond) assert.Equal(t, Counts{3, 2, 0, 2, 0}, customCB.counts) - assert.Error(t, succeed(customCB)) // over MaxRequests + assert.Nil(t, succeed(customCB)) // over MaxRequests assert.Nil(t, <-ch) assert.Equal(t, StateClosed, customCB.State()) assert.Equal(t, Counts{0, 0, 0, 0, 0}, customCB.counts) From a43dda4e2fb8792fedfd1fe77ead84b0d10a9937 Mon Sep 17 00:00:00 2001 From: Evgeniy Belousov Date: Fri, 28 Feb 2020 14:24:55 +0300 Subject: [PATCH 2/3] Added support for keep half-open state on failure --- gobreaker.go | 21 ++++++++++----------- gobreaker_test.go | 18 +++++++++++++++--- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/gobreaker.go b/gobreaker.go index 5ad7c3c..b296680 100644 --- a/gobreaker.go +++ b/gobreaker.go @@ -50,8 +50,6 @@ type Counts struct { ConsecutiveFailures uint32 } -type readyToFn func(counts Counts) bool - func (c *Counts) onRequest() { c.Requests++ } @@ -82,7 +80,7 @@ func (c *Counts) clear() { // // ReadyToClose is called with a copy of Counts for each request in the half-open state. // If ReadyToClose returns true, the CircuitBreaker will be placed into the close state. -// If ReadyToClose returns false, the CircuitBreaker will be placed into the open state. +// If ReadyToClose returns false, the CircuitBreaker will be placed into the open state if second returned value is true. // If ReadyToClose is nil, default ReadyToClose is used. // Default ReadyToClose returns true when the number of consecutive successes is more than 1. // @@ -102,20 +100,20 @@ func (c *Counts) clear() { // OnStateChange is called whenever the state of the CircuitBreaker changes. type Settings struct { Name string - ReadyToClose readyToFn + ReadyToClose func(counts Counts) (bool, bool) Interval time.Duration Timeout time.Duration - ReadyToTrip readyToFn + ReadyToTrip func(counts Counts) bool OnStateChange func(name string, from State, to State) } // CircuitBreaker is a state machine to prevent sending requests that are likely to fail. type CircuitBreaker struct { name string - readyToClose readyToFn + readyToClose func(counts Counts) (bool, bool) interval time.Duration timeout time.Duration - readyToTrip readyToFn + readyToTrip func(counts Counts) bool onStateChange func(name string, from State, to State) mutex sync.Mutex @@ -182,8 +180,8 @@ func defaultReadyToTrip(counts Counts) bool { return counts.ConsecutiveFailures > 5 } -func defaultReadyToClose(counts Counts) bool { - return counts.ConsecutiveSuccesses >= 1 +func defaultReadyToClose(counts Counts) (bool, bool) { + return counts.ConsecutiveSuccesses >= 1, true } // Name returns the name of the CircuitBreaker. @@ -287,7 +285,7 @@ func (cb *CircuitBreaker) onSuccess(state State, now time.Time) { cb.counts.onSuccess() case StateHalfOpen: cb.counts.onSuccess() - if cb.readyToClose(cb.counts) { + if ok, _ := cb.readyToClose(cb.counts); ok { cb.setState(StateClosed, now) } } @@ -303,7 +301,8 @@ func (cb *CircuitBreaker) onFailure(state State, now time.Time) { case StateHalfOpen: cb.counts.onFailure() - if false == cb.readyToClose(cb.counts) { + ok, nowOpen := cb.readyToClose(cb.counts) + if !ok && nowOpen { cb.setState(StateOpen, now) } } diff --git a/gobreaker_test.go b/gobreaker_test.go index 6b383e9..7017ba2 100644 --- a/gobreaker_test.go +++ b/gobreaker_test.go @@ -81,8 +81,20 @@ func causePanic(cb *CircuitBreaker) error { func newCustom() *CircuitBreaker { var customSt Settings customSt.Name = "cb" - customSt.ReadyToClose = func(counts Counts) bool { - return counts.ConsecutiveSuccesses >= 3 + customSt.ReadyToClose = func(counts Counts) (bool, bool) { + if counts.ConsecutiveSuccesses >= 3 { + return true, true + } + + numReqs := counts.Requests + failureRatio := float64(counts.TotalFailures) / float64(numReqs) + + var nowOpen bool + if numReqs >= 3 && failureRatio >= 0.6 { + nowOpen = true + } + + return false, nowOpen } customSt.Interval = time.Duration(30) * time.Second customSt.Timeout = time.Duration(90) * time.Second @@ -260,7 +272,7 @@ func TestCustomCircuitBreaker(t *testing.T) { ch := succeedLater(customCB, time.Duration(100)*time.Millisecond) // 3 consecutive successes time.Sleep(time.Duration(50) * time.Millisecond) assert.Equal(t, Counts{3, 2, 0, 2, 0}, customCB.counts) - assert.Nil(t, succeed(customCB)) // over MaxRequests + assert.Nil(t, succeed(customCB)) assert.Nil(t, <-ch) assert.Equal(t, StateClosed, customCB.State()) assert.Equal(t, Counts{0, 0, 0, 0, 0}, customCB.counts) From 4f079cb62cb32cd5edf7537b2ff9f28464d75fc5 Mon Sep 17 00:00:00 2001 From: Evgeniy Belousov Date: Fri, 28 Feb 2020 14:32:18 +0300 Subject: [PATCH 3/3] Update readme --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ce7d7a7..dcecb55 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ You can configure `CircuitBreaker` by the struct `Settings`: ```go type Settings struct { Name string - MaxRequests uint32 + ReadyToClose func(counts Counts) (bool, bool) Interval time.Duration Timeout time.Duration ReadyToTrip func(counts Counts) bool @@ -39,9 +39,11 @@ type Settings struct { - `Name` is the name of the `CircuitBreaker`. -- `MaxRequests` is the maximum number of requests allowed to pass through - when the `CircuitBreaker` is half-open. - If `MaxRequests` is 0, `CircuitBreaker` allows only 1 request. +- `ReadyToClose` is called with a copy of `Counts` for each request in the half-open state. + If `ReadyToClose` returns true, the `CircuitBreaker` will be placed into the close state. + If `ReadyToClose` returns false, the `CircuitBreaker` will be placed into the open state if second returned value is true. + If `ReadyToClose` is nil, default `ReadyToClose` is used. + Default `ReadyToClose` returns true when the number of consecutive successes is more than 1. - `Interval` is the cyclic period of the closed state for `CircuitBreaker` to clear the internal `Counts`, described later in this section.