forked from lightningnetwork/lnd
-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
multi: Automatically close non-reestablished channels
This adds automation for closing channels that haven't been reestablished for some time using the recently introduced metric. Channels with peers that have been online for some time but which have NOT been reestablished though the use of ChannelReestablish messages are good candidates for being force closed, because it is likely the remote peer has lost the ability to use them. Given that the metric tracks the total time across restarts and only while the remote peer is online, we use a default of 72h for the threshold to auto close the channel. This should be a reasonable compromise between not closing too fast on hubs (that are online 24/7) and ephemeral nodes (that may be online only for an hour or two a day). A config parameter is added to control the threshold time used for autoclosing and an itest is added that asserts the correct behavior.
- Loading branch information
Showing
11 changed files
with
316 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,119 @@ | ||
package automation | ||
|
||
import ( | ||
"context" | ||
"time" | ||
|
||
"github.com/decred/dcrlnd/channeldb" | ||
"github.com/decred/dcrlnd/lncfg" | ||
"github.com/decred/dcrlnd/lnrpc" | ||
) | ||
|
||
// Config are the config parameters for the automation server. | ||
type Config struct { | ||
*lncfg.Automation | ||
|
||
// CloseChannel should be set to the rpc server function that allows | ||
// closing a channel. | ||
CloseChannel func(in *lnrpc.CloseChannelRequest, | ||
updateStream lnrpc.Lightning_CloseChannelServer) error | ||
|
||
DB *channeldb.DB | ||
} | ||
|
||
// Server is set of automation services for dcrlnd nodes. | ||
type Server struct { | ||
cfg *Config | ||
ctx context.Context | ||
cancelCtx func() | ||
} | ||
|
||
// NewServer creates a new automation server. | ||
func NewServer(cfg *Config) *Server { | ||
ctx, cancel := context.WithCancel(context.Background()) | ||
s := &Server{ | ||
cfg: cfg, | ||
ctx: ctx, | ||
cancelCtx: cancel, | ||
} | ||
return s | ||
} | ||
|
||
// runForceCloseStaleChanReestablish autocloses channels where a remote peer | ||
// has been online without sending ChannelReestablish messages. | ||
func (s *Server) runForceCloseStaleChanReestablish() { | ||
// Use a default ticker for 1 hour, but reduce if the threshold is lower | ||
// than that (useful for tests). | ||
forceCloseInterval := time.Duration(s.cfg.ForceCloseChanReestablishWait) * time.Second | ||
tickerInterval := time.Hour | ||
if forceCloseInterval < tickerInterval { | ||
tickerInterval = forceCloseInterval | ||
} | ||
log.Debugf("Performing force close check for stale channels based on "+ | ||
"ChannelReestablish every %s", tickerInterval) | ||
|
||
ticker := time.NewTicker(tickerInterval) | ||
for { | ||
select { | ||
case <-s.ctx.Done(): | ||
ticker.Stop() | ||
return | ||
case <-ticker.C: | ||
} | ||
|
||
log.Debugf("Time to check channels for force close due to stale " + | ||
"chan reestablish messages") | ||
|
||
chans, err := s.cfg.DB.FetchAllOpenChannels() | ||
if err != nil { | ||
log.Errorf("Unable to list open channels: %v", err) | ||
continue | ||
} | ||
|
||
for _, c := range chans { | ||
sid := c.ShortChannelID | ||
waitTime, err := s.cfg.DB.GetChanReestablishWaitTime(sid) | ||
if err != nil { | ||
log.Errorf("Unable to get chan reestablish msg "+ | ||
"times for %s: %v", sid, err) | ||
continue | ||
} | ||
|
||
if waitTime < forceCloseInterval { | ||
log.Tracef("Skipping autoclose of %s due to low "+ | ||
"wait time %s", sid, waitTime) | ||
continue | ||
} | ||
|
||
// Start force close. | ||
chanPoint := c.FundingOutpoint | ||
log.Infof("Starting force-close attempt of channel %s (%s) "+ | ||
"due to channel reestablish msg wait time %s greater "+ | ||
"than max interval %s", chanPoint, | ||
sid, waitTime, forceCloseInterval) | ||
go func() { | ||
req := &lnrpc.CloseChannelRequest{ | ||
ChannelPoint: lnrpc.OutpointToChanPoint(&chanPoint), | ||
Force: true, | ||
} | ||
err = s.cfg.CloseChannel(req, nil) | ||
if err != nil { | ||
log.Errorf("Unable to force-close channel %s: %v", | ||
sid, err) | ||
} | ||
}() | ||
} | ||
} | ||
} | ||
|
||
func (s *Server) Start() error { | ||
if s.cfg.ForceCloseChanReestablishWait > 0 { | ||
go s.runForceCloseStaleChanReestablish() | ||
} | ||
return nil | ||
} | ||
|
||
func (s *Server) Stop() error { | ||
s.cancelCtx() | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
package automation | ||
|
||
import ( | ||
"github.com/decred/dcrlnd/build" | ||
"github.com/decred/slog" | ||
) | ||
|
||
// log is a logger that is initialized with no output filters. This | ||
// means the package will not perform any logging by default until the caller | ||
// requests it. | ||
var log slog.Logger | ||
|
||
// The default amount of logging is none. | ||
func init() { | ||
UseLogger(build.NewSubLogger("AUTO", nil)) | ||
} | ||
|
||
// DisableLog disables all library log output. Logging output is disabled | ||
// by default until UseLogger is called. | ||
func DisableLog() { | ||
UseLogger(slog.Disabled) | ||
} | ||
|
||
// UseLogger uses a specified Logger to output package logging info. | ||
// This should be used in preference to SetLogWriter if the caller is also | ||
// using slog. | ||
func UseLogger(logger slog.Logger) { | ||
log = logger | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
package lncfg | ||
|
||
// Automation holds some server level automation config options. | ||
type Automation struct { | ||
// ForceCloseChanReestablishWait is the time after which the automation | ||
// server force closes a channel where the local peer has sent | ||
// ChannelReestablish messages but the remote peer does not. | ||
ForceCloseChanReestablishWait int64 `long:"closechanreestablishwait" description:"Force close a channel if the difference between time channel reestablish msgs were sent and received is higher than the specified one"` | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
package itest | ||
|
||
import ( | ||
"time" | ||
|
||
"github.com/decred/dcrd/dcrutil/v4" | ||
"github.com/decred/dcrlnd/lnrpc" | ||
"github.com/decred/dcrlnd/lntest" | ||
"github.com/stretchr/testify/require" | ||
"matheusd.com/testctx" | ||
) | ||
|
||
func testMissingChanReestablishAutoClosesChan(net *lntest.NetworkHarness, t *harnessTest) { | ||
const ( | ||
chanAmt = dcrutil.Amount(10000000) | ||
pushAmt = dcrutil.Amount(5000000) | ||
) | ||
password := []byte("El Psy Kongroo") | ||
var err error | ||
|
||
// Create a new retore scenario. | ||
carolArgs := []string{"--automation.closechanreestablishwait=5"} | ||
carol := net.NewNode(t.t, "carol", carolArgs) | ||
defer shutdownAndAssert(net, t, carol) | ||
|
||
daveArgs := []string{"--nolisten", "--minbackoff=1h"} | ||
dave, mnemonic, _, err := net.NewNodeWithSeed( | ||
"dave", daveArgs, password, false, | ||
) | ||
require.Nil(t.t, err) | ||
defer shutdownAndAssert(net, t, dave) | ||
|
||
net.SendCoins(testctx.New(t), t.t, dcrutil.AtomsPerCoin, carol) | ||
net.SendCoins(testctx.New(t), t.t, dcrutil.AtomsPerCoin, dave) | ||
net.EnsureConnected(testctx.New(t), t.t, dave, carol) | ||
|
||
chanPoint := openChannelAndAssert( | ||
testctx.New(t), t, net, carol, dave, | ||
lntest.OpenChannelParams{ | ||
Amt: chanAmt, | ||
PushAmt: pushAmt, | ||
}, | ||
) | ||
|
||
// Wait for both sides to see the opened channel. | ||
err = dave.WaitForNetworkChannelOpen(testctx.New(t), chanPoint) | ||
require.Nil(t.t, err) | ||
err = carol.WaitForNetworkChannelOpen(testctx.New(t), chanPoint) | ||
require.Nil(t.t, err) | ||
|
||
// Perform a payment to assert channel is working. | ||
invoice := &lnrpc.Invoice{ | ||
Memo: "testing", | ||
Value: 100000, | ||
} | ||
invoiceResp, err := carol.AddInvoice(testctx.New(t), invoice) | ||
require.Nil(t.t, err) | ||
err = completePaymentRequests( | ||
testctx.New(t), dave, dave.RouterClient, | ||
[]string{invoiceResp.PaymentRequest}, true, | ||
) | ||
require.Nil(t.t, err) | ||
|
||
// Recreate Dave without the channel. | ||
err = net.ShutdownNode(dave) | ||
require.Nil(t.t, err) | ||
time.Sleep(time.Second) | ||
|
||
daveRestored, err := net.RestoreNodeWithSeed( | ||
"dave", nil, password, mnemonic, 1000, | ||
nil, copyPorts(dave), | ||
) | ||
require.Nil(t.t, err) | ||
assertNumPendingChannels(t, daveRestored, 0, 0, 0, 0) | ||
assertNodeNumChannels(t, daveRestored, 0) | ||
// ht.AssertNumEdges(daveRestored, 0, true) | ||
|
||
// Assert Carol does not autoclose and Dave does not have the channel. | ||
net.EnsureConnected(testctx.New(t), t.t, daveRestored, carol) | ||
time.Sleep(time.Second) | ||
assertNumPendingChannels(t, daveRestored, 0, 0, 0, 0) | ||
assertNodeNumChannels(t, daveRestored, 0) | ||
assertNumPendingChannels(t, carol, 0, 0, 0, 0) | ||
assertNodeNumChannels(t, carol, 1) | ||
|
||
// Assert Carol is tracking the time Dave has been online without | ||
// reestablishing the channel. | ||
require.Nil(t.t, net.DisconnectNodes(testctx.New(t), carol, daveRestored)) | ||
time.Sleep(time.Second) | ||
chanInfo, err := carol.ListChannels(testctx.New(t), &lnrpc.ListChannelsRequest{}) | ||
require.Nil(t.t, err) | ||
require.Len(t.t, chanInfo.Channels, 1) | ||
require.Greater(t.t, chanInfo.Channels[0].ChanReestablishWaitTimeMs, int64(2000)) | ||
|
||
// Wait long enough for Carol's automation to want to force-close the | ||
// channel. | ||
net.EnsureConnected(testctx.New(t), t.t, daveRestored, carol) | ||
time.Sleep(time.Second * 3) | ||
err = net.ShutdownNode(daveRestored) | ||
require.Nil(t.t, err) | ||
|
||
// Assert Carol force-closes the channel. | ||
assertNumPendingChannels(t, carol, 1, 0, 0, 0) | ||
assertNodeNumChannels(t, carol, 0) | ||
mineBlocks(t, net, 1, 1) | ||
cleanupForceClose(t, net, carol, chanPoint) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters