From 8608bce5df9ea11d56dd7cb6ca3b12a242e54c9c Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Fri, 21 Jul 2023 19:01:31 +0200 Subject: [PATCH 01/25] [WIP] Add MinBlockTime --- consensus/doc/CHANGELOG.md | 2 + consensus/e2e_tests/pacemaker_test.go | 50 +++++++++++ consensus/hotstuff_leader.go | 7 +- consensus/pacemaker/module.go | 90 +++++++++++++++++++- runtime/configs/config.go | 1 + runtime/configs/proto/consensus_config.proto | 1 + runtime/defaults/defaults.go | 1 + runtime/docs/CHANGELOG.md | 2 + runtime/manager_test.go | 1 + 9 files changed, 150 insertions(+), 5 deletions(-) diff --git a/consensus/doc/CHANGELOG.md b/consensus/doc/CHANGELOG.md index 4d5b89930..4bb70a11d 100644 --- a/consensus/doc/CHANGELOG.md +++ b/consensus/doc/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Implement minimum block production pace by delaying block preparation + ## [0.0.0.54] - 2023-06-13 - Fix tests diff --git a/consensus/e2e_tests/pacemaker_test.go b/consensus/e2e_tests/pacemaker_test.go index 6255b1fb0..e5e996b92 100644 --- a/consensus/e2e_tests/pacemaker_test.go +++ b/consensus/e2e_tests/pacemaker_test.go @@ -172,6 +172,56 @@ func forcePacemakerTimeout(t *testing.T, clockMock *clock.Mock, paceMakerTimeout advanceTime(t, clockMock, paceMakerTimeout+10*time.Millisecond) } +func TestPacemakerMinBlockTime(t *testing.T) { + // Test preparation + clockMock := clock.NewMock() + timeReminder(t, clockMock, time.Second) + + // UnitTestNet configs + paceMakerTimeoutMsec := uint64(10000) + consensusMessageTimeout := time.Duration(paceMakerTimeoutMsec / 5) // Must be smaller than pacemaker timeout because we expect a deterministic number of consensus messages. + paceMakerMinBlockTimeMsec := uint64(5000) // Make sure it is larger than the consensus message timeout + runtimeMgrs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) + for _, runtimeConfig := range runtimeMgrs { + consCfg := runtimeConfig.GetConfig().Consensus.PacemakerConfig + consCfg.TimeoutMsec = paceMakerTimeoutMsec + consCfg.MinBlockTimeMsec = paceMakerMinBlockTimeMsec + } + buses := GenerateBuses(t, runtimeMgrs) + + // Create & start test pocket nodes + eventsChannel := make(modules.EventsChannel, 100) + pocketNodes := CreateTestConsensusPocketNodes(t, buses, eventsChannel) + err := StartAllTestPocketNodes(t, pocketNodes) + require.NoError(t, err) + + // Debug message to start consensus by triggering next view + for _, pocketNode := range pocketNodes { + TriggerNextView(t, pocketNode) + } + + consMod := pocketNodes[1].GetBus().GetConsensusModule() + + newRoundMessages, err := waitForProposalMsgs(t, clockMock, eventsChannel, pocketNodes, 1, uint8(consensus.NewRound), 0, 0, numValidators*numValidators, consensusMessageTimeout, true) + require.NoError(t, err) + whenBroadcast := clockMock.Now() + broadcastMessages(t, newRoundMessages, pocketNodes) + finishedBroadcast := uint64(clockMock.Now().Sub(whenBroadcast).Milliseconds()) + beforePrepareTimeout := time.Duration((paceMakerMinBlockTimeMsec - finishedBroadcast) * uint64(time.Millisecond)) + + step := typesCons.HotstuffStep(consMod.CurrentStep()) + require.Equal(t, consensus.NewRound, step) + + advanceTime(t, clockMock, clock.Duration(beforePrepareTimeout)) + step = typesCons.HotstuffStep(consMod.CurrentStep()) + // Should still be blocking proposal step + require.Equal(t, consensus.NewRound, step) + + //advanceTime(t, clockMock, 8000*time.Millisecond) + //step = typesCons.HotstuffStep(consMod.CurrentStep()) + //require.Equal(t, consensus.Prepare, step) +} + // TODO: Implement these tests and use them as a starting point for new ones. Consider using ChatGPT to help you out :) func TestPacemakerDifferentHeightsCatchup(t *testing.T) { diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 1a3520870..0fa6889d4 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -35,8 +35,6 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *consensusM return } - // DISCUSS: Do we need to pause for `MinBlockFreqMSec` here to let more transactions or should we stick with optimistic responsiveness? - if err := m.didReceiveEnoughMessageForStep(NewRound); err != nil { m.logger.Info().Fields(hotstuffMsgToLoggingFields(msg)).Msgf("⏳ Waiting ⏳for more messages; %s", err.Error()) return @@ -64,6 +62,11 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *consensusM // TODO: Add test to make sure same block is not applied twice if round is interrupted after being 'Applied'. // TODO: Add more unit tests for these checks... if m.shouldPrepareNewBlock(highPrepareQC) { + doPrepare := m.paceMaker.ProcessDelayedBlockPrepare() + if !doPrepare { + m.logger.Info().Msg("skip prepare new block") + return + } block, err := m.prepareBlock(highPrepareQC) if err != nil { m.logger.Error().Err(err).Msg(typesCons.ErrPrepareBlock.Error()) diff --git a/consensus/pacemaker/module.go b/consensus/pacemaker/module.go index 3a9a16db1..e096ea822 100644 --- a/consensus/pacemaker/module.go +++ b/consensus/pacemaker/module.go @@ -3,6 +3,7 @@ package pacemaker import ( "context" "fmt" + "sync" "time" consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" @@ -38,6 +39,7 @@ type Pacemaker interface { PacemakerDebug ShouldHandleMessage(message *typesCons.HotstuffMessage) (bool, error) + ProcessDelayedBlockPrepare() bool RestartTimer() NewHeight() @@ -48,9 +50,10 @@ type pacemaker struct { base_modules.IntegrableModule base_modules.InterruptableModule - pacemakerCfg *configs.PacemakerConfig - roundTimeout time.Duration - roundCancelFunc context.CancelFunc + pacemakerCfg *configs.PacemakerConfig + roundTimeout time.Duration + roundCancelFunc context.CancelFunc + latestPrepareRequest latestPrepareRequest // Only used for development and debugging. debug pacemakerDebug @@ -60,6 +63,15 @@ type pacemaker struct { logPrefix string } +// Structure to handle delaying block preparation (reaping the block mempool) +type latestPrepareRequest struct { + m sync.Mutex + ch chan bool + cancelFunc context.CancelFunc + blockProposed bool + deadlinePassed bool +} + func CreatePacemaker(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { return new(pacemaker).Create(bus, options...) } @@ -85,6 +97,13 @@ func (*pacemaker) Create(bus modules.Bus, options ...modules.ModuleOption) (modu debugTimeBetweenStepsMsec: m.pacemakerCfg.GetDebugTimeBetweenStepsMsec(), quorumCertificate: nil, } + m.latestPrepareRequest = latestPrepareRequest{ + m: sync.Mutex{}, + ch: nil, + cancelFunc: nil, + blockProposed: false, + deadlinePassed: false, + } return m, nil } @@ -92,6 +111,7 @@ func (*pacemaker) Create(bus modules.Bus, options ...modules.ModuleOption) (modu func (m *pacemaker) Start() error { m.logger = logger.Global.CreateLoggerForModule(m.GetModuleName()) m.RestartTimer() + m.RestartBlockProposalTimer() return nil } @@ -171,6 +191,7 @@ func (m *pacemaker) ShouldHandleMessage(msg *typesCons.HotstuffMessage) (bool, e func (m *pacemaker) RestartTimer() { // NOTE: Not deferring a cancel call because this function is asynchronous. + // DISCUSS: Should we have a lock to manipulate m.roundCancelFunc? if m.roundCancelFunc != nil { m.roundCancelFunc() } @@ -191,6 +212,45 @@ func (m *pacemaker) RestartTimer() { }() } +func (m *pacemaker) RestartBlockProposalTimer() { + m.latestPrepareRequest.m.Lock() + defer m.latestPrepareRequest.m.Unlock() + + if m.latestPrepareRequest.ch != nil { + m.latestPrepareRequest.ch <- false + } + + if m.latestPrepareRequest.cancelFunc != nil { + m.latestPrepareRequest.cancelFunc() + } + + m.latestPrepareRequest.blockProposed = false + m.latestPrepareRequest.deadlinePassed = false + + ctx, cancel := context.WithCancel(context.TODO()) + m.latestPrepareRequest.cancelFunc = cancel + clock := m.GetBus().GetRuntimeMgr().GetClock() + minBlockTime := time.Duration(m.pacemakerCfg.MinBlockTimeMsec * uint64(time.Millisecond)) + + go func() { + select { + case <-ctx.Done(): + return + case <-clock.After(minBlockTime): + m.latestPrepareRequest.m.Lock() + defer m.latestPrepareRequest.m.Unlock() + + if m.latestPrepareRequest.ch != nil { + m.latestPrepareRequest.blockProposed = true + m.latestPrepareRequest.ch <- true + m.latestPrepareRequest.ch = nil + } + + m.latestPrepareRequest.deadlinePassed = true + } + }() +} + func (m *pacemaker) InterruptRound(reason string) { defer m.RestartTimer() @@ -225,6 +285,7 @@ func (m *pacemaker) InterruptRound(reason string) { func (m *pacemaker) NewHeight() { defer m.RestartTimer() + defer m.RestartBlockProposalTimer() consensusMod := m.GetBus().GetConsensusModule() consensusMod.ResetRound(true) @@ -243,6 +304,29 @@ func (m *pacemaker) NewHeight() { ) } +func (m *pacemaker) ProcessDelayedBlockPrepare() bool { + m.latestPrepareRequest.m.Lock() + + if m.latestPrepareRequest.blockProposed { + return false + } + + if m.latestPrepareRequest.deadlinePassed { + return true + } + + // there is already a block preparer candidate, we cancel it and start a new one + // DISCUSS: This is needed if we want to get the latest QC for the block proposal. + if m.latestPrepareRequest.ch != nil { + m.latestPrepareRequest.ch <- false + } + + m.latestPrepareRequest.ch = make(chan bool) + m.latestPrepareRequest.m.Unlock() + + return <-m.latestPrepareRequest.ch +} + func (m *pacemaker) startNextView(qc *typesCons.QuorumCertificate, forceNextView bool) { defer m.RestartTimer() diff --git a/runtime/configs/config.go b/runtime/configs/config.go index a4cbb0e53..317fe6170 100644 --- a/runtime/configs/config.go +++ b/runtime/configs/config.go @@ -122,6 +122,7 @@ func NewDefaultConfig(options ...func(*Config)) *Config { TimeoutMsec: defaults.DefaultPacemakerTimeoutMsec, Manual: defaults.DefaultPacemakerManual, DebugTimeBetweenStepsMsec: defaults.DefaultPacemakerDebugTimeBetweenStepsMsec, + MinBlockTimeMsec: defaults.DefaultPacemakerMinBlockTimeMsec, }, }, Utility: &UtilityConfig{ diff --git a/runtime/configs/proto/consensus_config.proto b/runtime/configs/proto/consensus_config.proto index b2c29559b..5635a620d 100644 --- a/runtime/configs/proto/consensus_config.proto +++ b/runtime/configs/proto/consensus_config.proto @@ -15,4 +15,5 @@ message PacemakerConfig { uint64 timeout_msec = 1; bool manual = 2; uint64 debug_time_between_steps_msec = 3; + uint64 min_block_time_msec = 4; } diff --git a/runtime/defaults/defaults.go b/runtime/defaults/defaults.go index 6a66726b3..484a3afaf 100644 --- a/runtime/defaults/defaults.go +++ b/runtime/defaults/defaults.go @@ -43,6 +43,7 @@ var ( DefaultPacemakerTimeoutMsec = uint64(10000) DefaultPacemakerManual = true DefaultPacemakerDebugTimeBetweenStepsMsec = uint64(1000) + DefaultPacemakerMinBlockTimeMsec = uint64(5000) // utility DefaultUtilityMaxMempoolTransactionBytes = uint64(1024 ^ 3) // 1GB V0 defaults DefaultUtilityMaxMempoolTransactions = uint32(9000) diff --git a/runtime/docs/CHANGELOG.md b/runtime/docs/CHANGELOG.md index 410e0664c..12e3a5078 100644 --- a/runtime/docs/CHANGELOG.md +++ b/runtime/docs/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Add a new MinBlockTimeMsec config field to consensus + ## [0.0.0.44] - 2023-06-26 - Add a new ServiceConfig field to servicer config diff --git a/runtime/manager_test.go b/runtime/manager_test.go index dbd1b009f..8f0dbb0f9 100644 --- a/runtime/manager_test.go +++ b/runtime/manager_test.go @@ -1775,6 +1775,7 @@ func TestNewManagerFromReaders(t *testing.T) { TimeoutMsec: 10000, Manual: true, DebugTimeBetweenStepsMsec: 1000, + MinBlockTimeMsec: 5000, }, ServerModeEnabled: true, }, From 4a25b62f067773fe47e55a7981c28d6c3b7c2954 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Tue, 25 Jul 2023 16:40:16 +0200 Subject: [PATCH 02/25] [Consensus] Feat: Configurable min block production time --- consensus/e2e_tests/pacemaker_test.go | 77 ++++++++++++---- consensus/e2e_tests/utils_test.go | 2 + consensus/hotstuff_leader.go | 2 +- consensus/pacemaker/module.go | 123 +++++++++++++++----------- 4 files changed, 136 insertions(+), 68 deletions(-) diff --git a/consensus/e2e_tests/pacemaker_test.go b/consensus/e2e_tests/pacemaker_test.go index e5e996b92..76a821dfc 100644 --- a/consensus/e2e_tests/pacemaker_test.go +++ b/consensus/e2e_tests/pacemaker_test.go @@ -172,13 +172,18 @@ func forcePacemakerTimeout(t *testing.T, clockMock *clock.Mock, paceMakerTimeout advanceTime(t, clockMock, paceMakerTimeout+10*time.Millisecond) } +// TODO: Add more tests for minBlockTime behavior: +// 1. Block preparation triggers ASAP if conditions are met AFTER minBlockTime has triggered. +// 2. Block preparation is always discarded if a new one with better QC is received within minBlockTime. +// 3. Mempool reaped is the one present at minBlockTime or later. +// 4. Successive blocks timings are at least minBlockTime apart. func TestPacemakerMinBlockTime(t *testing.T) { // Test preparation clockMock := clock.NewMock() timeReminder(t, clockMock, time.Second) // UnitTestNet configs - paceMakerTimeoutMsec := uint64(10000) + paceMakerTimeoutMsec := uint64(300000) consensusMessageTimeout := time.Duration(paceMakerTimeoutMsec / 5) // Must be smaller than pacemaker timeout because we expect a deterministic number of consensus messages. paceMakerMinBlockTimeMsec := uint64(5000) // Make sure it is larger than the consensus message timeout runtimeMgrs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) @@ -195,31 +200,73 @@ func TestPacemakerMinBlockTime(t *testing.T) { err := StartAllTestPocketNodes(t, pocketNodes) require.NoError(t, err) + replicas := IdToNodeMapping{} + // First round ever has leaderId=2 ((height+round+step-1)%numValidators)+1 + // See: consensus/leader_election/module.go#electNextLeaderDeterministicRoundRobin + leaderId := typesCons.NodeId(2) + leader := IdToNodeMapping{} + numReplicas := len(pocketNodes) - 1 + // Debug message to start consensus by triggering next view - for _, pocketNode := range pocketNodes { + // Get leader out of replica set + for id, pocketNode := range pocketNodes { TriggerNextView(t, pocketNode) - } + if id == leaderId { + leader[id] = pocketNode + } else { + replicas[id] = pocketNode + } - consMod := pocketNodes[1].GetBus().GetConsensusModule() + // Right after triggering the next view + // Consensus started and all nodes are at NewRound step + step := typesCons.HotstuffStep(pocketNode.GetBus().GetConsensusModule().CurrentStep()) + require.Equal(t, consensus.NewRound, step) + } - newRoundMessages, err := waitForProposalMsgs(t, clockMock, eventsChannel, pocketNodes, 1, uint8(consensus.NewRound), 0, 0, numValidators*numValidators, consensusMessageTimeout, true) + newRoundMessages, err := WaitForNetworkConsensusEvents( + t, clockMock, eventsChannel, typesCons.HotstuffStep(consensus.NewRound), typesCons.HotstuffMessageType(consensus.NewRound), + numReplicas, // We want new round messages from replicas only + consensusMessageTimeout, false, + ) require.NoError(t, err) - whenBroadcast := clockMock.Now() - broadcastMessages(t, newRoundMessages, pocketNodes) - finishedBroadcast := uint64(clockMock.Now().Sub(whenBroadcast).Milliseconds()) - beforePrepareTimeout := time.Duration((paceMakerMinBlockTimeMsec - finishedBroadcast) * uint64(time.Millisecond)) - step := typesCons.HotstuffStep(consMod.CurrentStep()) + // Broadcast new round messages to leader to build a block + broadcastMessages(t, newRoundMessages, leader) + + var step typesCons.HotstuffStep + var pivotTime = 1 * time.Millisecond // Min time it takes to switch from NewRound to Prepare step + + // Give go routines time to trigger + advanceTime(t, clockMock, 0) + + // We get consensus module from leader to get its POV + leaderConsensusModule := leader[leaderId].GetBus().GetConsensusModule() + + // Make sure all nodes are aligned to the same leader + for _, pocketNode := range pocketNodes { + nodeLeader := pocketNode.GetBus().GetConsensusModule().GetLeaderForView(1, 0, uint8(consensus.NewRound)) + require.Equal(t, typesCons.NodeId(nodeLeader), leaderId) + } + + // Timer is blocking the proposal step + step = typesCons.HotstuffStep(leaderConsensusModule.CurrentStep()) require.Equal(t, consensus.NewRound, step) - advanceTime(t, clockMock, clock.Duration(beforePrepareTimeout)) - step = typesCons.HotstuffStep(consMod.CurrentStep()) + // Advance time right before minBlockTime triggers + advanceTime(t, clockMock, time.Duration(paceMakerMinBlockTimeMsec*uint64(time.Millisecond))-pivotTime) + // Should still be blocking proposal step + step = typesCons.HotstuffStep(leaderConsensusModule.CurrentStep()) require.Equal(t, consensus.NewRound, step) - //advanceTime(t, clockMock, 8000*time.Millisecond) - //step = typesCons.HotstuffStep(consMod.CurrentStep()) - //require.Equal(t, consensus.Prepare, step) + // Advance time just enough to trigger minBlockTime + advanceTime(t, clockMock, pivotTime) + step = typesCons.HotstuffStep(leaderConsensusModule.CurrentStep()) + + // Time advanced by minBlockTime + require.Equal(t, uint64(clockMock.Now().UnixMilli()), paceMakerMinBlockTimeMsec) + // Leader is at proposal step + require.Equal(t, consensus.Prepare, step) } // TODO: Implement these tests and use them as a starting point for new ones. Consider using ChatGPT to help you out :) diff --git a/consensus/e2e_tests/utils_test.go b/consensus/e2e_tests/utils_test.go index fef2fbc00..780c72d66 100644 --- a/consensus/e2e_tests/utils_test.go +++ b/consensus/e2e_tests/utils_test.go @@ -840,6 +840,8 @@ func advanceTime(t *testing.T, clck *clock.Mock, duration time.Duration) { clck.Add(duration) t.Logf("[⌚ CLOCK ⏩] advanced by %v", duration) logTime(t, clck) + // Give goroutines a chance to run + clck.Add(0) } // sleep pauses the goroutine for the given duration on the mock clock and logs what just happened. diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 0fa6889d4..542ebc0f3 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -62,7 +62,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *consensusM // TODO: Add test to make sure same block is not applied twice if round is interrupted after being 'Applied'. // TODO: Add more unit tests for these checks... if m.shouldPrepareNewBlock(highPrepareQC) { - doPrepare := m.paceMaker.ProcessDelayedBlockPrepare() + doPrepare := <-m.paceMaker.ProcessDelayedBlockPrepare(m.height) if !doPrepare { m.logger.Info().Msg("skip prepare new block") return diff --git a/consensus/pacemaker/module.go b/consensus/pacemaker/module.go index e096ea822..bf99558b4 100644 --- a/consensus/pacemaker/module.go +++ b/consensus/pacemaker/module.go @@ -39,7 +39,7 @@ type Pacemaker interface { PacemakerDebug ShouldHandleMessage(message *typesCons.HotstuffMessage) (bool, error) - ProcessDelayedBlockPrepare() bool + ProcessDelayedBlockPrepare(height uint64) chan bool RestartTimer() NewHeight() @@ -70,6 +70,7 @@ type latestPrepareRequest struct { cancelFunc context.CancelFunc blockProposed bool deadlinePassed bool + delayedHeight uint64 } func CreatePacemaker(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { @@ -103,6 +104,7 @@ func (*pacemaker) Create(bus modules.Bus, options ...modules.ModuleOption) (modu cancelFunc: nil, blockProposed: false, deadlinePassed: false, + delayedHeight: 0, } return m, nil @@ -111,7 +113,6 @@ func (*pacemaker) Create(bus modules.Bus, options ...modules.ModuleOption) (modu func (m *pacemaker) Start() error { m.logger = logger.Global.CreateLoggerForModule(m.GetModuleName()) m.RestartTimer() - m.RestartBlockProposalTimer() return nil } @@ -212,45 +213,6 @@ func (m *pacemaker) RestartTimer() { }() } -func (m *pacemaker) RestartBlockProposalTimer() { - m.latestPrepareRequest.m.Lock() - defer m.latestPrepareRequest.m.Unlock() - - if m.latestPrepareRequest.ch != nil { - m.latestPrepareRequest.ch <- false - } - - if m.latestPrepareRequest.cancelFunc != nil { - m.latestPrepareRequest.cancelFunc() - } - - m.latestPrepareRequest.blockProposed = false - m.latestPrepareRequest.deadlinePassed = false - - ctx, cancel := context.WithCancel(context.TODO()) - m.latestPrepareRequest.cancelFunc = cancel - clock := m.GetBus().GetRuntimeMgr().GetClock() - minBlockTime := time.Duration(m.pacemakerCfg.MinBlockTimeMsec * uint64(time.Millisecond)) - - go func() { - select { - case <-ctx.Done(): - return - case <-clock.After(minBlockTime): - m.latestPrepareRequest.m.Lock() - defer m.latestPrepareRequest.m.Unlock() - - if m.latestPrepareRequest.ch != nil { - m.latestPrepareRequest.blockProposed = true - m.latestPrepareRequest.ch <- true - m.latestPrepareRequest.ch = nil - } - - m.latestPrepareRequest.deadlinePassed = true - } - }() -} - func (m *pacemaker) InterruptRound(reason string) { defer m.RestartTimer() @@ -285,7 +247,6 @@ func (m *pacemaker) InterruptRound(reason string) { func (m *pacemaker) NewHeight() { defer m.RestartTimer() - defer m.RestartBlockProposalTimer() consensusMod := m.GetBus().GetConsensusModule() consensusMod.ResetRound(true) @@ -304,27 +265,85 @@ func (m *pacemaker) NewHeight() { ) } -func (m *pacemaker) ProcessDelayedBlockPrepare() bool { +// This is called each time there is a NewRound message received by the leader from replicas +// With the introduction of MinBlockTimeMsec delay, multiple concurrent calls may happen +// It makes sure that: +// - Block proposal is made by only one of the possible `HotstuffLeaderMessageHandler.HandleNewRoundMessage()` concurrent (because delayed) calls +// - If the timer expires, the first call to this method will trigger the block proposal +// - If a late message is received after a block is proposed by another call, the late message is discarded +// - Reads and affectations to pacemaker.latestPrepareRequest state are protected by a mutex +func (m *pacemaker) ProcessDelayedBlockPrepare(currentHeight uint64) chan bool { m.latestPrepareRequest.m.Lock() + defer m.latestPrepareRequest.m.Unlock() - if m.latestPrepareRequest.blockProposed { - return false + // Prepare channel for the the current request + ch := make(chan bool) + + // First time to build a block for current height, cancel previous timer if any + if m.latestPrepareRequest.delayedHeight < currentHeight { + // Discard the request building the old height + if m.latestPrepareRequest.ch != nil { + m.latestPrepareRequest.ch <- false + } + // Discard the timer for the old height + if m.latestPrepareRequest.cancelFunc != nil { + m.latestPrepareRequest.cancelFunc() + } + + m.latestPrepareRequest.ch = ch + m.latestPrepareRequest.blockProposed = false + m.latestPrepareRequest.deadlinePassed = false + m.latestPrepareRequest.delayedHeight = currentHeight + + // DISCUSS: This may be the the wrong time/place to start the timer, + // this means that its starts after the first NewRound message satisfying quorum is received + // We may start it when first NewRound message ever is received + minBlockTime := time.Duration(m.pacemakerCfg.MinBlockTimeMsec * uint64(time.Millisecond)) + ctx, cancel := context.WithCancel(context.TODO()) + m.latestPrepareRequest.cancelFunc = cancel + + go func() { + select { + case <-ctx.Done(): + return + case <-m.GetBus().GetRuntimeMgr().GetClock().After(minBlockTime): + m.latestPrepareRequest.m.Lock() + defer m.latestPrepareRequest.m.Unlock() + + // After the timeout, if there was any candidate request waiting for a signal, tell it to build the block + if m.latestPrepareRequest.ch != nil { + m.latestPrepareRequest.ch <- true + m.latestPrepareRequest.blockProposed = true + } + + // From now on, build the block ASAP + m.latestPrepareRequest.deadlinePassed = true + } + }() + + return ch } - if m.latestPrepareRequest.deadlinePassed { - return true + if m.latestPrepareRequest.blockProposed { + go func() { ch <- false }() + + return ch } - // there is already a block preparer candidate, we cancel it and start a new one - // DISCUSS: This is needed if we want to get the latest QC for the block proposal. if m.latestPrepareRequest.ch != nil { m.latestPrepareRequest.ch <- false } - m.latestPrepareRequest.ch = make(chan bool) - m.latestPrepareRequest.m.Unlock() + m.latestPrepareRequest.ch = ch + + if m.latestPrepareRequest.deadlinePassed { + go func() { + ch <- true + m.latestPrepareRequest.blockProposed = true + }() + } - return <-m.latestPrepareRequest.ch + return ch } func (m *pacemaker) startNextView(qc *typesCons.QuorumCertificate, forceNextView bool) { From 1028ae9dd6484b608463125e5f959276fbf333b0 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Wed, 26 Jul 2023 17:17:01 +0200 Subject: [PATCH 03/25] [Consensus] Refactor: decouple timer registration from subscription --- consensus/doc/CHANGELOG.md | 2 - consensus/e2e_tests/pacemaker_test.go | 27 +++- consensus/hotstuff_leader.go | 10 +- consensus/pacemaker/module.go | 151 +++++++++---------- runtime/configs/proto/consensus_config.proto | 2 +- runtime/defaults/defaults.go | 2 +- runtime/docs/CHANGELOG.md | 2 - 7 files changed, 103 insertions(+), 93 deletions(-) diff --git a/consensus/doc/CHANGELOG.md b/consensus/doc/CHANGELOG.md index 4bb70a11d..4d5b89930 100644 --- a/consensus/doc/CHANGELOG.md +++ b/consensus/doc/CHANGELOG.md @@ -7,8 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- Implement minimum block production pace by delaying block preparation - ## [0.0.0.54] - 2023-06-13 - Fix tests diff --git a/consensus/e2e_tests/pacemaker_test.go b/consensus/e2e_tests/pacemaker_test.go index 76a821dfc..7af5c5ffb 100644 --- a/consensus/e2e_tests/pacemaker_test.go +++ b/consensus/e2e_tests/pacemaker_test.go @@ -172,12 +172,7 @@ func forcePacemakerTimeout(t *testing.T, clockMock *clock.Mock, paceMakerTimeout advanceTime(t, clockMock, paceMakerTimeout+10*time.Millisecond) } -// TODO: Add more tests for minBlockTime behavior: -// 1. Block preparation triggers ASAP if conditions are met AFTER minBlockTime has triggered. -// 2. Block preparation is always discarded if a new one with better QC is received within minBlockTime. -// 3. Mempool reaped is the one present at minBlockTime or later. -// 4. Successive blocks timings are at least minBlockTime apart. -func TestPacemakerMinBlockTime(t *testing.T) { +func TestPacemaker_MinBlockTime(t *testing.T) { // Test preparation clockMock := clock.NewMock() timeReminder(t, clockMock, time.Second) @@ -269,6 +264,26 @@ func TestPacemakerMinBlockTime(t *testing.T) { require.Equal(t, consensus.Prepare, step) } +// TODO: Block preparation triggers ASAP if conditions are met AFTER minBlockTime has triggered. +func TestPacemaker_MinBlockTime_BlockPrepAsapAfterTrigger(t *testing.T) { + t.Skip() +} + +// TODO: Block preparation is always discarded if a new one with better QC is received within minBlockTime. +func TestPacemaker_MinBlockTime_AllowOnlyLatestBlockPrep(t *testing.T) { + t.Skip() +} + +// TODO: Mempool reaped is the one present at minBlockTime or later. +func TestPacemaker_MinBlockTime_DelayReapMempool(t *testing.T) { + t.Skip() +} + +// TODO: Successive blocks timings are at least minBlockTime apart. +func TestPacemaker_MinBlockTime_BehaviorAcrossMultipleBlocks(t *testing.T) { + t.Skip() +} + // TODO: Implement these tests and use them as a starting point for new ones. Consider using ChatGPT to help you out :) func TestPacemakerDifferentHeightsCatchup(t *testing.T) { diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 542ebc0f3..3b4da94cb 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -62,9 +62,13 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *consensusM // TODO: Add test to make sure same block is not applied twice if round is interrupted after being 'Applied'. // TODO: Add more unit tests for these checks... if m.shouldPrepareNewBlock(highPrepareQC) { - doPrepare := <-m.paceMaker.ProcessDelayedBlockPrepare(m.height) - if !doPrepare { - m.logger.Info().Msg("skip prepare new block") + // Place this where we want to start the timer + // Placed here, timer starts when we receive enough NewRound messages + m.paceMaker.RegisterMinBlockTimeDelay() + + // This function delays block preparation and returns false if a concurrent preparation request with higher QC is available + if shouldPrepareBlock := m.paceMaker.DelayBlockPreparation(); !shouldPrepareBlock { + m.logger.Info().Msg("skip prepare new block, a candidate with higher QC is available") return } block, err := m.prepareBlock(highPrepareQC) diff --git a/consensus/pacemaker/module.go b/consensus/pacemaker/module.go index bf99558b4..b32ccbdda 100644 --- a/consensus/pacemaker/module.go +++ b/consensus/pacemaker/module.go @@ -39,7 +39,8 @@ type Pacemaker interface { PacemakerDebug ShouldHandleMessage(message *typesCons.HotstuffMessage) (bool, error) - ProcessDelayedBlockPrepare(height uint64) chan bool + RegisterMinBlockTimeDelay() + DelayBlockPreparation() bool RestartTimer() NewHeight() @@ -50,10 +51,10 @@ type pacemaker struct { base_modules.IntegrableModule base_modules.InterruptableModule - pacemakerCfg *configs.PacemakerConfig - roundTimeout time.Duration - roundCancelFunc context.CancelFunc - latestPrepareRequest latestPrepareRequest + pacemakerCfg *configs.PacemakerConfig + roundTimeout time.Duration + roundCancelFunc context.CancelFunc + prepareStepDelayer prepareStepDelayer // Only used for development and debugging. debug pacemakerDebug @@ -64,13 +65,13 @@ type pacemaker struct { } // Structure to handle delaying block preparation (reaping the block mempool) -type latestPrepareRequest struct { - m sync.Mutex - ch chan bool - cancelFunc context.CancelFunc +// by adding a delay before the next prepare request +type prepareStepDelayer struct { + m sync.Mutex // Mutex locking access to this structure and prevent inconsistent state + ch chan bool // Whenever there is a block proposal request arriving before the timeout, this channel is used to signal to it to whether build the block or not (if better candidate request with higher QC) + cancelFunc context.CancelFunc // This is used to cancel an ongoing timeout. It should not happen, but future code changes may no longer preserve this guarantee, so this feature maintain its own cancellation logic blockProposed bool deadlinePassed bool - delayedHeight uint64 } func CreatePacemaker(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { @@ -98,13 +99,12 @@ func (*pacemaker) Create(bus modules.Bus, options ...modules.ModuleOption) (modu debugTimeBetweenStepsMsec: m.pacemakerCfg.GetDebugTimeBetweenStepsMsec(), quorumCertificate: nil, } - m.latestPrepareRequest = latestPrepareRequest{ + m.prepareStepDelayer = prepareStepDelayer{ m: sync.Mutex{}, ch: nil, cancelFunc: nil, blockProposed: false, deadlinePassed: false, - delayedHeight: 0, } return m, nil @@ -265,85 +265,80 @@ func (m *pacemaker) NewHeight() { ) } -// This is called each time there is a NewRound message received by the leader from replicas -// With the introduction of MinBlockTimeMsec delay, multiple concurrent calls may happen -// It makes sure that: -// - Block proposal is made by only one of the possible `HotstuffLeaderMessageHandler.HandleNewRoundMessage()` concurrent (because delayed) calls -// - If the timer expires, the first call to this method will trigger the block proposal -// - If a late message is received after a block is proposed by another call, the late message is discarded -// - Reads and affectations to pacemaker.latestPrepareRequest state are protected by a mutex -func (m *pacemaker) ProcessDelayedBlockPrepare(currentHeight uint64) chan bool { - m.latestPrepareRequest.m.Lock() - defer m.latestPrepareRequest.m.Unlock() +// This is called each time the system wants a delayed signal to build a block +func (m *pacemaker) RegisterMinBlockTimeDelay() { + // Discard any previous timer + // DISCUSS: This should not happen, Identify cases where an active timer has to be discarded, remove cancellation logic if none + if m.prepareStepDelayer.cancelFunc != nil { + m.prepareStepDelayer.cancelFunc() + } - // Prepare channel for the the current request - ch := make(chan bool) + m.prepareStepDelayer.blockProposed = false + m.prepareStepDelayer.deadlinePassed = false - // First time to build a block for current height, cancel previous timer if any - if m.latestPrepareRequest.delayedHeight < currentHeight { - // Discard the request building the old height - if m.latestPrepareRequest.ch != nil { - m.latestPrepareRequest.ch <- false - } - // Discard the timer for the old height - if m.latestPrepareRequest.cancelFunc != nil { - m.latestPrepareRequest.cancelFunc() - } + minBlockTime := time.Duration(m.pacemakerCfg.MinBlockTimeMsec * uint64(time.Millisecond)) + ctx, cancel := context.WithCancel(context.TODO()) + m.prepareStepDelayer.cancelFunc = cancel - m.latestPrepareRequest.ch = ch - m.latestPrepareRequest.blockProposed = false - m.latestPrepareRequest.deadlinePassed = false - m.latestPrepareRequest.delayedHeight = currentHeight - - // DISCUSS: This may be the the wrong time/place to start the timer, - // this means that its starts after the first NewRound message satisfying quorum is received - // We may start it when first NewRound message ever is received - minBlockTime := time.Duration(m.pacemakerCfg.MinBlockTimeMsec * uint64(time.Millisecond)) - ctx, cancel := context.WithCancel(context.TODO()) - m.latestPrepareRequest.cancelFunc = cancel - - go func() { - select { - case <-ctx.Done(): - return - case <-m.GetBus().GetRuntimeMgr().GetClock().After(minBlockTime): - m.latestPrepareRequest.m.Lock() - defer m.latestPrepareRequest.m.Unlock() - - // After the timeout, if there was any candidate request waiting for a signal, tell it to build the block - if m.latestPrepareRequest.ch != nil { - m.latestPrepareRequest.ch <- true - m.latestPrepareRequest.blockProposed = true - } - - // From now on, build the block ASAP - m.latestPrepareRequest.deadlinePassed = true + go func() { + select { + case <-ctx.Done(): + return + case <-m.GetBus().GetRuntimeMgr().GetClock().After(minBlockTime): + m.prepareStepDelayer.m.Lock() + defer m.prepareStepDelayer.m.Unlock() + + // After the timeout, if there was any candidate request waiting for a signal, tell it to build the block + if m.prepareStepDelayer.ch != nil { + m.prepareStepDelayer.ch <- true + close(m.prepareStepDelayer.ch) + m.prepareStepDelayer.blockProposed = true } - }() - return ch - } + // From now on, build the block ASAP + m.prepareStepDelayer.deadlinePassed = true - if m.latestPrepareRequest.blockProposed { - go func() { ch <- false }() + // No need to cancel the context anymore + m.prepareStepDelayer.cancelFunc = nil + } + }() +} - return ch +// This is called when conditions are met by the leader to build a block but still needs to wait for the MinBlockTimeMsec delay before reaping the mempool +// With MinBlockTimeMsec delay, multiple concurrent calls may happen +// It makes sure that: +// - Block proposal is made by only one of the possible `HotstuffLeaderMessageHandler.HandleNewRoundMessage()` concurrent (because delayed) calls +// - If the timer expires, the first call to this method will trigger the block proposal +// - If a late message is received AFTER the a block is marked as proposed by another call, the late message is discarded +// - Reads and affectations to pacemaker.DelayBlockPreparation state are protected by a mutex +func (m *pacemaker) DelayBlockPreparation() bool { + m.prepareStepDelayer.m.Lock() + + if m.prepareStepDelayer.blockProposed { + m.prepareStepDelayer.m.Unlock() + return false } - if m.latestPrepareRequest.ch != nil { - m.latestPrepareRequest.ch <- false + // If there already is a channel signaling block proposal, make sure it gives up + if m.prepareStepDelayer.ch != nil { + m.prepareStepDelayer.ch <- false + close(m.prepareStepDelayer.ch) } - m.latestPrepareRequest.ch = ch - - if m.latestPrepareRequest.deadlinePassed { - go func() { - ch <- true - m.latestPrepareRequest.blockProposed = true - }() + // Deadline has passed, no need to have a channel, propose a block now + if m.prepareStepDelayer.deadlinePassed { + m.prepareStepDelayer.blockProposed = true + m.prepareStepDelayer.m.Unlock() + return true } - return ch + // We still need to wait, create a channel and discard the old candidate if any + ch := make(chan bool) + m.prepareStepDelayer.ch = ch + // We cannot defer the unlock here because the channel read is blocking + m.prepareStepDelayer.m.Unlock() + + return <-ch } func (m *pacemaker) startNextView(qc *typesCons.QuorumCertificate, forceNextView bool) { diff --git a/runtime/configs/proto/consensus_config.proto b/runtime/configs/proto/consensus_config.proto index 5635a620d..e13f494b4 100644 --- a/runtime/configs/proto/consensus_config.proto +++ b/runtime/configs/proto/consensus_config.proto @@ -15,5 +15,5 @@ message PacemakerConfig { uint64 timeout_msec = 1; bool manual = 2; uint64 debug_time_between_steps_msec = 3; - uint64 min_block_time_msec = 4; + uint64 min_block_time_msec = 4; // consenus protocol could produce blocks as soon as quorum is reached. This option allows to set min time between blocks and give more time to the mempool to fill up } diff --git a/runtime/defaults/defaults.go b/runtime/defaults/defaults.go index 484a3afaf..ed3febdb6 100644 --- a/runtime/defaults/defaults.go +++ b/runtime/defaults/defaults.go @@ -43,7 +43,7 @@ var ( DefaultPacemakerTimeoutMsec = uint64(10000) DefaultPacemakerManual = true DefaultPacemakerDebugTimeBetweenStepsMsec = uint64(1000) - DefaultPacemakerMinBlockTimeMsec = uint64(5000) + DefaultPacemakerMinBlockTimeMsec = uint64(5000) // 5 seconds // utility DefaultUtilityMaxMempoolTransactionBytes = uint64(1024 ^ 3) // 1GB V0 defaults DefaultUtilityMaxMempoolTransactions = uint32(9000) diff --git a/runtime/docs/CHANGELOG.md b/runtime/docs/CHANGELOG.md index 12e3a5078..410e0664c 100644 --- a/runtime/docs/CHANGELOG.md +++ b/runtime/docs/CHANGELOG.md @@ -7,8 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- Add a new MinBlockTimeMsec config field to consensus - ## [0.0.0.44] - 2023-06-26 - Add a new ServiceConfig field to servicer config From a7fd743b4a0cf8a74d321b5b71aeff69dfdb9ada Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Mon, 31 Jul 2023 10:04:13 +0200 Subject: [PATCH 04/25] address review comments --- consensus/e2e_tests/pacemaker_test.go | 27 ++++---- consensus/e2e_tests/utils_test.go | 2 - consensus/hotstuff_leader.go | 5 +- consensus/pacemaker/module.go | 67 +++++++++++--------- runtime/configs/proto/consensus_config.proto | 4 +- 5 files changed, 52 insertions(+), 53 deletions(-) diff --git a/consensus/e2e_tests/pacemaker_test.go b/consensus/e2e_tests/pacemaker_test.go index 7af5c5ffb..be6622cde 100644 --- a/consensus/e2e_tests/pacemaker_test.go +++ b/consensus/e2e_tests/pacemaker_test.go @@ -172,7 +172,7 @@ func forcePacemakerTimeout(t *testing.T, clockMock *clock.Mock, paceMakerTimeout advanceTime(t, clockMock, paceMakerTimeout+10*time.Millisecond) } -func TestPacemaker_MinBlockTime(t *testing.T) { +func TestPacemaker_MinBlockTime_DelayBlockPrep(t *testing.T) { // Test preparation clockMock := clock.NewMock() timeReminder(t, clockMock, time.Second) @@ -180,7 +180,7 @@ func TestPacemaker_MinBlockTime(t *testing.T) { // UnitTestNet configs paceMakerTimeoutMsec := uint64(300000) consensusMessageTimeout := time.Duration(paceMakerTimeoutMsec / 5) // Must be smaller than pacemaker timeout because we expect a deterministic number of consensus messages. - paceMakerMinBlockTimeMsec := uint64(5000) // Make sure it is larger than the consensus message timeout + paceMakerMinBlockTimeMsec := uint64(paceMakerTimeoutMsec / 6) runtimeMgrs := GenerateNodeRuntimeMgrs(t, numValidators, clockMock) for _, runtimeConfig := range runtimeMgrs { consCfg := runtimeConfig.GetConfig().Consensus.PacemakerConfig @@ -195,22 +195,15 @@ func TestPacemaker_MinBlockTime(t *testing.T) { err := StartAllTestPocketNodes(t, pocketNodes) require.NoError(t, err) - replicas := IdToNodeMapping{} // First round ever has leaderId=2 ((height+round+step-1)%numValidators)+1 // See: consensus/leader_election/module.go#electNextLeaderDeterministicRoundRobin leaderId := typesCons.NodeId(2) - leader := IdToNodeMapping{} numReplicas := len(pocketNodes) - 1 // Debug message to start consensus by triggering next view // Get leader out of replica set - for id, pocketNode := range pocketNodes { + for _, pocketNode := range pocketNodes { TriggerNextView(t, pocketNode) - if id == leaderId { - leader[id] = pocketNode - } else { - replicas[id] = pocketNode - } // Right after triggering the next view // Consensus started and all nodes are at NewRound step @@ -226,16 +219,16 @@ func TestPacemaker_MinBlockTime(t *testing.T) { require.NoError(t, err) // Broadcast new round messages to leader to build a block - broadcastMessages(t, newRoundMessages, leader) + broadcastMessages(t, newRoundMessages, IdToNodeMapping{leaderId: pocketNodes[leaderId]}) var step typesCons.HotstuffStep - var pivotTime = 1 * time.Millisecond // Min time it takes to switch from NewRound to Prepare step + minTimeIncrement := 1 * time.Millisecond // Min time it takes to switch from NewRound to Prepare step // Give go routines time to trigger advanceTime(t, clockMock, 0) // We get consensus module from leader to get its POV - leaderConsensusModule := leader[leaderId].GetBus().GetConsensusModule() + leaderConsensusModule := pocketNodes[leaderId].GetBus().GetConsensusModule() // Make sure all nodes are aligned to the same leader for _, pocketNode := range pocketNodes { @@ -248,14 +241,15 @@ func TestPacemaker_MinBlockTime(t *testing.T) { require.Equal(t, consensus.NewRound, step) // Advance time right before minBlockTime triggers - advanceTime(t, clockMock, time.Duration(paceMakerMinBlockTimeMsec*uint64(time.Millisecond))-pivotTime) + paceMakerPreActivationTime := time.Duration(paceMakerMinBlockTimeMsec*uint64(time.Millisecond)) - minTimeIncrement + advanceTime(t, clockMock, paceMakerPreActivationTime) // Should still be blocking proposal step step = typesCons.HotstuffStep(leaderConsensusModule.CurrentStep()) require.Equal(t, consensus.NewRound, step) // Advance time just enough to trigger minBlockTime - advanceTime(t, clockMock, pivotTime) + advanceTime(t, clockMock, minTimeIncrement) step = typesCons.HotstuffStep(leaderConsensusModule.CurrentStep()) // Time advanced by minBlockTime @@ -275,12 +269,13 @@ func TestPacemaker_MinBlockTime_AllowOnlyLatestBlockPrep(t *testing.T) { } // TODO: Mempool reaped is the one present at minBlockTime or later. +// Since leader could just reap an earlier mempool and just blocks broadcasting the proposal func TestPacemaker_MinBlockTime_DelayReapMempool(t *testing.T) { t.Skip() } // TODO: Successive blocks timings are at least minBlockTime apart. -func TestPacemaker_MinBlockTime_BehaviorAcrossMultipleBlocks(t *testing.T) { +func TestPacemaker_MinBlockTime_ConsecutiveBlocksAtLeastMinBlockTimeApart(t *testing.T) { t.Skip() } diff --git a/consensus/e2e_tests/utils_test.go b/consensus/e2e_tests/utils_test.go index 780c72d66..fef2fbc00 100644 --- a/consensus/e2e_tests/utils_test.go +++ b/consensus/e2e_tests/utils_test.go @@ -840,8 +840,6 @@ func advanceTime(t *testing.T, clck *clock.Mock, duration time.Duration) { clck.Add(duration) t.Logf("[⌚ CLOCK ⏩] advanced by %v", duration) logTime(t, clck) - // Give goroutines a chance to run - clck.Add(0) } // sleep pauses the goroutine for the given duration on the mock clock and logs what just happened. diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 3b4da94cb..cc067098e 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -62,9 +62,8 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *consensusM // TODO: Add test to make sure same block is not applied twice if round is interrupted after being 'Applied'. // TODO: Add more unit tests for these checks... if m.shouldPrepareNewBlock(highPrepareQC) { - // Place this where we want to start the timer - // Placed here, timer starts when we receive enough NewRound messages - m.paceMaker.RegisterMinBlockTimeDelay() + // Leader should prepare a new block. Introducing a delay based on configurations. + m.paceMaker.StartMinBlockTimeDelay() // This function delays block preparation and returns false if a concurrent preparation request with higher QC is available if shouldPrepareBlock := m.paceMaker.DelayBlockPreparation(); !shouldPrepareBlock { diff --git a/consensus/pacemaker/module.go b/consensus/pacemaker/module.go index b32ccbdda..6734a471c 100644 --- a/consensus/pacemaker/module.go +++ b/consensus/pacemaker/module.go @@ -39,7 +39,7 @@ type Pacemaker interface { PacemakerDebug ShouldHandleMessage(message *typesCons.HotstuffMessage) (bool, error) - RegisterMinBlockTimeDelay() + StartMinBlockTimeDelay() DelayBlockPreparation() bool RestartTimer() @@ -64,14 +64,15 @@ type pacemaker struct { logPrefix string } -// Structure to handle delaying block preparation (reaping the block mempool) -// by adding a delay before the next prepare request +// prepareStepDelayer delays block preparation (mempool reaping) +// by adding a delay before the next prepare request to prevent block creation type prepareStepDelayer struct { - m sync.Mutex // Mutex locking access to this structure and prevent inconsistent state - ch chan bool // Whenever there is a block proposal request arriving before the timeout, this channel is used to signal to it to whether build the block or not (if better candidate request with higher QC) - cancelFunc context.CancelFunc // This is used to cancel an ongoing timeout. It should not happen, but future code changes may no longer preserve this guarantee, so this feature maintain its own cancellation logic - blockProposed bool - deadlinePassed bool + m sync.Mutex // mutex locking access to this structure and prevent inconsistent state + ch chan bool // whenever there is a block proposal request arriving before the timeout, this channel is used to signal to it to whether build the block or not (if better candidate request with higher QC) + cancelFunc context.CancelFunc // cancels an ongoing timeout. It should not happen, but future code changes may no longer preserve this guarantee, so this feature maintain its own cancellation logic + shouldProposeBlock bool // a flag to capture whether a block was proposed, so later calls will skip building the block + delayExhausted bool // the delay for this step/round has already passed, so should create the block ASAP + minBlockTime time.Duration // the minimum time to wait before trying to propose a block } func CreatePacemaker(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { @@ -100,11 +101,12 @@ func (*pacemaker) Create(bus modules.Bus, options ...modules.ModuleOption) (modu quorumCertificate: nil, } m.prepareStepDelayer = prepareStepDelayer{ - m: sync.Mutex{}, - ch: nil, - cancelFunc: nil, - blockProposed: false, - deadlinePassed: false, + m: sync.Mutex{}, + ch: nil, + cancelFunc: nil, + shouldProposeBlock: false, + delayExhausted: false, + minBlockTime: time.Duration(m.pacemakerCfg.MinBlockTimeMsec * uint64(time.Millisecond)), } return m, nil @@ -192,7 +194,6 @@ func (m *pacemaker) ShouldHandleMessage(msg *typesCons.HotstuffMessage) (bool, e func (m *pacemaker) RestartTimer() { // NOTE: Not deferring a cancel call because this function is asynchronous. - // DISCUSS: Should we have a lock to manipulate m.roundCancelFunc? if m.roundCancelFunc != nil { m.roundCancelFunc() } @@ -265,38 +266,41 @@ func (m *pacemaker) NewHeight() { ) } -// This is called each time the system wants a delayed signal to build a block -func (m *pacemaker) RegisterMinBlockTimeDelay() { - // Discard any previous timer - // DISCUSS: This should not happen, Identify cases where an active timer has to be discarded, remove cancellation logic if none +// StartMinBlockTimeDelay should be called when a delay should be introduced into proposing a new block +func (m *pacemaker) StartMinBlockTimeDelay() { + // Discard any previous timer if one exists if m.prepareStepDelayer.cancelFunc != nil { + m.logger.Warn().Msg("RegisterMinBlockTimeDelay has an existing timer which should not happen. Releasing for now...") m.prepareStepDelayer.cancelFunc() } - m.prepareStepDelayer.blockProposed = false - m.prepareStepDelayer.deadlinePassed = false + m.prepareStepDelayer.shouldProposeBlock = false + m.prepareStepDelayer.delayExhausted = false - minBlockTime := time.Duration(m.pacemakerCfg.MinBlockTimeMsec * uint64(time.Millisecond)) ctx, cancel := context.WithCancel(context.TODO()) m.prepareStepDelayer.cancelFunc = cancel + // Start a timer to wait for the MinBlockTimeMsec delay + // If a channel is provided, signal when the timer expires to it go func() { select { + // only called if the delay timer is explicitly cancelled case <-ctx.Done(): return - case <-m.GetBus().GetRuntimeMgr().GetClock().After(minBlockTime): + // called after the minimum block delay is exhausted meaning it is time to propose a new block + case <-m.GetBus().GetRuntimeMgr().GetClock().After(m.prepareStepDelayer.minBlockTime): m.prepareStepDelayer.m.Lock() defer m.prepareStepDelayer.m.Unlock() - // After the timeout, if there was any candidate request waiting for a signal, tell it to build the block + // After the timeout, if there was any `HotstuffLeaderMessageHandler.HandleNewRoundMessage()` call delayed to propose a block, unblock it by emitting true if m.prepareStepDelayer.ch != nil { m.prepareStepDelayer.ch <- true close(m.prepareStepDelayer.ch) - m.prepareStepDelayer.blockProposed = true + m.prepareStepDelayer.shouldProposeBlock = true } // From now on, build the block ASAP - m.prepareStepDelayer.deadlinePassed = true + m.prepareStepDelayer.delayExhausted = true // No need to cancel the context anymore m.prepareStepDelayer.cancelFunc = nil @@ -304,30 +308,31 @@ func (m *pacemaker) RegisterMinBlockTimeDelay() { }() } -// This is called when conditions are met by the leader to build a block but still needs to wait for the MinBlockTimeMsec delay before reaping the mempool +// DelayBlockPreparation is called when conditions are met by the leader to build a block but still needs to wait for the MinBlockTimeMsec delay before reaping the mempool. // With MinBlockTimeMsec delay, multiple concurrent calls may happen +// DelayBlockPreparation is a synchronous blocking function that waits for channel to emit whether to propose a block or not given multiple HotstuffLeaderMessageHandler.HandleNewRoundMessage calling this function concurrently. // It makes sure that: // - Block proposal is made by only one of the possible `HotstuffLeaderMessageHandler.HandleNewRoundMessage()` concurrent (because delayed) calls // - If the timer expires, the first call to this method will trigger the block proposal // - If a late message is received AFTER the a block is marked as proposed by another call, the late message is discarded -// - Reads and affectations to pacemaker.DelayBlockPreparation state are protected by a mutex +// - Reads and assignments to pacemaker.prepareStepDelayer state are protected by a mutex func (m *pacemaker) DelayBlockPreparation() bool { m.prepareStepDelayer.m.Lock() - if m.prepareStepDelayer.blockProposed { + if m.prepareStepDelayer.shouldProposeBlock { m.prepareStepDelayer.m.Unlock() return false } - // If there already is a channel signaling block proposal, make sure it gives up + // If there already is a channel signaling block proposal, make sure it does not propose a block if m.prepareStepDelayer.ch != nil { m.prepareStepDelayer.ch <- false close(m.prepareStepDelayer.ch) } // Deadline has passed, no need to have a channel, propose a block now - if m.prepareStepDelayer.deadlinePassed { - m.prepareStepDelayer.blockProposed = true + if m.prepareStepDelayer.delayExhausted { + m.prepareStepDelayer.shouldProposeBlock = true m.prepareStepDelayer.m.Unlock() return true } diff --git a/runtime/configs/proto/consensus_config.proto b/runtime/configs/proto/consensus_config.proto index e13f494b4..af81be3bc 100644 --- a/runtime/configs/proto/consensus_config.proto +++ b/runtime/configs/proto/consensus_config.proto @@ -15,5 +15,7 @@ message PacemakerConfig { uint64 timeout_msec = 1; bool manual = 2; uint64 debug_time_between_steps_msec = 3; - uint64 min_block_time_msec = 4; // consenus protocol could produce blocks as soon as quorum is reached. This option allows to set min time between blocks and give more time to the mempool to fill up + // consenus could produce blocks as soon as a quorum is reached; responsivness per the Hotstuff whitepaper. + // This option allows to set min time between blocks and gives more time to the mempool to fill up; similar to timeout_propose in Tendermint. + uint64 min_block_time_msec = 4; } From 867e8415d5c6e6c9e7e61e1a6b5a5d79be7207f6 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 24 Jul 2023 11:24:18 +0100 Subject: [PATCH 05/25] [Docs] Update development docs to warn to not use the changelog hook (#923) Co-authored-by: Daniel Olshansky --- docs/development/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/development/README.md b/docs/development/README.md index 351ec2da8..3f62d8a6b 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -86,9 +86,7 @@ Optionally activate changelog pre-commit hook cp .githooks/pre-commit .git/hooks/pre-commit chmod +x .git/hooks/pre-commit ``` - -_Please note that the Github workflow will still prevent this from merging -unless the CHANGELOG is updated._ +_**NOTE**: The pre-commit changelog verification has been disabled during the developement of V1 as of 2023-05-16 to unblock development velocity; see more details [here](https://github.com/pokt-network/pocket/assets/1892194/394fdb09-e388-44aa-820d-e9d5a23578cf). This check is no longer done in the CI and is not recommended for local development either currently._ ### Pocket Network CLI From 0d448c9aaa3ad1cff6b07d36232950d5e8318829 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 24 Jul 2023 22:47:00 +0100 Subject: [PATCH 06/25] [IBC] chore: Rename FlushAllEntries => FlushCachesToStore (#934) --- ibc/ibc_handle_event_test.go | 2 +- ibc/module.go | 2 +- ibc/store/bulk_store_cache.go | 6 +++--- ibc/store/provable_store.go | 4 ++-- ibc/store/provable_store_test.go | 12 ++++++------ internal/testutil/ibc/mock.go | 2 +- shared/modules/ibc_store_module.go | 4 ++-- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ibc/ibc_handle_event_test.go b/ibc/ibc_handle_event_test.go index 9d6ed3ebe..422f450f6 100644 --- a/ibc/ibc_handle_event_test.go +++ b/ibc/ibc_handle_event_test.go @@ -81,7 +81,7 @@ func TestHandleEvent_FlushCaches(t *testing.T) { require.NoError(t, cache.Stop()) // flush the cache - err = ibcHost.GetBus().GetBulkStoreCacher().FlushAllEntries() + err = ibcHost.GetBus().GetBulkStoreCacher().FlushCachesToStore() require.NoError(t, err) cache, err = kvstore.NewKVStore(tmpDir) diff --git a/ibc/module.go b/ibc/module.go index 24736e9d4..aae9b4581 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -95,7 +95,7 @@ func (m *ibcModule) HandleEvent(event *anypb.Any) error { } // Flush all caches to disk for last height bsc := m.GetBus().GetBulkStoreCacher() - if err := bsc.FlushAllEntries(); err != nil { + if err := bsc.FlushCachesToStore(); err != nil { return err } // Prune old cache entries diff --git a/ibc/store/bulk_store_cache.go b/ibc/store/bulk_store_cache.go index 953e9ca3b..0e71de3cf 100644 --- a/ibc/store/bulk_store_cache.go +++ b/ibc/store/bulk_store_cache.go @@ -124,8 +124,8 @@ func (s *bulkStoreCache) GetAllStores() map[string]modules.ProvableStore { return s.ls.stores } -// FlushAllEntries caches all the entries for all stores in the bulkStoreCache -func (s *bulkStoreCache) FlushAllEntries() error { +// FlushdCachesToStore caches all the entries for all stores in the bulkStoreCache +func (s *bulkStoreCache) FlushCachesToStore() error { s.ls.m.Lock() defer s.ls.m.Unlock() s.logger.Info().Msg("🚽 Flushing All Cache Entries to Disk 🚽") @@ -134,7 +134,7 @@ func (s *bulkStoreCache) FlushAllEntries() error { return err } for _, store := range s.ls.stores { - if err := store.FlushEntries(disk); err != nil { + if err := store.FlushCache(disk); err != nil { s.logger.Error().Err(err).Str("store", string(store.GetCommitmentPrefix())).Msg("🚨 Error Flushing Cache 🚨") return err } diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index c3ff8a171..df4612725 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -166,8 +166,8 @@ func (p *provableStore) Root() ics23.CommitmentRoot { return root } -// FlushEntries writes all local changes to disk and clears the in-memory cache -func (p *provableStore) FlushEntries(store kvstore.KVStore) error { +// FlushCache writes all local changes to disk and clears the in-memory cache +func (p *provableStore) FlushCache(store kvstore.KVStore) error { p.m.Lock() defer p.m.Unlock() for _, entry := range p.cache { diff --git a/ibc/store/provable_store_test.go b/ibc/store/provable_store_test.go index 10fdaf3c0..174d62827 100644 --- a/ibc/store/provable_store_test.go +++ b/ibc/store/provable_store_test.go @@ -148,7 +148,7 @@ func TestProvableStore_GetAndProve(t *testing.T) { } } -func TestProvableStore_FlushEntries(t *testing.T) { +func TestProvableStore_FlushCache(t *testing.T) { provableStore := newTestProvableStore(t) kvs := []struct { key []byte @@ -177,7 +177,7 @@ func TestProvableStore_FlushEntries(t *testing.T) { } } cache := kvstore.NewMemKVStore() - require.NoError(t, provableStore.FlushEntries(cache)) + require.NoError(t, provableStore.FlushCache(cache)) keys, values, err := cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, keys, 3) @@ -221,7 +221,7 @@ func TestProvableStore_PruneCache(t *testing.T) { } } cache := kvstore.NewMemKVStore() - require.NoError(t, provableStore.FlushEntries(cache)) + require.NoError(t, provableStore.FlushCache(cache)) keys, _, err := cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, keys, 3) // 3 entries in cache should be flushed to disk @@ -264,12 +264,12 @@ func TestProvableStore_RestoreCache(t *testing.T) { } cache := kvstore.NewMemKVStore() - require.NoError(t, provableStore.FlushEntries(cache)) + require.NoError(t, provableStore.FlushCache(cache)) keys, values, err := cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, keys, 3) require.NoError(t, cache.ClearAll()) - require.NoError(t, provableStore.FlushEntries(cache)) + require.NoError(t, provableStore.FlushCache(cache)) newKeys, _, err := cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, newKeys, 0) @@ -284,7 +284,7 @@ func TestProvableStore_RestoreCache(t *testing.T) { newKeys, _, err = cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, newKeys, 0) - require.NoError(t, provableStore.FlushEntries(cache)) + require.NoError(t, provableStore.FlushCache(cache)) newKeys, newValues, err := cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, newKeys, 3) diff --git a/internal/testutil/ibc/mock.go b/internal/testutil/ibc/mock.go index d1bc22728..03ab3faa3 100644 --- a/internal/testutil/ibc/mock.go +++ b/internal/testutil/ibc/mock.go @@ -74,7 +74,7 @@ func baseBulkStoreCacherMock(t gocuke.TestingT, bus modules.Bus) *mockModules.Mo storeMock.EXPECT().AddStore(gomock.Any()).Return(nil).AnyTimes() storeMock.EXPECT().GetStore(gomock.Any()).Return(provableStoreMock, nil).AnyTimes() storeMock.EXPECT().RemoveStore(gomock.Any()).Return(nil).AnyTimes() - storeMock.EXPECT().FlushAllEntries().Return(nil).AnyTimes() + storeMock.EXPECT().FlushCachesToStore().Return(nil).AnyTimes() storeMock.EXPECT().PruneCaches(gomock.Any()).Return(nil).AnyTimes() storeMock.EXPECT().RestoreCaches(gomock.Any()).Return(nil).AnyTimes() diff --git a/shared/modules/ibc_store_module.go b/shared/modules/ibc_store_module.go index 2a308a66b..fc0196804 100644 --- a/shared/modules/ibc_store_module.go +++ b/shared/modules/ibc_store_module.go @@ -25,7 +25,7 @@ type BulkStoreCacher interface { GetStore(name string) (ProvableStore, error) RemoveStore(name string) error GetAllStores() map[string]ProvableStore - FlushAllEntries() error + FlushCachesToStore() error PruneCaches(height uint64) error RestoreCaches(height uint64) error } @@ -44,7 +44,7 @@ type ProvableStore interface { Delete(key []byte) error GetCommitmentPrefix() coreTypes.CommitmentPrefix Root() ics23.CommitmentRoot - FlushEntries(kvstore.KVStore) error + FlushCache(kvstore.KVStore) error PruneCache(store kvstore.KVStore, height uint64) error RestoreCache(store kvstore.KVStore, height uint64) error } From accccfcb575fc2257b5d323ccd385c09726b279f Mon Sep 17 00:00:00 2001 From: Arash <23505281+adshmh@users.noreply.github.com> Date: Mon, 24 Jul 2023 20:14:55 -0400 Subject: [PATCH 07/25] [Utility] Feat: add client-side session cache (#888) ## Description Add a client-side cache for sessions and use it in the `servicer` command. ## Issue Fixes #791 ## Type of change Please mark the relevant option(s): - [x] New feature, functionality or library - [ ] Bug fix - [ ] Code health or cleanup - [ ] Major breaking change - [ ] Documentation - [ ] Other ## List of changes - A session cache in the client package - Use the new session cache in the servicer command ## Testing - [x] `make develop_test`; if any code changes were made - [x] `make test_e2e` on [k8s LocalNet](https://github.com/pokt-network/pocket/blob/main/build/localnet/README.md); if any code changes were made - [ ] `e2e-devnet-test` passes tests on [DevNet](https://pocketnetwork.notion.site/How-to-DevNet-ff1598f27efe44c09f34e2aa0051f0dd); if any code was changed - [ ] [Docker Compose LocalNet](https://github.com/pokt-network/pocket/blob/main/docs/development/README.md); if any major functionality was changed or introduced - [x] [k8s LocalNet](https://github.com/pokt-network/pocket/blob/main/build/localnet/README.md); if any infrastructure or configuration changes were made ## Required Checklist - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have added, or updated, [`godoc` format comments](https://go.dev/blog/godoc) on touched members (see: [tip.golang.org/doc/comment](https://tip.golang.org/doc/comment)) - [x] I have tested my changes using the available tooling ### If Applicable Checklist - [ ] I have updated the corresponding README(s); local and/or global - [x] I have added tests that prove my fix is effective or that my feature works - [ ] I have added, or updated, [mermaid.js](https://mermaid-js.github.io) diagrams in the corresponding README(s) - [ ] I have added, or updated, documentation and [mermaid.js](https://mermaid-js.github.io) diagrams in `shared/docs/*` if I updated `shared/*`README(s) --------- Co-authored-by: Daniel Olshansky --- app/client/cli/cache/session.go | 87 +++++++++++++++++++++++++++ app/client/cli/cache/session_test.go | 75 ++++++++++++++++++++++++ app/client/cli/servicer.go | 62 +++++++++++++++++++- app/client/cli/servicer_test.go | 88 ++++++++++++++++++++++++++++ 4 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 app/client/cli/cache/session.go create mode 100644 app/client/cli/cache/session_test.go create mode 100644 app/client/cli/servicer_test.go diff --git a/app/client/cli/cache/session.go b/app/client/cli/cache/session.go new file mode 100644 index 000000000..6412459fa --- /dev/null +++ b/app/client/cli/cache/session.go @@ -0,0 +1,87 @@ +package cache + +// TODO: add a TTL for cached sessions, since we know the sessions' length +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/pokt-network/pocket/persistence/kvstore" + "github.com/pokt-network/pocket/rpc" +) + +var errSessionNotFound = errors.New("session not found in cache") + +// SessionCache defines the set of methods used to interact with the client-side session cache +type SessionCache interface { + Get(appAddr, chain string) (*rpc.Session, error) + Set(session *rpc.Session) error + Stop() error +} + +// sessionCache stores and retrieves sessions for application+relaychain pairs +// +// It uses a key-value store as backing storage +type sessionCache struct { + // store is the local store for cached sessions + store kvstore.KVStore +} + +// NewSessionCache returns a session cache backed by a kvstore using the provided database path. +func NewSessionCache(databasePath string) (SessionCache, error) { + store, err := kvstore.NewKVStore(databasePath) + if err != nil { + return nil, fmt.Errorf("Error initializing key-value store using path %s: %w", databasePath, err) + } + + return &sessionCache{ + store: store, + }, nil +} + +// Get returns the cached session, if found, for an app+chain combination. +// The caller is responsible to verify that the returned session is valid for the current block height. +// Get is NOT safe to use concurrently +// DISCUSS: do we need concurrency here? +func (s *sessionCache) Get(appAddr, chain string) (*rpc.Session, error) { + key := sessionKey(appAddr, chain) + bz, err := s.store.Get(key) + if err != nil { + return nil, fmt.Errorf("error getting session from the store: %s %w", err.Error(), errSessionNotFound) + } + + var session rpc.Session + if err := json.Unmarshal(bz, &session); err != nil { + return nil, fmt.Errorf("error unmarshalling session from store: %w", err) + } + + return &session, nil +} + +// Set stores the provided session in the cache with the key being the app+chain combination. +// For each app+chain combination, a single session will be stored. Subsequent calls to Set will overwrite the entry for the provided app and chain. +// Set is NOT safe to use concurrently +func (s *sessionCache) Set(session *rpc.Session) error { + bz, err := json.Marshal(*session) + if err != nil { + return fmt.Errorf("error marshalling session for app: %s, chain: %s, session height: %d: %w", session.Application.Address, session.Chain, session.SessionHeight, err) + } + + key := sessionKey(session.Application.Address, session.Chain) + if err := s.store.Set(key, bz); err != nil { + return fmt.Errorf("error storing session for app: %s, chain: %s, session height: %d in the cache: %w", session.Application.Address, session.Chain, session.SessionHeight, err) + } + return nil +} + +// Stop call stop on the backing store. No calls should be made to Get or Set after calling Stop. +func (s *sessionCache) Stop() error { + return s.store.Stop() +} + +// sessionKey returns a key to get/set a session, based on application's address and the relay chain. +// +// The height is not used as part of the key, because for each app+chain combination only one session, i.e. the current one, is of interest. +func sessionKey(appAddr, chain string) []byte { + return []byte(fmt.Sprintf("%s-%s", appAddr, chain)) +} diff --git a/app/client/cli/cache/session_test.go b/app/client/cli/cache/session_test.go new file mode 100644 index 000000000..4b5afbaec --- /dev/null +++ b/app/client/cli/cache/session_test.go @@ -0,0 +1,75 @@ +package cache + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/pokt-network/pocket/rpc" +) + +func TestGet(t *testing.T) { + const ( + app1 = "app1Addr" + relaychainEth = "ETH-Goerli" + numSessionBlocks = 4 + sessionHeight = 8 + sessionNumber = 2 + ) + + session1 := &rpc.Session{ + Application: rpc.ProtocolActor{ + ActorType: rpc.Application, + Address: "app1Addr", + Chains: []string{relaychainEth}, + }, + Chain: relaychainEth, + NumSessionBlocks: numSessionBlocks, + SessionHeight: sessionHeight, + SessionNumber: sessionNumber, + } + + testCases := []struct { + name string + cacheContents []*rpc.Session + app string + chain string + expected *rpc.Session + expectedErr error + }{ + { + name: "Return cached session", + cacheContents: []*rpc.Session{session1}, + app: app1, + chain: relaychainEth, + expected: session1, + }, + { + name: "Error returned for session not found in cache", + app: "foo", + chain: relaychainEth, + expectedErr: errSessionNotFound, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dbPath, err := os.MkdirTemp("", "cacheStoragePath") + require.NoError(t, err) + defer os.RemoveAll(dbPath) + + cache, err := NewSessionCache(dbPath) + require.NoError(t, err) + + for _, s := range tc.cacheContents { + err := cache.Set(s) + require.NoError(t, err) + } + + got, err := cache.Get(tc.app, tc.chain) + require.ErrorIs(t, err, tc.expectedErr) + require.EqualValues(t, tc.expected, got) + }) + } +} diff --git a/app/client/cli/servicer.go b/app/client/cli/servicer.go index 5787d2d7e..0ed35dff4 100644 --- a/app/client/cli/servicer.go +++ b/app/client/cli/servicer.go @@ -4,19 +4,39 @@ import ( "context" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" "github.com/spf13/cobra" + "github.com/pokt-network/pocket/app/client/cli/cache" "github.com/pokt-network/pocket/app/client/cli/flags" + "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/rpc" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" ) +// IMPROVE: make this configurable +const sessionCacheDBPath = "/tmp" + +var ( + errNoSessionCache = errors.New("session cache not set up") + errSessionNotFoundInCache = errors.New("session not found in cache") + errNoMatchingSessionInCache = errors.New("no session matching the requested height found in cache") + + sessionCache cache.SessionCache +) + func init() { rootCmd.AddCommand(NewServicerCommand()) + + var err error + sessionCache, err = cache.NewSessionCache(sessionCacheDBPath) + if err != nil { + logger.Global.Warn().Err(err).Msg("failed to initialize session cache") + } } func NewServicerCommand() *cobra.Command { @@ -52,6 +72,12 @@ Will prompt the user for the *application* account passphrase`, Aliases: []string{}, Args: cobra.ExactArgs(4), RunE: func(cmd *cobra.Command, args []string) error { + defer func() { + if err := sessionCache.Stop(); err != nil { + logger.Global.Warn().Err(err).Msg("failed to stop session cache") + } + }() + applicationAddr := args[0] servicerAddr := args[1] chain := args[2] @@ -115,6 +141,25 @@ func validateServicer(session *rpc.Session, servicerAddress string) (*rpc.Protoc return nil, fmt.Errorf("Error getting servicer: address %s does not match any servicers in the session %d", servicerAddress, session.SessionNumber) } +// getSessionFromCache uses the client-side session cache to fetch a session for app+chain combination at the provided height, if one has already been retrieved and cached. +func getSessionFromCache(c cache.SessionCache, appAddress, chain string, height int64) (*rpc.Session, error) { + if c == nil { + return nil, errNoSessionCache + } + + session, err := c.Get(appAddress, chain) + if err != nil { + return nil, fmt.Errorf("%w: %s", errSessionNotFoundInCache, err.Error()) + } + + // verify the cached session matches the provided height + if height >= session.SessionHeight && height < session.SessionHeight+session.NumSessionBlocks { + return session, nil + } + + return nil, errNoMatchingSessionInCache +} + func getCurrentSession(ctx context.Context, appAddress, chain string) (*rpc.Session, error) { // CONSIDERATION: passing 0 as the height value to get the current session seems more optimal than this. currentHeight, err := getCurrentHeight(ctx) @@ -122,6 +167,11 @@ func getCurrentSession(ctx context.Context, appAddress, chain string) (*rpc.Sess return nil, fmt.Errorf("Error getting current session: %w", err) } + session, err := getSessionFromCache(sessionCache, appAddress, chain, currentHeight) + if err == nil { + return session, nil + } + req := rpc.SessionRequest{ AppAddress: appAddress, Chain: chain, @@ -148,7 +198,17 @@ func getCurrentSession(ctx context.Context, appAddress, chain string) (*rpc.Sess return nil, fmt.Errorf("Error getting current session: Unexpected response %v", resp) } - return resp.JSON200, nil + session = resp.JSON200 + if sessionCache == nil { + logger.Global.Warn().Msg("session cache not available: cannot cache the retrieved session") + return session, nil + } + + if err := sessionCache.Set(session); err != nil { + logger.Global.Warn().Err(err).Msg("failed to store session in cache") + } + + return session, nil } // REFACTOR: reuse this function in all the query commands diff --git a/app/client/cli/servicer_test.go b/app/client/cli/servicer_test.go new file mode 100644 index 000000000..cd84a1e87 --- /dev/null +++ b/app/client/cli/servicer_test.go @@ -0,0 +1,88 @@ +package cli + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/pokt-network/pocket/app/client/cli/cache" + "github.com/pokt-network/pocket/rpc" +) + +const ( + testRelaychainEth = "ETH-Goerli" + testSessionHeight = 8 + testCurrentHeight = 9 +) + +func TestGetSessionFromCache(t *testing.T) { + const app1Addr = "app1Addr" + + testCases := []struct { + name string + cachedSessions []*rpc.Session + expected *rpc.Session + expectedErr error + }{ + { + name: "cached session is returned", + cachedSessions: []*rpc.Session{testSession(app1Addr, testSessionHeight)}, + expected: testSession(app1Addr, testSessionHeight), + }, + { + name: "nil session cache returns an error", + expectedErr: errNoSessionCache, + }, + { + name: "session not found in cache", + cachedSessions: []*rpc.Session{testSession("foo", testSessionHeight)}, + expectedErr: errSessionNotFoundInCache, + }, + { + name: "cached session does not match the provided height", + cachedSessions: []*rpc.Session{testSession(app1Addr, 9999999)}, + expectedErr: errNoMatchingSessionInCache, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var c cache.SessionCache + // prepare cache with test session for this unit test + if len(tc.cachedSessions) > 0 { + dbPath, err := os.MkdirTemp("", "cliCacheStoragePath") + require.NoError(t, err) + defer os.RemoveAll(dbPath) + + c, err = cache.NewSessionCache(dbPath) + require.NoError(t, err) + + for _, s := range tc.cachedSessions { + err := c.Set(s) + require.NoError(t, err) + } + } + + got, err := getSessionFromCache(c, app1Addr, testRelaychainEth, testCurrentHeight) + require.ErrorIs(t, err, tc.expectedErr) + require.EqualValues(t, tc.expected, got) + }) + } +} + +func testSession(appAddr string, height int64) *rpc.Session { + const numSessionBlocks = 4 + + return &rpc.Session{ + Application: rpc.ProtocolActor{ + ActorType: rpc.Application, + Address: appAddr, + Chains: []string{testRelaychainEth}, + }, + Chain: testRelaychainEth, + NumSessionBlocks: numSessionBlocks, + SessionHeight: height, + SessionNumber: (height / numSessionBlocks), // assumes numSessionBlocks never changed + } +} From 3165b8d40856e1ee619eb6bdaeb41e4f35b2fb00 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Wed, 26 Jul 2023 09:39:04 +0100 Subject: [PATCH 08/25] [IBC] Clone `cosmos/ics23` protobuf definitions into IBC repo (#922) --- Makefile | 22 ++++ ibc/types/proofs.go | 197 +++++++++++++++++++++++++++++ ibc/types/proto/proofs.proto | 238 +++++++++++++++++++++++++++++++++++ 3 files changed, 457 insertions(+) create mode 100644 ibc/types/proofs.go create mode 100644 ibc/types/proto/proofs.proto diff --git a/Makefile b/Makefile index 003c72b11..0d59b64d4 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,15 @@ CWD ?= CURRENT_WORKING_DIRECTIONRY_NOT_SUPPLIED # `VERBOSE_TEST="" make test_persistence` is an easy way to run the same tests without verbose output VERBOSE_TEST ?= -v +# Detect OS using the $(shell uname -s) command +ifeq ($(shell uname -s),Darwin) + # Add macOS-specific commands here + SEDI = sed -i '' +else ifeq ($(shell uname -s),Linux) + # Add Linux-specific commands here + SEDI = sed -i +endif + .SILENT: .PHONY: list ## List all make targets @@ -309,6 +318,9 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the $(PROTOC_SHARED) -I=./p2p/types/proto --go_out=./p2p/types ./p2p/types/proto/*.proto # IBC + @if test ! -e "./ibc/types/proto/proofs.proto"; then \ + make download_ics23_proto; \ + fi $(PROTOC_SHARED) -I=./ibc/types/proto --go_out=./ibc/types ./ibc/types/proto/*.proto # echo "View generated proto files by running: make protogen_show" @@ -316,6 +328,16 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the # CONSIDERATION: Some proto files contain unused gRPC services so we may need to add the following # if/when we decide to include it: `grpc--go-grpc_opt=paths=source_relative --go-grpc_out=./output/path` +.PHONY: download_ics23_proto +download_ics23_proto: + echo "Downloading cosmos/ics23 proto definitions..."; \ + curl -s -o ./ibc/types/proto/proofs.proto https://raw.githubusercontent.com/cosmos/ics23/master/proto/cosmos/ics23/v1/proofs.proto; \ + $(SEDI) \ + -e '/^package/{N;d;}' \ + -e 's@github.com/.*"@github.com/pokt-network/pocket/ibc/types"@g' \ + ./ibc/types/proto/proofs.proto && \ + awk 'BEGIN { print "// ===== !! THIS IS CLONED FROM cosmos/ics23 !! =====\n" } { print }' ./ibc/types/proto/proofs.proto > tmpfile && mv tmpfile ./ibc/types/proto/proofs.proto; \ + .PHONY: protogen_docker_m1 ## TECHDEBT: Test, validate & update. protogen_docker_m1: docker_check diff --git a/ibc/types/proofs.go b/ibc/types/proofs.go new file mode 100644 index 000000000..cf25da5ab --- /dev/null +++ b/ibc/types/proofs.go @@ -0,0 +1,197 @@ +package types + +import ics23 "github.com/cosmos/ics23/go" + +// Copy of ics23.SmtSpec +// Ref: https://github.com/cosmos/ics23/blob/daa1760cb80f8607494ecf9e40482e66717a24e0/go/proof.go#L47 +var SmtSpec = &ProofSpec{ + LeafSpec: &LeafOp{ + Hash: HashOp_SHA256, + PrehashKey: HashOp_SHA256, + PrehashValue: HashOp_SHA256, + Length: LengthOp_NO_PREFIX, + Prefix: []byte{0}, + }, + InnerSpec: &InnerSpec{ + ChildOrder: []int32{0, 1}, + ChildSize: 32, + MinPrefixLength: 1, + MaxPrefixLength: 1, + EmptyChild: make([]byte, 32), + Hash: HashOp_SHA256, + }, + MaxDepth: 256, + PrehashKeyBeforeComparison: true, +} + +func (p *ProofSpec) ConvertToIcs23ProofSpec() *ics23.ProofSpec { + if p == nil { + return nil + } + ics := new(ics23.ProofSpec) + ics.LeafSpec = p.LeafSpec.convertToIcs23LeafOp() + ics.InnerSpec = p.InnerSpec.convertToIcs23InnerSpec() + ics.MaxDepth = p.MaxDepth + ics.MinDepth = p.MinDepth + ics.PrehashKeyBeforeComparison = p.PrehashKeyBeforeComparison + return ics +} + +func ConvertFromIcs23ProofSpec(p *ics23.ProofSpec) *ProofSpec { + if p == nil { + return nil + } + spc := new(ProofSpec) + spc.LeafSpec = convertFromIcs23LeafOp(p.LeafSpec) + spc.InnerSpec = convertFromIcs23InnerSpec(p.InnerSpec) + spc.MaxDepth = p.MaxDepth + spc.MinDepth = p.MinDepth + spc.PrehashKeyBeforeComparison = p.PrehashKeyBeforeComparison + return spc +} + +func (l *LeafOp) convertToIcs23LeafOp() *ics23.LeafOp { + if l == nil { + return nil + } + ics := new(ics23.LeafOp) + ics.Hash = l.Hash.convertToIcs23HashOp() + ics.PrehashKey = l.PrehashKey.convertToIcs23HashOp() + ics.PrehashValue = l.PrehashValue.convertToIcs23HashOp() + ics.Length = l.Length.convertToIcs23LenthOp() + ics.Prefix = l.Prefix + return ics +} + +func convertFromIcs23LeafOp(l *ics23.LeafOp) *LeafOp { + if l == nil { + return nil + } + op := new(LeafOp) + op.Hash = convertFromIcs23HashOp(l.Hash) + op.PrehashKey = convertFromIcs23HashOp(l.PrehashKey) + op.PrehashValue = convertFromIcs23HashOp(l.PrehashValue) + op.Length = convertFromIcs23LengthOp(l.Length) + op.Prefix = l.Prefix + return op +} + +func (i *InnerSpec) convertToIcs23InnerSpec() *ics23.InnerSpec { + if i == nil { + return nil + } + ics := new(ics23.InnerSpec) + ics.ChildOrder = i.ChildOrder + ics.ChildSize = i.ChildSize + ics.MinPrefixLength = i.MinPrefixLength + ics.MaxPrefixLength = i.MaxPrefixLength + ics.EmptyChild = i.EmptyChild + ics.Hash = i.Hash.convertToIcs23HashOp() + return ics +} + +func convertFromIcs23InnerSpec(i *ics23.InnerSpec) *InnerSpec { + if i == nil { + return nil + } + spec := new(InnerSpec) + spec.ChildOrder = i.ChildOrder + spec.ChildSize = i.ChildSize + spec.MinPrefixLength = i.MinPrefixLength + spec.MaxPrefixLength = i.MaxPrefixLength + spec.EmptyChild = i.EmptyChild + spec.Hash = convertFromIcs23HashOp(i.Hash) + return spec +} + +func (h HashOp) convertToIcs23HashOp() ics23.HashOp { + switch h { + case HashOp_NO_HASH: + return ics23.HashOp_NO_HASH + case HashOp_SHA256: + return ics23.HashOp_SHA256 + case HashOp_SHA512: + return ics23.HashOp_SHA512 + case HashOp_KECCAK: + return ics23.HashOp_KECCAK + case HashOp_RIPEMD160: + return ics23.HashOp_RIPEMD160 + case HashOp_BITCOIN: + return ics23.HashOp_BITCOIN + case HashOp_SHA512_256: + return ics23.HashOp_SHA512_256 + default: + panic("unknown hash op") + } +} + +func convertFromIcs23HashOp(h ics23.HashOp) HashOp { + switch h { + case ics23.HashOp_NO_HASH: + return HashOp_NO_HASH + case ics23.HashOp_SHA256: + return HashOp_SHA256 + case ics23.HashOp_SHA512: + return HashOp_SHA512 + case ics23.HashOp_KECCAK: + return HashOp_KECCAK + case ics23.HashOp_RIPEMD160: + return HashOp_RIPEMD160 + case ics23.HashOp_BITCOIN: + return HashOp_BITCOIN + case ics23.HashOp_SHA512_256: + return HashOp_SHA512_256 + default: + panic("unknown hash op") + } +} + +func (l LengthOp) convertToIcs23LenthOp() ics23.LengthOp { + switch l { + case LengthOp_NO_PREFIX: + return ics23.LengthOp_NO_PREFIX + case LengthOp_VAR_PROTO: + return ics23.LengthOp_VAR_PROTO + case LengthOp_VAR_RLP: + return ics23.LengthOp_VAR_RLP + case LengthOp_FIXED32_BIG: + return ics23.LengthOp_FIXED32_BIG + case LengthOp_FIXED32_LITTLE: + return ics23.LengthOp_FIXED32_LITTLE + case LengthOp_FIXED64_BIG: + return ics23.LengthOp_FIXED64_BIG + case LengthOp_FIXED64_LITTLE: + return ics23.LengthOp_FIXED64_LITTLE + case LengthOp_REQUIRE_32_BYTES: + return ics23.LengthOp_REQUIRE_32_BYTES + case LengthOp_REQUIRE_64_BYTES: + return ics23.LengthOp_REQUIRE_64_BYTES + default: + panic("unknown length op") + } +} + +func convertFromIcs23LengthOp(l ics23.LengthOp) LengthOp { + switch l { + case ics23.LengthOp_NO_PREFIX: + return LengthOp_NO_PREFIX + case ics23.LengthOp_VAR_PROTO: + return LengthOp_VAR_PROTO + case ics23.LengthOp_VAR_RLP: + return LengthOp_VAR_RLP + case ics23.LengthOp_FIXED32_BIG: + return LengthOp_FIXED32_BIG + case ics23.LengthOp_FIXED32_LITTLE: + return LengthOp_FIXED32_LITTLE + case ics23.LengthOp_FIXED64_BIG: + return LengthOp_FIXED64_BIG + case ics23.LengthOp_FIXED64_LITTLE: + return LengthOp_FIXED64_LITTLE + case ics23.LengthOp_REQUIRE_32_BYTES: + return LengthOp_REQUIRE_32_BYTES + case ics23.LengthOp_REQUIRE_64_BYTES: + return LengthOp_REQUIRE_64_BYTES + default: + panic("unknown length op") + } +} diff --git a/ibc/types/proto/proofs.proto b/ibc/types/proto/proofs.proto new file mode 100644 index 000000000..e3659215e --- /dev/null +++ b/ibc/types/proto/proofs.proto @@ -0,0 +1,238 @@ +// ===== !! THIS IS CLONED FROM cosmos/ics23 !! ===== + +syntax = "proto3"; + +option go_package = "github.com/pokt-network/pocket/ibc/types"; + +enum HashOp { + // NO_HASH is the default if no data passed. Note this is an illegal argument some places. + NO_HASH = 0; + SHA256 = 1; + SHA512 = 2; + KECCAK = 3; + RIPEMD160 = 4; + BITCOIN = 5; // ripemd160(sha256(x)) + SHA512_256 = 6; +} + +/** +LengthOp defines how to process the key and value of the LeafOp +to include length information. After encoding the length with the given +algorithm, the length will be prepended to the key and value bytes. +(Each one with it's own encoded length) +*/ +enum LengthOp { + // NO_PREFIX don't include any length info + NO_PREFIX = 0; + // VAR_PROTO uses protobuf (and go-amino) varint encoding of the length + VAR_PROTO = 1; + // VAR_RLP uses rlp int encoding of the length + VAR_RLP = 2; + // FIXED32_BIG uses big-endian encoding of the length as a 32 bit integer + FIXED32_BIG = 3; + // FIXED32_LITTLE uses little-endian encoding of the length as a 32 bit integer + FIXED32_LITTLE = 4; + // FIXED64_BIG uses big-endian encoding of the length as a 64 bit integer + FIXED64_BIG = 5; + // FIXED64_LITTLE uses little-endian encoding of the length as a 64 bit integer + FIXED64_LITTLE = 6; + // REQUIRE_32_BYTES is like NONE, but will fail if the input is not exactly 32 bytes (sha256 output) + REQUIRE_32_BYTES = 7; + // REQUIRE_64_BYTES is like NONE, but will fail if the input is not exactly 64 bytes (sha512 output) + REQUIRE_64_BYTES = 8; +} + +/** +ExistenceProof takes a key and a value and a set of steps to perform on it. +The result of peforming all these steps will provide a "root hash", which can +be compared to the value in a header. + +Since it is computationally infeasible to produce a hash collission for any of the used +cryptographic hash functions, if someone can provide a series of operations to transform +a given key and value into a root hash that matches some trusted root, these key and values +must be in the referenced merkle tree. + +The only possible issue is maliablity in LeafOp, such as providing extra prefix data, +which should be controlled by a spec. Eg. with lengthOp as NONE, + prefix = FOO, key = BAR, value = CHOICE +and + prefix = F, key = OOBAR, value = CHOICE +would produce the same value. + +With LengthOp this is tricker but not impossible. Which is why the "leafPrefixEqual" field +in the ProofSpec is valuable to prevent this mutability. And why all trees should +length-prefix the data before hashing it. +*/ +message ExistenceProof { + bytes key = 1; + bytes value = 2; + LeafOp leaf = 3; + repeated InnerOp path = 4; +} + +/* +NonExistenceProof takes a proof of two neighbors, one left of the desired key, +one right of the desired key. If both proofs are valid AND they are neighbors, +then there is no valid proof for the given key. +*/ +message NonExistenceProof { + bytes key = 1; // TODO: remove this as unnecessary??? we prove a range + ExistenceProof left = 2; + ExistenceProof right = 3; +} + +/* +CommitmentProof is either an ExistenceProof or a NonExistenceProof, or a Batch of such messages +*/ +message CommitmentProof { + oneof proof { + ExistenceProof exist = 1; + NonExistenceProof nonexist = 2; + BatchProof batch = 3; + CompressedBatchProof compressed = 4; + } +} + +/** +LeafOp represents the raw key-value data we wish to prove, and +must be flexible to represent the internal transformation from +the original key-value pairs into the basis hash, for many existing +merkle trees. + +key and value are passed in. So that the signature of this operation is: + leafOp(key, value) -> output + +To process this, first prehash the keys and values if needed (ANY means no hash in this case): + hkey = prehashKey(key) + hvalue = prehashValue(value) + +Then combine the bytes, and hash it + output = hash(prefix || length(hkey) || hkey || length(hvalue) || hvalue) +*/ +message LeafOp { + HashOp hash = 1; + HashOp prehash_key = 2; + HashOp prehash_value = 3; + LengthOp length = 4; + // prefix is a fixed bytes that may optionally be included at the beginning to differentiate + // a leaf node from an inner node. + bytes prefix = 5; +} + +/** +InnerOp represents a merkle-proof step that is not a leaf. +It represents concatenating two children and hashing them to provide the next result. + +The result of the previous step is passed in, so the signature of this op is: + innerOp(child) -> output + +The result of applying InnerOp should be: + output = op.hash(op.prefix || child || op.suffix) + + where the || operator is concatenation of binary data, +and child is the result of hashing all the tree below this step. + +Any special data, like prepending child with the length, or prepending the entire operation with +some value to differentiate from leaf nodes, should be included in prefix and suffix. +If either of prefix or suffix is empty, we just treat it as an empty string +*/ +message InnerOp { + HashOp hash = 1; + bytes prefix = 2; + bytes suffix = 3; +} + +/** +ProofSpec defines what the expected parameters are for a given proof type. +This can be stored in the client and used to validate any incoming proofs. + + verify(ProofSpec, Proof) -> Proof | Error + +As demonstrated in tests, if we don't fix the algorithm used to calculate the +LeafHash for a given tree, there are many possible key-value pairs that can +generate a given hash (by interpretting the preimage differently). +We need this for proper security, requires client knows a priori what +tree format server uses. But not in code, rather a configuration object. +*/ +message ProofSpec { + // any field in the ExistenceProof must be the same as in this spec. + // except Prefix, which is just the first bytes of prefix (spec can be longer) + LeafOp leaf_spec = 1; + InnerSpec inner_spec = 2; + // max_depth (if > 0) is the maximum number of InnerOps allowed (mainly for fixed-depth tries) + int32 max_depth = 3; + // min_depth (if > 0) is the minimum number of InnerOps allowed (mainly for fixed-depth tries) + int32 min_depth = 4; + // prehash_key_before_comparison is a flag that indicates whether to use the + // prehash_key specified by LeafOp to compare lexical ordering of keys for + // non-existence proofs. + bool prehash_key_before_comparison = 5; +} + +/* +InnerSpec contains all store-specific structure info to determine if two proofs from a +given store are neighbors. + +This enables: + + isLeftMost(spec: InnerSpec, op: InnerOp) + isRightMost(spec: InnerSpec, op: InnerOp) + isLeftNeighbor(spec: InnerSpec, left: InnerOp, right: InnerOp) +*/ +message InnerSpec { + // Child order is the ordering of the children node, must count from 0 + // iavl tree is [0, 1] (left then right) + // merk is [0, 2, 1] (left, right, here) + repeated int32 child_order = 1; + int32 child_size = 2; + int32 min_prefix_length = 3; + int32 max_prefix_length = 4; + // empty child is the prehash image that is used when one child is nil (eg. 20 bytes of 0) + bytes empty_child = 5; + // hash is the algorithm that must be used for each InnerOp + HashOp hash = 6; +} + +/* +BatchProof is a group of multiple proof types than can be compressed +*/ +message BatchProof { + repeated BatchEntry entries = 1; +} + +// Use BatchEntry not CommitmentProof, to avoid recursion +message BatchEntry { + oneof proof { + ExistenceProof exist = 1; + NonExistenceProof nonexist = 2; + } +} + +/****** all items here are compressed forms *******/ + +message CompressedBatchProof { + repeated CompressedBatchEntry entries = 1; + repeated InnerOp lookup_inners = 2; +} + +// Use BatchEntry not CommitmentProof, to avoid recursion +message CompressedBatchEntry { + oneof proof { + CompressedExistenceProof exist = 1; + CompressedNonExistenceProof nonexist = 2; + } +} + +message CompressedExistenceProof { + bytes key = 1; + bytes value = 2; + LeafOp leaf = 3; + // these are indexes into the lookup_inners table in CompressedBatchProof + repeated int32 path = 4; +} + +message CompressedNonExistenceProof { + bytes key = 1; // TODO: remove this as unnecessary??? we prove a range + CompressedExistenceProof left = 2; + CompressedExistenceProof right = 3; +} From 990321e2dff8ad72c5629bafe47bab49166a457a Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 26 Jul 2023 11:03:39 +0200 Subject: [PATCH 09/25] [CLI] Consistent config/flag parsing & common helpers (#891) Co-authored-by: harry <53987565+h5law@users.noreply.github.com> Co-authored-by: Daniel Olshansky --- app/client/cli/debug.go | 155 +++++------------------------- app/client/cli/helpers/common.go | 54 +++++++++-- app/client/cli/helpers/context.go | 14 +++ app/client/cli/helpers/setup.go | 20 +++- runtime/manager.go | 1 + 5 files changed, 106 insertions(+), 138 deletions(-) diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index 5ff6d521b..99d5b83de 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -1,8 +1,6 @@ package cli import ( - "errors" - "fmt" "os" "github.com/manifoldco/promptui" @@ -11,10 +9,7 @@ import ( "github.com/pokt-network/pocket/app/client/cli/helpers" "github.com/pokt-network/pocket/logger" - "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" - typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/shared/messaging" - "github.com/pokt-network/pocket/shared/modules" ) // TECHDEBT: Lowercase variables / constants that do not need to be exported. @@ -28,26 +23,20 @@ const ( PromptSendBlockRequest string = "BlockRequest (broadcast)" ) -var ( - items = []string{ - PromptPrintNodeState, - PromptTriggerNextView, - PromptTogglePacemakerMode, - PromptResetToGenesis, - PromptShowLatestBlockInStore, - PromptSendMetadataRequest, - PromptSendBlockRequest, - } -) +var items = []string{ + PromptPrintNodeState, + PromptTriggerNextView, + PromptTogglePacemakerMode, + PromptResetToGenesis, + PromptShowLatestBlockInStore, + PromptSendMetadataRequest, + PromptSendBlockRequest, +} func init() { dbgUI := newDebugUICommand() dbgUI.AddCommand(newDebugUISubCommands()...) rootCmd.AddCommand(dbgUI) - - dbg := newDebugCommand() - dbg.AddCommand(debugCommands()...) - rootCmd.AddCommand(dbg) } // newDebugUISubCommands builds out the list of debug subcommands by matching the @@ -60,7 +49,7 @@ func newDebugUISubCommands() []*cobra.Command { commands[idx] = &cobra.Command{ Use: promptItem, PersistentPreRunE: helpers.P2PDependenciesPreRunE, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { handleSelect(cmd, cmd.Use) }, ValidArgs: items, @@ -81,56 +70,7 @@ func newDebugUICommand() *cobra.Command { } } -// newDebugCommand returns the cobra CLI for the Debug command. -func newDebugCommand() *cobra.Command { - return &cobra.Command{ - Use: "Debug", - Aliases: []string{"d"}, - Short: "Debug utility for rapid development", - Args: cobra.MaximumNArgs(1), - PersistentPreRunE: helpers.P2PDependenciesPreRunE, - } -} - -func debugCommands() []*cobra.Command { - cmds := []*cobra.Command{ - { - Use: "TriggerView", - Aliases: []string{"next", "trigger", "view"}, - Short: "Trigger the next view in consensus", - Long: "Sends a message to all visible nodes on the network to start the next view (height/step/round) in consensus", - Args: cobra.ExactArgs(0), - RunE: func(cmd *cobra.Command, args []string) error { - m := &messaging.DebugMessage{ - Action: messaging.DebugMessageAction_DEBUG_CONSENSUS_TRIGGER_NEXT_VIEW, - Type: messaging.DebugMessageRoutingType_DEBUG_MESSAGE_TYPE_BROADCAST, - Message: nil, - } - broadcastDebugMessage(cmd, m) - return nil - }, - }, - { - Use: "TogglePacemakerMode", - Short: "Toggle the pacemaker", - Long: "Toggle the consensus pacemaker either on or off so the chain progresses on its own or loses liveness", - Aliases: []string{"togglePaceMaker"}, - Args: cobra.ExactArgs(0), - RunE: func(cmd *cobra.Command, args []string) error { - m := &messaging.DebugMessage{ - Action: messaging.DebugMessageAction_DEBUG_CONSENSUS_TOGGLE_PACE_MAKER_MODE, - Type: messaging.DebugMessageRoutingType_DEBUG_MESSAGE_TYPE_BROADCAST, - Message: nil, - } - broadcastDebugMessage(cmd, m) - return nil - }, - }, - } - return cmds -} - -func runDebug(cmd *cobra.Command, args []string) (err error) { +func runDebug(cmd *cobra.Command, _ []string) (err error) { for { if selection, err := promptGetInput(); err == nil { handleSelect(cmd, selection) @@ -218,32 +158,20 @@ func handleSelect(cmd *cobra.Command, selection string) { } } -// Broadcast to the entire validator set +// Broadcast to the entire network. func broadcastDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { anyProto, err := anypb.New(debugMsg) if err != nil { logger.Global.Fatal().Err(err).Msg("Failed to create Any proto") } - // TODO(olshansky): Once we implement the cleanup layer in RainTree, we'll be able to use - // broadcast. The reason it cannot be done right now is because this client is not in the - // address book of the actual validator nodes, so `validator1` never receives the message. - // p2pMod.Broadcast(anyProto) - - pstore, err := fetchPeerstore(cmd) + bus, err := helpers.GetBusFromCmd(cmd) if err != nil { - logger.Global.Fatal().Err(err).Msg("Unable to retrieve the pstore") + logger.Global.Fatal().Err(err).Msg("Failed to retrieve bus from command") } - for _, val := range pstore.GetPeerList() { - addr := val.GetAddress() - if err != nil { - logger.Global.Fatal().Err(err).Msg("Failed to convert validator address into pocketCrypto.Address") - } - if err := helpers.P2PMod.Send(addr, anyProto); err != nil { - logger.Global.Error().Err(err).Msg("Failed to send debug message") - } + if err := bus.GetP2PModule().Broadcast(anyProto); err != nil { + logger.Global.Error().Err(err).Msg("Failed to broadcast debug message") } - } // Send to just a single (i.e. first) validator in the set @@ -253,62 +181,29 @@ func sendDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { logger.Global.Error().Err(err).Msg("Failed to create Any proto") } - pstore, err := fetchPeerstore(cmd) + pstore, err := helpers.FetchPeerstore(cmd) if err != nil { logger.Global.Fatal().Err(err).Msg("Unable to retrieve the pstore") } - var validatorAddress []byte if pstore.Size() == 0 { logger.Global.Fatal().Msg("No validators found") } // if the message needs to be broadcast, it'll be handled by the business logic of the message handler - validatorAddress = pstore.GetPeerList()[0].GetAddress() + // + // TODO(#936): The statement above is false. Using `#Send()` will only + // be unicast with no opportunity for further propagation. + firstStakedActorAddress := pstore.GetPeerList()[0].GetAddress() if err != nil { logger.Global.Fatal().Err(err).Msg("Failed to convert validator address into pocketCrypto.Address") } - if err := helpers.P2PMod.Send(validatorAddress, anyProto); err != nil { - logger.Global.Error().Err(err).Msg("Failed to send debug message") - } -} - -// fetchPeerstore retrieves the providers from the CLI context and uses them to retrieve the address book for the current height -func fetchPeerstore(cmd *cobra.Command) (typesP2P.Peerstore, error) { - bus, ok := helpers.GetValueFromCLIContext[modules.Bus](cmd, helpers.BusCLICtxKey) - if !ok || bus == nil { - return nil, errors.New("retrieving bus from CLI context") - } - // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider - // is retrievable as a proper submodule - pstoreProvider, err := bus.GetModulesRegistry().GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) - if err != nil { - return nil, errors.New("retrieving peerstore provider") - } - currentHeightProvider := bus.GetCurrentHeightProvider() - - height := currentHeightProvider.CurrentHeight() - pstore, err := pstoreProvider.(peerstore_provider.PeerstoreProvider).GetStakedPeerstoreAtHeight(height) - if err != nil { - return nil, fmt.Errorf("retrieving peerstore at height %d", height) - } - // Inform the client's main P2P that a the blockchain is at a new height so it can, if needed, update its view of the validator set - err = sendConsensusNewHeightEventToP2PModule(height, bus) + bus, err := helpers.GetBusFromCmd(cmd) if err != nil { - return nil, errors.New("sending consensus new height event") + logger.Global.Fatal().Err(err).Msg("Failed to retrieve bus from command") } - return pstore, nil -} - -// sendConsensusNewHeightEventToP2PModule mimicks the consensus module sending a ConsensusNewHeightEvent to the p2p module -// This is necessary because the debug client is not a validator and has no consensus module but it has to update the peerstore -// depending on the changes in the validator set. -// TODO(#613): Make the debug client mimic a full node. -func sendConsensusNewHeightEventToP2PModule(height uint64, bus modules.Bus) error { - newHeightEvent, err := messaging.PackMessage(&messaging.ConsensusNewHeightEvent{Height: height}) - if err != nil { - logger.Global.Fatal().Err(err).Msg("Failed to pack consensus new height event") + if err := bus.GetP2PModule().Send(firstStakedActorAddress, anyProto); err != nil { + logger.Global.Error().Err(err).Msg("Failed to send debug message") } - return bus.GetP2PModule().HandleEvent(newHeightEvent.Content) } diff --git a/app/client/cli/helpers/common.go b/app/client/cli/helpers/common.go index b9f6d547b..647d4241d 100644 --- a/app/client/cli/helpers/common.go +++ b/app/client/cli/helpers/common.go @@ -1,14 +1,56 @@ package helpers import ( + "errors" + "fmt" + + "github.com/spf13/cobra" + + "github.com/pokt-network/pocket/logger" + "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" + "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/runtime" + "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" ) -var ( - // TECHDEBT: Accept reading this from `Datadir` and/or as a flag. - genesisPath = runtime.GetEnv("GENESIS_PATH", "build/config/genesis.json") +// TECHDEBT: Accept reading this from `Datadir` and/or as a flag. +var genesisPath = runtime.GetEnv("GENESIS_PATH", "build/config/genesis.json") - // P2PMod is initialized in order to broadcast a message to the local network - P2PMod modules.P2PModule -) +// FetchPeerstore retrieves the providers from the CLI context and uses them to retrieve the address book for the current height +func FetchPeerstore(cmd *cobra.Command) (types.Peerstore, error) { + bus, err := GetBusFromCmd(cmd) + if err != nil { + return nil, err + } + // TECHDEBT(#811): use `bus.GetPeerstoreProvider()` after peerstore provider + // is retrievable as a proper submodule + pstoreProvider, err := bus.GetModulesRegistry().GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) + if err != nil { + return nil, errors.New("retrieving peerstore provider") + } + currentHeightProvider := bus.GetCurrentHeightProvider() + height := currentHeightProvider.CurrentHeight() + pstore, err := pstoreProvider.(peerstore_provider.PeerstoreProvider).GetStakedPeerstoreAtHeight(height) + if err != nil { + return nil, fmt.Errorf("retrieving peerstore at height %d", height) + } + // Inform the client's main P2P that a the blockchain is at a new height so it can, if needed, update its view of the validator set + if err := sendConsensusNewHeightEventToP2PModule(height, bus); err != nil { + return nil, errors.New("sending consensus new height event") + } + return pstore, nil +} + +// sendConsensusNewHeightEventToP2PModule mimicks the consensus module sending a ConsensusNewHeightEvent to the p2p module +// This is necessary because the debug client is not a validator and has no consensus module but it has to update the peerstore +// depending on the changes in the validator set. +// TODO(#613): Make the debug client mimic a full node. +// TECHDEBT: This may no longer be required (https://github.com/pokt-network/pocket/pull/891/files#r1262710098) +func sendConsensusNewHeightEventToP2PModule(height uint64, bus modules.Bus) error { + newHeightEvent, err := messaging.PackMessage(&messaging.ConsensusNewHeightEvent{Height: height}) + if err != nil { + logger.Global.Fatal().Err(err).Msg("Failed to pack consensus new height event") + } + return bus.GetP2PModule().HandleEvent(newHeightEvent.Content) +} diff --git a/app/client/cli/helpers/context.go b/app/client/cli/helpers/context.go index f9f3f4549..f1494c5ac 100644 --- a/app/client/cli/helpers/context.go +++ b/app/client/cli/helpers/context.go @@ -2,12 +2,17 @@ package helpers import ( "context" + "fmt" "github.com/spf13/cobra" + + "github.com/pokt-network/pocket/shared/modules" ) const BusCLICtxKey cliContextKey = "bus" +var ErrCxtFromBus = fmt.Errorf("could not get context from bus") + // NOTE: this is required by the linter, otherwise a simple string constant would have been enough type cliContextKey string @@ -19,3 +24,12 @@ func GetValueFromCLIContext[T any](cmd *cobra.Command, key cliContextKey) (T, bo value, ok := cmd.Context().Value(key).(T) return value, ok } + +func GetBusFromCmd(cmd *cobra.Command) (modules.Bus, error) { + bus, ok := GetValueFromCLIContext[modules.Bus](cmd, BusCLICtxKey) + if !ok { + return nil, ErrCxtFromBus + } + + return bus, nil +} diff --git a/app/client/cli/helpers/setup.go b/app/client/cli/helpers/setup.go index 34605bd1e..3766b6ae9 100644 --- a/app/client/cli/helpers/setup.go +++ b/app/client/cli/helpers/setup.go @@ -16,11 +16,27 @@ import ( "github.com/pokt-network/pocket/shared/modules" ) +// debugPrivKey is used in the generation of a runtime config to provide a private key to the P2P and Consensus modules +// this is not a private key used for sending transactions, but is used for the purposes of broadcasting messages etc. +// this must be done as the CLI does not take a node configuration file and still requires a Private Key for modules +const debugPrivKey = "09fc8ee114e678e665d09179acb9a30060f680df44ba06b51434ee47940a8613be19b2b886e743eb1ff7880968d6ce1a46350315e569243e747a227ee8faec3d" + // P2PDependenciesPreRunE initializes peerstore & current height providers, and a // p2p module which consumes them. Everything is registered to the bus. func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { // TECHDEBT: this was being used for backwards compatibility with LocalNet and need to re-evaluate if its still necessary flags.ConfigPath = runtime.GetEnv("CONFIG_PATH", "build/config/config.validator1.json") + configs.ParseConfig(flags.ConfigPath) + + // set final `remote_cli_url` value; order of precedence: flag > env var > config > default + flags.RemoteCLIURL = viper.GetString("remote_cli_url") + + // By this time, the config path should be set. + // This is only being called for viper related side effects + // TECHDEBT(#907): refactor and improve how viper is used to parse configs throughout the codebase + _ = configs.ParseConfig(flags.ConfigPath) + // set final `remote_cli_url` value; order of precedence: flag > env var > config > default + flags.RemoteCLIURL = viper.GetString("remote_cli_url") // By this time, the config path should be set. // This is only being called for viper related side effects @@ -32,7 +48,7 @@ func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { runtimeMgr := runtime.NewManagerFromFiles( flags.ConfigPath, genesisPath, runtime.WithClientDebugMode(), - runtime.WithRandomPK(), + runtime.WithPK(debugPrivKey), ) bus := runtimeMgr.GetBus() @@ -79,7 +95,7 @@ func setupAndStartP2PModule(rm runtime.Manager) { } var ok bool - P2PMod, ok = mod.(modules.P2PModule) + P2PMod, ok := mod.(modules.P2PModule) if !ok { logger.Global.Fatal().Msgf("unexpected P2P module type: %T", mod) } diff --git a/runtime/manager.go b/runtime/manager.go index 151f2c198..da6209957 100644 --- a/runtime/manager.go +++ b/runtime/manager.go @@ -104,6 +104,7 @@ func WithRandomPK() func(*Manager) { return WithPK(privateKey.String()) } +// TECHDEBT(#750): separate consensus and P2P (identity vs communication) keys. func WithPK(pk string) func(*Manager) { return func(b *Manager) { if b.config.Consensus == nil { From 21d70241814d33426953f0355fd4d7764fe4d79c Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Wed, 26 Jul 2023 10:30:32 +0100 Subject: [PATCH 10/25] [IBC] Change Events to not have a Height field and use uint64 in queries (#931) --- ibc/events/event_manager.go | 9 +++++---- ibc/store/provable_store.go | 4 ++-- ibc/store/provable_store_test.go | 2 +- persistence/gov.go | 5 +++++ persistence/ibc.go | 6 +++--- persistence/test/ibc_test.go | 20 ++++++++------------ persistence/types/ibc.go | 6 +++--- shared/core/types/ibc_events.go | 5 +++++ shared/core/types/proto/ibc_events.proto | 16 +++++----------- shared/modules/ibc_event_module.go | 6 +++--- shared/modules/persistence_module.go | 3 ++- 11 files changed, 42 insertions(+), 40 deletions(-) create mode 100644 shared/core/types/ibc_events.go diff --git a/ibc/events/event_manager.go b/ibc/events/event_manager.go index 19e48cb95..0b233e139 100644 --- a/ibc/events/event_manager.go +++ b/ibc/events/event_manager.go @@ -1,7 +1,7 @@ package events import ( - coreTypes "github.com/pokt-network/pocket/shared/core/types" + core_types "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" ) @@ -42,14 +42,15 @@ func (*EventManager) Create(bus modules.Bus, options ...modules.EventLoggerOptio func (e *EventManager) GetModuleName() string { return modules.EventLoggerModuleName } -func (e *EventManager) EmitEvent(event *coreTypes.IBCEvent) error { +func (e *EventManager) EmitEvent(event *core_types.IBCEvent) error { wCtx := e.GetBus().GetPersistenceModule().NewWriteContext() defer wCtx.Release() return wCtx.SetIBCEvent(event) } -func (e *EventManager) QueryEvents(topic string, height uint64) ([]*coreTypes.IBCEvent, error) { - rCtx, err := e.GetBus().GetPersistenceModule().NewReadContext(int64(height)) +func (e *EventManager) QueryEvents(topic string, height uint64) ([]*core_types.IBCEvent, error) { + currHeight := e.GetBus().GetConsensusModule().CurrentHeight() + rCtx, err := e.GetBus().GetPersistenceModule().NewReadContext(int64(currHeight)) if err != nil { return nil, err } diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index df4612725..0ab6bd6c7 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -67,8 +67,8 @@ func newProvableStore(bus modules.Bus, prefix coreTypes.CommitmentPrefix, privat // keys are automatically prefixed with the CommitmentPrefix if not present func (p *provableStore) Get(key []byte) ([]byte, error) { prefixed := applyPrefix(p.prefix, key) - currHeight := int64(p.bus.GetConsensusModule().CurrentHeight()) - rCtx, err := p.bus.GetPersistenceModule().NewReadContext(currHeight) + currHeight := p.bus.GetConsensusModule().CurrentHeight() + rCtx, err := p.bus.GetPersistenceModule().NewReadContext(int64(currHeight)) if err != nil { return nil, err } diff --git a/ibc/store/provable_store_test.go b/ibc/store/provable_store_test.go index 174d62827..a62160a62 100644 --- a/ibc/store/provable_store_test.go +++ b/ibc/store/provable_store_test.go @@ -432,7 +432,7 @@ func newPersistenceMock(t *testing.T, EXPECT(). GetIBCStoreEntry(gomock.Any(), gomock.Any()). DoAndReturn( - func(key []byte, _ int64) ([]byte, error) { + func(key []byte, _ uint64) ([]byte, error) { value, ok := dbMap[hex.EncodeToString(key)] if !ok { return nil, coreTypes.ErrIBCKeyDoesNotExist(string(key)) diff --git a/persistence/gov.go b/persistence/gov.go index 5ddec2884..73694ac6e 100644 --- a/persistence/gov.go +++ b/persistence/gov.go @@ -17,6 +17,11 @@ func (p *PostgresContext) GetVersionAtHeight(height int64) (string, error) { return "", nil } +// TODO(#882): Implement this function +func (p *PostgresContext) GetRevisionNumber(height int64) uint64 { + return 1 +} + // TODO: Implement this function func (p *PostgresContext) GetSupportedChains(height int64) ([]string, error) { // This is a placeholder function for the RPC endpoint "v1/query/supportedchains" diff --git a/persistence/ibc.go b/persistence/ibc.go index fc9affea4..0544d7197 100644 --- a/persistence/ibc.go +++ b/persistence/ibc.go @@ -14,14 +14,14 @@ import ( // SetIBCStoreEntry sets the key value pair in the IBC store postgres table at the current height func (p *PostgresContext) SetIBCStoreEntry(key, value []byte) error { ctx, tx := p.getCtxAndTx() - if _, err := tx.Exec(ctx, pTypes.InsertIBCStoreEntryQuery(p.Height, key, value)); err != nil { + if _, err := tx.Exec(ctx, pTypes.InsertIBCStoreEntryQuery(uint64(p.Height), key, value)); err != nil { return err } return nil } // GetIBCStoreEntry returns the stored value for the key at the height provided from the IBC store table -func (p *PostgresContext) GetIBCStoreEntry(key []byte, height int64) ([]byte, error) { +func (p *PostgresContext) GetIBCStoreEntry(key []byte, height uint64) ([]byte, error) { ctx, tx := p.getCtxAndTx() row := tx.QueryRow(ctx, pTypes.GetIBCStoreEntryQuery(height, key)) var valueHex string @@ -50,7 +50,7 @@ func (p *PostgresContext) SetIBCEvent(event *coreTypes.IBCEvent) error { return err } eventHex := hex.EncodeToString(eventBz) - if _, err := tx.Exec(ctx, pTypes.InsertIBCEventQuery(p.Height, typeStr, eventHex)); err != nil { + if _, err := tx.Exec(ctx, pTypes.InsertIBCEventQuery(uint64(p.Height), typeStr, eventHex)); err != nil { return err } return nil diff --git a/persistence/test/ibc_test.go b/persistence/test/ibc_test.go index 2fcf86f4e..b2885c155 100644 --- a/persistence/test/ibc_test.go +++ b/persistence/test/ibc_test.go @@ -75,7 +75,7 @@ func TestIBC_GetIBCStoreEntry(t *testing.T) { testCases := []struct { name string - height int64 + height uint64 key []byte expectedValue []byte expectedErr error @@ -133,13 +133,12 @@ var ( baseAttributeValue = []byte("testValue") ) -func TestIBCSetEvent(t *testing.T) { +func TestIBC_SetIBCEvent(t *testing.T) { // Setup database db := NewTestPostgresContext(t, 1) // Add a single event at height 1 event := new(coreTypes.IBCEvent) event.Topic = "test" - event.Height = 1 event.Attributes = append(event.Attributes, &coreTypes.Attribute{ Key: baseAttributeKey, Value: baseAttributeValue, @@ -216,7 +215,6 @@ func TestIBCSetEvent(t *testing.T) { db.Height = int64(tc.height) event := new(coreTypes.IBCEvent) event.Topic = tc.topic - event.Height = tc.height for _, attr := range tc.attributes { event.Attributes = append(event.Attributes, &coreTypes.Attribute{ Key: attr.key, @@ -233,7 +231,7 @@ func TestIBCSetEvent(t *testing.T) { } } -func TestGetIBCEvent(t *testing.T) { +func TestIBC_GetIBCEvent(t *testing.T) { // Setup database db := NewTestPostgresContext(t, 1) // Add events "testKey0", "testKey1", "testKey2", "testKey3" @@ -242,10 +240,6 @@ func TestGetIBCEvent(t *testing.T) { for i := 0; i < 4; i++ { event := new(coreTypes.IBCEvent) event.Topic = "test" - event.Height = uint64(i + 1) - if i == 3 { - event.Height = uint64(i) // add a second event at height 3 - } s := strconv.Itoa(i) event.Attributes = append(event.Attributes, &coreTypes.Attribute{ Key: []byte("testKey" + s), @@ -253,8 +247,11 @@ func TestGetIBCEvent(t *testing.T) { }) events = append(events, event) } - for _, event := range events { - db.Height = int64(event.Height) + for i, event := range events { + db.Height = int64(i + 1) + if i == 3 { // add 2 events at height 3 + db.Height = int64(i) + } require.NoError(t, db.SetIBCEvent(event)) } @@ -301,7 +298,6 @@ func TestGetIBCEvent(t *testing.T) { require.NoError(t, err) require.Len(t, got, tc.expectedLength) for i, index := range tc.eventsIndexes { - require.Equal(t, events[index].Height, got[i].Height) require.Equal(t, events[index].Topic, got[i].Topic) require.Equal(t, events[index].Attributes[0].Key, got[i].Attributes[0].Key) require.Equal(t, events[index].Attributes[0].Value, got[i].Attributes[0].Value) diff --git a/persistence/types/ibc.go b/persistence/types/ibc.go index a783bcf82..7b0d01ee5 100644 --- a/persistence/types/ibc.go +++ b/persistence/types/ibc.go @@ -23,7 +23,7 @@ const ( ) // InsertIBCStoreEntryQuery returns the query to insert a key/value pair into the ibc_entries table -func InsertIBCStoreEntryQuery(height int64, key, value []byte) string { +func InsertIBCStoreEntryQuery(height uint64, key, value []byte) string { return fmt.Sprintf( `INSERT INTO %s(height, key, value) VALUES(%d, '%s', '%s')`, IBCStoreTableName, @@ -34,7 +34,7 @@ func InsertIBCStoreEntryQuery(height int64, key, value []byte) string { } // InsertIBCEventQuery returns the query to insert an event into the ibc_events table -func InsertIBCEventQuery(height int64, topic, eventHex string) string { +func InsertIBCEventQuery(height uint64, topic, eventHex string) string { return fmt.Sprintf( `INSERT INTO %s(height, topic, event) VALUES(%d, '%s', '%s')`, IBCEventLogTableName, @@ -45,7 +45,7 @@ func InsertIBCEventQuery(height int64, topic, eventHex string) string { } // GetIBCStoreEntryQuery returns the latest value for the key at the height provided or at the last updated height -func GetIBCStoreEntryQuery(height int64, key []byte) string { +func GetIBCStoreEntryQuery(height uint64, key []byte) string { return fmt.Sprintf( `SELECT value FROM %s WHERE height <= %d AND key = '%s' ORDER BY height DESC LIMIT 1`, IBCStoreTableName, diff --git a/shared/core/types/ibc_events.go b/shared/core/types/ibc_events.go new file mode 100644 index 000000000..3c3c7580a --- /dev/null +++ b/shared/core/types/ibc_events.go @@ -0,0 +1,5 @@ +package types + +func NewAttribute(key, value []byte) *Attribute { + return &Attribute{Key: key, Value: value} +} diff --git a/shared/core/types/proto/ibc_events.proto b/shared/core/types/proto/ibc_events.proto index 15041214a..0ad47774b 100644 --- a/shared/core/types/proto/ibc_events.proto +++ b/shared/core/types/proto/ibc_events.proto @@ -4,18 +4,12 @@ package core; option go_package = "github.com/pokt-network/pocket/shared/core/types"; -// Attribute represents a key-value pair in an IBC event +message IBCEvent { + string topic = 1; + repeated Attribute attributes = 2; +} + message Attribute { bytes key = 1; bytes value = 2; } - -// IBCEvent are used after a series of insertions/updates/deletions to the IBC store -// they capture the type of changes made, such as creating a new light client, or -// opening a connection. They also capture the height at which the change was made -// and the different key-value pairs that were modified in the attributes field. -message IBCEvent { - string topic = 1; - uint64 height = 2; - repeated Attribute attributes = 3; -} diff --git a/shared/modules/ibc_event_module.go b/shared/modules/ibc_event_module.go index 56a80f07f..d3f628770 100644 --- a/shared/modules/ibc_event_module.go +++ b/shared/modules/ibc_event_module.go @@ -3,7 +3,7 @@ package modules //go:generate mockgen -destination=./mocks/ibc_event_module_mock.go github.com/pokt-network/pocket/shared/modules EventLogger import ( - coreTypes "github.com/pokt-network/pocket/shared/core/types" + core_types "github.com/pokt-network/pocket/shared/core/types" ) const EventLoggerModuleName = "event_logger" @@ -16,6 +16,6 @@ type EventLogger interface { Submodule eventLoggerFactory - EmitEvent(event *coreTypes.IBCEvent) error - QueryEvents(topic string, height uint64) ([]*coreTypes.IBCEvent, error) + EmitEvent(event *core_types.IBCEvent) error + QueryEvents(topic string, height uint64) ([]*core_types.IBCEvent, error) } diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index b510d835b..6ee724d63 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -161,6 +161,7 @@ type PersistenceReadContext interface { // Version queries GetVersionAtHeight(height int64) (string, error) // TODO: Implement this + GetRevisionNumber(height int64) uint64 // TODO(#882): Implement this // Supported Chains Queries GetSupportedChains(height int64) ([]string, error) // TODO: Implement this @@ -245,7 +246,7 @@ type PersistenceReadContext interface { // IBC Queries // GetIBCStoreEntry returns the value of the key at the given height from the ibc_entries table - GetIBCStoreEntry(key []byte, height int64) ([]byte, error) + GetIBCStoreEntry(key []byte, height uint64) ([]byte, error) // GetIBCEvent returns the matching IBC events for any topic at the height provied GetIBCEvents(height uint64, topic string) ([]*coreTypes.IBCEvent, error) } From c67fa141817f71df829c469b34e2a2efc69c7ca3 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Wed, 26 Jul 2023 10:58:19 +0100 Subject: [PATCH 11/25] [IBC] Add ICS-02 Client Interfaces (#932) --- Makefile | 2 + .../light_clients/types/proto/pocket.proto | 59 +++++ ibc/client/types/proto/wasm.proto | 35 +++ shared/modules/bus_module.go | 1 + shared/modules/ibc_client_module.go | 212 ++++++++++++++++++ shared/modules/ibc_host_module.go | 177 +-------------- shared/modules/ibc_module.go | 153 ++++++++++++- 7 files changed, 462 insertions(+), 177 deletions(-) create mode 100644 ibc/client/light_clients/types/proto/pocket.proto create mode 100644 ibc/client/types/proto/wasm.proto create mode 100644 shared/modules/ibc_client_module.go diff --git a/Makefile b/Makefile index 0d59b64d4..9ec1c636a 100644 --- a/Makefile +++ b/Makefile @@ -322,6 +322,8 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the make download_ics23_proto; \ fi $(PROTOC_SHARED) -I=./ibc/types/proto --go_out=./ibc/types ./ibc/types/proto/*.proto + $(PROTOC_SHARED) -I=./ibc/client/types/proto --go_out=./ibc/client/types ./ibc/client/types/proto/*.proto + $(PROTOC_SHARED) -I=./ibc/client/types/proto -I=./ibc/client/light_clients/types/proto -I=./shared/core/types/proto -I=./ibc/types/proto --go_out=./ibc/client/light_clients/types ./ibc/client/light_clients/types/proto/*.proto # echo "View generated proto files by running: make protogen_show" diff --git a/ibc/client/light_clients/types/proto/pocket.proto b/ibc/client/light_clients/types/proto/pocket.proto new file mode 100644 index 000000000..aa828e265 --- /dev/null +++ b/ibc/client/light_clients/types/proto/pocket.proto @@ -0,0 +1,59 @@ +syntax = "proto3"; + +package core; + +option go_package = "github.com/pokt-network/pocket/ibc/client/light_client/types"; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; +import "proofs.proto"; +import "wasm.proto"; +import "block.proto"; + +// PocketConsensusState defines the ibc client consensus state for Pocket +message PocketConsensusState { + google.protobuf.Timestamp timestamp = 1; // unix nano timestamp of the block + string state_hash = 2; // hex encoded root state tree hash + map state_tree_hashes = 3; // map of state tree hashes; map[TreeName]hex(TreeRootHash) + string next_val_set_hash = 4; // hex encoded sha3_256 hash of the next validator set +} + +// PocketClientState defines the ibc client state for Pocket +message PocketClientState { + string network_id = 1; // network identifier string + Fraction trust_level = 2; // fraction of the validator set that is required to sign off on new blocks + google.protobuf.Duration trusting_period = 3; // the duration of the period since the LastestTimestamp where the state can be upgraded + google.protobuf.Duration unbonding_period = 4; // the duration of the staking unbonding period + google.protobuf.Duration max_clock_drift = 5; // the max duration a new header's time can be in the future + Height latest_height = 6; // the latest height the client was updated to + uint64 frozen_height = 7; // the height at which the client was frozen due to a misbehaviour + ProofSpec proof_spec = 8; // ics23 proof spec used in verifying proofs + // RESEARCH: Figure out exactly what this is for in tendermint, why it is needed and if we need it also + // repeated string upgrade_path = 9; // the upgrade path for the new client state +} + +// Fraction defines a positive rational number +message Fraction { + uint64 numerator = 1; + uint64 denominator = 2; +} + +// PocketHeader defines the ibc client header for the Pocket network +message PocketHeader { + BlockHeader block_header = 1; // pocket consensus block header + ValidatorSet validator_set = 2; // new validator set for the updating client + // the consensus state at trusted_height must be within the unbonding_period to correctly verify the new header + Height trusted_height = 3; // height of the ConsensusState stored used to verify the new header + // trusted_validators must hash to the ConsensusState.NextValSetHash as this is the last trusted validator set + // hashed using SHA3Hash(validatorSetBytes) in shared/crypto/sha3.go + ValidatorSet trusted_validators = 4; // already stored validator set used to verify the update +} + +// PocketMisbehaviour defines the ibc client misbehaviour for the Pocket network +// +// The two conflicting headers are submitted as evidence to verify the Pocket +// network has misbehaved. +message PocketMisbehaviour { + PocketHeader header_1 = 1; // the first header + PocketHeader header_2 = 2; // the second header +} diff --git a/ibc/client/types/proto/wasm.proto b/ibc/client/types/proto/wasm.proto new file mode 100644 index 000000000..ebe6ce5e1 --- /dev/null +++ b/ibc/client/types/proto/wasm.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package core; + +option go_package = "github.com/pokt-network/pocket/ibc/client/types"; + +// ClientState for a Wasm light client +message ClientState { + bytes data = 1; // opaque data passed to the wasm client + bytes wasm_checksum = 2; // checksum of the wasm client code + Height recent_height = 3; // latest height of the client +} + +// ConsensusState for a Wasm light client +message ConsensusState { + bytes data = 1; // opaque data passed to the wasm client + uint64 timestamp = 2; // unix nano timestamp of the block +} + +// Header for a Wasm light client +message Header { + bytes data = 1; // opaque data passed to the wasm client + Height height = 2; // height of the header +} + +// Misbehaviour for a Wasm light client +message Misbehaviour { + bytes data = 1; // opaque data passed to the wasm client +} + +// Height represents the height of a client +message Height { + uint64 revision_number = 1; + uint64 revision_height = 2; +} diff --git a/shared/modules/bus_module.go b/shared/modules/bus_module.go index 3981f9f8c..14b329643 100644 --- a/shared/modules/bus_module.go +++ b/shared/modules/bus_module.go @@ -45,4 +45,5 @@ type Bus interface { GetIBCHost() IBCHostSubmodule GetBulkStoreCacher() BulkStoreCacher GetEventLogger() EventLogger + GetClientManager() ClientManager } diff --git a/shared/modules/ibc_client_module.go b/shared/modules/ibc_client_module.go new file mode 100644 index 000000000..d4dc1d716 --- /dev/null +++ b/shared/modules/ibc_client_module.go @@ -0,0 +1,212 @@ +package modules + +//go:generate mockgen -destination=./mocks/ibc_client_module_mock.go github.com/pokt-network/pocket/shared/modules ClientManager + +import ( + "google.golang.org/protobuf/proto" +) + +type ClientStatus string + +const ( + ClientManagerModuleName = "client_manager" + + // Client Status types + ActiveStatus ClientStatus = "active" + ExpiredStatus ClientStatus = "expired" + FrozenStatus ClientStatus = "frozen" + UnauthorizedStatus ClientStatus = "unauthorized" + UnknownStatus ClientStatus = "unknown" +) + +type ClientManagerOption func(ClientManager) + +type clientManagerFactory = FactoryWithOptions[ClientManager, ClientManagerOption] + +// ClientManager is the interface that defines the methods needed to interact with an +// IBC light client it manages the different lifecycle methods for the different clients +// https://github.com/cosmos/ibc/tree/main/spec/core/ics-002-client-semantics +type ClientManager interface { + Submodule + clientManagerFactory + + // === Client Lifecycle Management === + + // CreateClient creates a new client with the given client state and initial consensus state + // and initialises its unique identifier in the IBC store + CreateClient(ClientState, ConsensusState) (string, error) + + // UpdateClient updates an existing client with the given ClientMessage, given that + // the ClientMessage can be verified using the existing ClientState and ConsensusState + UpdateClient(identifier string, clientMessage ClientMessage) error + + // UpgradeClient upgrades an existing client with the given identifier using the + // ClientState and ConsenusState provided. It can only do so if the new client + // was committed to by the old client at the specified upgrade height + UpgradeClient( + identifier string, + clientState ClientState, consensusState ConsensusState, + proofUpgradeClient, proofUpgradeConsState []byte, + ) error + + // === Client Queries === + + // GetConsensusState returns the ConsensusState at the given height for the given client + GetConsensusState(identifier string, height Height) (ConsensusState, error) + + // GetClientState returns the ClientState for the given client + GetClientState(identifier string) (ClientState, error) + + // GetHostConsensusState returns the ConsensusState at the given height for the host chain + GetHostConsensusState(height Height) (ConsensusState, error) + + // GetHostClientState returns the ClientState at the provided height for the host chain + GetHostClientState(height Height) (ClientState, error) + + // GetCurrentHeight returns the current IBC client height of the network + GetCurrentHeight() (Height, error) + + // VerifyHostClientState verifies the client state for a client running on a + // counterparty chain is valid, checking against the current host client state + VerifyHostClientState(ClientState) error +} + +// ClientState is an interface that defines the methods required by a clients +// implementation of their own client state object +// +// ClientState is an opaque data structure defined by a client type. It may keep +// arbitrary internal state to track verified roots and past misbehaviours. +type ClientState interface { + proto.Message + + GetData() []byte + GetWasmChecksum() []byte + ClientType() string + GetLatestHeight() Height + Validate() error + + // Status returns the status of the client. Only Active clients are allowed + // to process packets. + Status(clientStore ProvableStore) ClientStatus + + // GetTimestampAtHeight must return the timestamp for the consensus state + // associated with the provided height. + GetTimestampAtHeight(clientStore ProvableStore, height Height) (uint64, error) + + // Initialise is called upon client creation, it allows the client to perform + // validation on the initial consensus state and set the client state, + // consensus state and any client-specific metadata necessary for correct + // light client operation in the provided client store. + Initialise(clientStore ProvableStore, consensusState ConsensusState) error + + // VerifyMembership is a generic proof verification method which verifies a + // proof of the existence of a value at a given CommitmentPath at the + // specified height. The path is expected to be the full CommitmentPath + VerifyMembership( + clientStore ProvableStore, + height Height, + delayTimePeriod, delayBlockPeriod uint64, + proof, path, value []byte, + ) error + + // VerifyNonMembership is a generic proof verification method which verifies + // the absence of a given CommitmentPath at a specified height. The path is + // expected to be the full CommitmentPath + VerifyNonMembership( + clientStore ProvableStore, + height Height, + delayTimePeriod, delayBlockPeriod uint64, + proof, path []byte, + ) error + + // VerifyClientMessage verifies a ClientMessage. A ClientMessage could be a + // Header, Misbehaviour, or batch update. It must handle each type of + // ClientMessage appropriately. Calls to CheckForMisbehaviour, UpdateState, + // and UpdateStateOnMisbehaviour will assume that the content of the + // ClientMessage has been verified and can be trusted. An error should be + // returned if the ClientMessage fails to verify. + VerifyClientMessage(clientStore ProvableStore, clientMsg ClientMessage) error + + // Checks for evidence of a misbehaviour in Header or Misbehaviour type. + // It assumes the ClientMessage has already been verified. + CheckForMisbehaviour(clientStore ProvableStore, clientMsg ClientMessage) bool + + // UpdateStateOnMisbehaviour should perform appropriate state changes on a + // client state given that misbehaviour has been detected and verified + UpdateStateOnMisbehaviour(clientStore ProvableStore, clientMsg ClientMessage) error + + // UpdateState updates and stores as necessary any associated information + // for an IBC client, such as the ClientState and corresponding ConsensusState. + // Upon successful update, a consensus height is returned. + // It assumes the ClientMessage has already been verified. + UpdateState(clientStore ProvableStore, clientMsg ClientMessage) (Height, error) + + // Upgrade functions + // NOTE: proof heights are not included as upgrade to a new revision is expected to pass only on the last + // height committed by the current revision. Clients are responsible for ensuring that the planned last + // height of the current revision is somehow encoded in the proof verification process. + // This is to ensure that no premature upgrades occur, since upgrade plans committed to by the counterparty + // may be cancelled or modified before the last planned height. + // If the upgrade is verified, the upgraded client and consensus states must be set in the client store. + VerifyUpgradeAndUpdateState( + clientStore ProvableStore, + newClient ClientState, + newConsState ConsensusState, + proofUpgradeClient, + proofUpgradeConsState []byte, + ) error +} + +// ConsensusState is an interface that defines the methods required by a clients +// implementation of their own consensus state object +// +// ConsensusState is an opaque data structure defined by a client type, used by the +// validity predicate to verify new commits & state roots. Likely the structure will +// contain the last commit produced by the consensus process, including signatures +// and validator set metadata. +type ConsensusState interface { + proto.Message + + GetData() []byte + ClientType() string + GetTimestamp() uint64 + ValidateBasic() error +} + +// ClientMessage is an interface that defines the methods required by a clients +// implementation of their own client message object +// +// A ClientMessage is an opaque data structure defined by a client type which +// provides information to update the client. ClientMessages can be submitted +// to an associated client to add new ConsensusState(s) and/or update the +// ClientState. They likely contain a height, a proof, a commitment root, and +// possibly updates to the validity predicate. +type ClientMessage interface { + proto.Message + + GetData() []byte + ClientType() string + ValidateBasic() error +} + +// Height is an interface that defines the methods required by a clients +// implementation of their own height object +// +// Heights usually have two components: revision number and revision height. +type Height interface { + IsZero() bool + LT(Height) bool + LTE(Height) bool + EQ(Height) bool + GT(Height) bool + GTE(Height) bool + Increment() Height + Decrement() Height + GetRevisionNumber() uint64 + GetRevisionHeight() uint64 + ToString() string // must define a determinstic `String()` method not the generated protobuf method +} + +func (s ClientStatus) String() string { + return string(s) +} diff --git a/shared/modules/ibc_host_module.go b/shared/modules/ibc_host_module.go index 9a94d5b0f..c009337cf 100644 --- a/shared/modules/ibc_host_module.go +++ b/shared/modules/ibc_host_module.go @@ -4,7 +4,7 @@ import ( "github.com/pokt-network/pocket/runtime/configs" ) -//go:generate mockgen -destination=./mocks/ibc_host_module_mock.go github.com/pokt-network/pocket/shared/modules IBCHostSubmodule,IBCHandler +//go:generate mockgen -destination=./mocks/ibc_host_module_mock.go github.com/pokt-network/pocket/shared/modules IBCHostSubmodule const IBCHostSubmoduleName = "ibc_host" @@ -22,184 +22,9 @@ type IBCHostSubmodule interface { Submodule ibcHostFactory - // IBC related operations - IBCHandler - // GetTimestamp returns the current unix timestamp for the host machine GetTimestamp() uint64 // GetProvableStore returns an instance of a ProvableStore managed by the StoreManager GetProvableStore(name string) (ProvableStore, error) } - -// INCOMPLETE: Split into multiple interfaces per ICS component and embed in the handler -// IBCHandler is the interface through which the different IBC sub-modules can be interacted with -// https://github.com/cosmos/ibc/tree/main/spec/core/ics-025-handler-interface -type IBCHandler interface { - // === Client Lifecycle Management === - // https://github.com/cosmos/ibc/tree/main/spec/core/ics-002-client-semantics - - // CreateClient creates a new client with the given client state and initial consensus state - // and initialises its unique identifier in the IBC store - // CreateClient(clientState clientState, consensusState consensusState) error - - // UpdateClient updates an existing client with the given ClientMessage, given that - // the ClientMessage can be verified using the existing ClientState and ConsensusState - // UpdateClient(identifier Identifier, clientMessage ClientMessage) error - - // QueryConsensusState returns the ConsensusState at the given height for the given client - // QueryConsensusState(identifier Identifier, height Height) ConsensusState - - // QueryClientState returns the ClientState for the given client - // QueryClientState(identifier Identifier) ClientState - - // SubmitMisbehaviour submits evidence for a misbehaviour to the client, possibly invalidating - // previously valid state roots and thus preventing future updates - // SubmitMisbehaviour(identifier Identifier, clientMessage ClientMessage) error - - // === Connection Lifecycle Management === - // https://github.com/cosmos/ibc/tree/main/spec/core/ics-003-connection-semantics - - // ConnOpenInit attempts to initialise a connection to a given counterparty chain (executed on source chain) - /** - ConnOpenInit( - counterpartyPrefix CommitmentPrefix, - clientIdentifier, counterpartyClientIdentifier Identifier, - version: string, // Optional: If version is included, the handshake must fail if the version is not the same - delayPeriodTime, delayPeriodBlocks uint64, - ) error - **/ - - // ConnOpenTry relays a notice of a connection attempt to a counterpaty chain (executed on destination chain) - /** - ConnOpenTry( - counterpartyPrefix CommitmentPrefix, - counterpartyConnectionIdentifier, counterpartyClientIdentifier, clientIdentifier Identifier, - clientState ClientState, - counterpartyVersions []string, - delayPeriodTime, delayPeriodBlocks uint64, - proofInit, proofClient, proofConsensus ics23.CommitmentProof, - proofHeight, consensusHeight Height, - hostConsensusStateProof bytes, - ) error - **/ - - // ConnOpenAck relays the acceptance of a connection open attempt from counterparty chain (executed on source chain) - /** - ConnOpenAck( - identifier, counterpartyIdentifier Identifier, - clientState ClientState, - version string, - proofTry, proofClient, proofConsensus ics23.CommitmentProof, - proofHeight, consensusHeight Height, - hostConsensusStateProof bytes, - ) error - **/ - - // ConnOpenConfirm confirms opening of a connection to the counterparty chain after which the - // connection is open to both chains (executed on destination chain) - // ConnOpenConfirm(identifier Identifier, proofAck ics23.CommitmentProof, proofHeight Height) error - - // QueryConnection returns the ConnectionEnd for the given connection identifier - // QueryConnection(identifier Identifier) (ConnectionEnd, error) - - // QueryClientConnections returns the list of connection identifiers associated with a given client - // QueryClientConnections(clientIdentifier Identifier) ([]Identifier, error) - - // === Channel Lifecycle Management === - // https://github.com/cosmos/ibc/tree/main/spec/core/ics-004-channel-and-packet-semantics - - // ChanOpenInit initialises a channel opening handshake with a counterparty chain (executed on source chain) - /** - ChanOpenInit( - order ChannelOrder, - connectionHops []Identifier, - portIdentifier, counterpartyPortIdentifier Identifier, - version string, - ) (channelIdentifier Identifier, channelCapability CapabilityKey, err Error) - **/ - - // ChanOpenTry attempts to accept the channel opening handshake from a counterparty chain (executed on destination chain) - /** - ChanOpenTry( - order ChannelOrder, - connectionHops []Identifier, - portIdentifier, counterpartyPortIdentifier, counterpartyChannelIdentifier Identifier, - version, counterpartyVersion string, - proofInit ics23.CommitmentProof, - ) (channelIdentifier Identifier, channelCapability CapabilityKey, err Error) - **/ - - // ChanOpenAck relays acceptance of a channel opening handshake from a counterparty chain (executed on source chain) - /** - ChanOpenAck( - portIdentifier, channelIdentifier, counterpartyChannelIdentifier Identifier, - counterpartyVersion string, - proofTry ics23.CommitmentProof, - proofHeight Height, - ) error - **/ - - // ChanOpenConfirm acknowledges the acknowledgment of the channel opening hanshake on the counterparty - // chain after which the channel opening handshake is complete (executed on destination chain) - // ChanOpenConfirm(portIdentifier, channelIdentifier Identifier, proofAck ics23.CommitmentProof, proofHeight Height) error - - // ChanCloseInit is called to close the ChannelEnd with the given identifier on the host machine - // ChanCloseInit(portIdentifier, channelIdentifier Identifier) error - - // ChanCloseConfirm is called to close the ChannelEnd on the counterparty chain as the other end is closed - // ChanCloseConfirm(portIdentifier, channelIdentifier Identifier, proofInit ics23.CommitmentProof, proofHeight Height) error - - // === Packet Relaying === - - // SendPacket is called to send an IBC packet on the channel with the given identifier - /** - SendPacket( - capability CapabilityKey, - sourcePort Identifier, - sourceChannel Identifier, - timeoutHeight Height, - timeoutTimestamp uint64, - data []byte, - ) (sequence uint64, err error) - **/ - - // RecvPacket is called in order to receive an IBC packet on the corresponding channel end - // on the counterpaty chain - // RecvPacket(packet OpaquePacket, proof ics23.CommitmentProof, proofHeight Height, relayer string) (Packet, error) - - // AcknowledgePacket is called to acknowledge the receipt of an IBC packet to the corresponding chain - /** - AcknowledgePacket( - packet OpaquePacket, - acknowledgement []byte, - proof ics23.CommitmentProof, - proofHeight Height, - relayer string, - ) (Packet, error) - **/ - - // TimeoutPacket is called to timeout an IBC packet on the corresponding channel end after the - // timeout height or timeout timestamp has passed and the packet has not been committed - /** - TimeoutPacket( - packet OpaquePacket, - proof ics23.CommitmentProof, - proofHeight Height, - nextSequenceRecv *uint64, - relayer string, - ) (Packet, error) - **/ - - // TimeoutOnClose is called to prove to the counterparty chain that the channel end has been - // closed and that the packet sent over this channel will not be received - /** - TimeoutOnClose( - packet OpaquePacket, - proof, proofClosed ics23.CommitmentProof, - proofHeight Height, - nextSequenceRecv *uint64, - relayer string, - ) (Packet, error) - **/ -} diff --git a/shared/modules/ibc_module.go b/shared/modules/ibc_module.go index 20ed2fa65..ab66fb1b1 100644 --- a/shared/modules/ibc_module.go +++ b/shared/modules/ibc_module.go @@ -2,7 +2,7 @@ package modules import "google.golang.org/protobuf/types/known/anypb" -//go:generate mockgen -destination=./mocks/ibc_module_mock.go github.com/pokt-network/pocket/shared/modules IBCModule +//go:generate mockgen -destination=./mocks/ibc_module_mock.go github.com/pokt-network/pocket/shared/modules IBCModule,IBCHandler const IBCModuleName = "ibc" @@ -11,3 +11,154 @@ type IBCModule interface { HandleEvent(*anypb.Any) error } + +// INCOMPLETE: Split into multiple interfaces per ICS component and embed in the handler +// IBCHandler is the interface through which the different IBC sub-modules can be interacted with +// https://github.com/cosmos/ibc/tree/main/spec/core/ics-025-handler-interface +type IBCHandler interface { + // === Connection Lifecycle Management === + // https://github.com/cosmos/ibc/tree/main/spec/core/ics-003-connection-semantics + + // ConnOpenInit attempts to initialise a connection to a given counterparty chain (executed on source chain) + /** + ConnOpenInit( + counterpartyPrefix CommitmentPrefix, + clientIdentifier, counterpartyClientIdentifier Identifier, + version: string, // Optional: If version is included, the handshake must fail if the version is not the same + delayPeriodTime, delayPeriodBlocks uint64, + ) error + **/ + + // ConnOpenTry relays a notice of a connection attempt to a counterpaty chain (executed on destination chain) + /** + ConnOpenTry( + counterpartyPrefix CommitmentPrefix, + counterpartyConnectionIdentifier, counterpartyClientIdentifier, clientIdentifier Identifier, + clientState ClientState, + counterpartyVersions []string, + delayPeriodTime, delayPeriodBlocks uint64, + proofInit, proofClient, proofConsensus ics23.CommitmentProof, + proofHeight, consensusHeight Height, + hostConsensusStateProof bytes, + ) error + **/ + + // ConnOpenAck relays the acceptance of a connection open attempt from counterparty chain (executed on source chain) + /** + ConnOpenAck( + identifier, counterpartyIdentifier Identifier, + clientState ClientState, + version string, + proofTry, proofClient, proofConsensus ics23.CommitmentProof, + proofHeight, consensusHeight Height, + hostConsensusStateProof bytes, + ) error + **/ + + // ConnOpenConfirm confirms opening of a connection to the counterparty chain after which the + // connection is open to both chains (executed on destination chain) + // ConnOpenConfirm(identifier Identifier, proofAck ics23.CommitmentProof, proofHeight Height) error + + // QueryConnection returns the ConnectionEnd for the given connection identifier + // QueryConnection(identifier Identifier) (ConnectionEnd, error) + + // QueryClientConnections returns the list of connection identifiers associated with a given client + // QueryClientConnections(clientIdentifier Identifier) ([]Identifier, error) + + // === Channel Lifecycle Management === + // https://github.com/cosmos/ibc/tree/main/spec/core/ics-004-channel-and-packet-semantics + + // ChanOpenInit initialises a channel opening handshake with a counterparty chain (executed on source chain) + /** + ChanOpenInit( + order ChannelOrder, + connectionHops []Identifier, + portIdentifier, counterpartyPortIdentifier Identifier, + version string, + ) (channelIdentifier Identifier, channelCapability CapabilityKey, err Error) + **/ + + // ChanOpenTry attempts to accept the channel opening handshake from a counterparty chain (executed on destination chain) + /** + ChanOpenTry( + order ChannelOrder, + connectionHops []Identifier, + portIdentifier, counterpartyPortIdentifier, counterpartyChannelIdentifier Identifier, + version, counterpartyVersion string, + proofInit ics23.CommitmentProof, + ) (channelIdentifier Identifier, channelCapability CapabilityKey, err Error) + **/ + + // ChanOpenAck relays acceptance of a channel opening handshake from a counterparty chain (executed on source chain) + /** + ChanOpenAck( + portIdentifier, channelIdentifier, counterpartyChannelIdentifier Identifier, + counterpartyVersion string, + proofTry ics23.CommitmentProof, + proofHeight Height, + ) error + **/ + + // ChanOpenConfirm acknowledges the acknowledgment of the channel opening hanshake on the counterparty + // chain after which the channel opening handshake is complete (executed on destination chain) + // ChanOpenConfirm(portIdentifier, channelIdentifier Identifier, proofAck ics23.CommitmentProof, proofHeight Height) error + + // ChanCloseInit is called to close the ChannelEnd with the given identifier on the host machine + // ChanCloseInit(portIdentifier, channelIdentifier Identifier) error + + // ChanCloseConfirm is called to close the ChannelEnd on the counterparty chain as the other end is closed + // ChanCloseConfirm(portIdentifier, channelIdentifier Identifier, proofInit ics23.CommitmentProof, proofHeight Height) error + + // === Packet Relaying === + + // SendPacket is called to send an IBC packet on the channel with the given identifier + /** + SendPacket( + capability CapabilityKey, + sourcePort Identifier, + sourceChannel Identifier, + timeoutHeight Height, + timeoutTimestamp uint64, + data []byte, + ) (sequence uint64, err error) + **/ + + // RecvPacket is called in order to receive an IBC packet on the corresponding channel end + // on the counterpaty chain + // RecvPacket(packet OpaquePacket, proof ics23.CommitmentProof, proofHeight Height, relayer string) (Packet, error) + + // AcknowledgePacket is called to acknowledge the receipt of an IBC packet to the corresponding chain + /** + AcknowledgePacket( + packet OpaquePacket, + acknowledgement []byte, + proof ics23.CommitmentProof, + proofHeight Height, + relayer string, + ) (Packet, error) + **/ + + // TimeoutPacket is called to timeout an IBC packet on the corresponding channel end after the + // timeout height or timeout timestamp has passed and the packet has not been committed + /** + TimeoutPacket( + packet OpaquePacket, + proof ics23.CommitmentProof, + proofHeight Height, + nextSequenceRecv *uint64, + relayer string, + ) (Packet, error) + **/ + + // TimeoutOnClose is called to prove to the counterparty chain that the channel end has been + // closed and that the packet sent over this channel will not be received + /** + TimeoutOnClose( + packet OpaquePacket, + proof, proofClosed ics23.CommitmentProof, + proofHeight Height, + nextSequenceRecv *uint64, + relayer string, + ) (Packet, error) + **/ +} From db8d8d6a16a27877cf57b9eabc3de2db3256c9d4 Mon Sep 17 00:00:00 2001 From: d7t Date: Wed, 26 Jul 2023 11:55:50 -0600 Subject: [PATCH 12/25] [Persistence] Adds `node` subcommand to CLI (#935) --- app/client/cli/node.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 app/client/cli/node.go diff --git a/app/client/cli/node.go b/app/client/cli/node.go new file mode 100644 index 000000000..96a8674c6 --- /dev/null +++ b/app/client/cli/node.go @@ -0,0 +1,18 @@ +package cli + +import "github.com/spf13/cobra" + +func init() { + nodeCmd := NewNodeCommand() + rootCmd.AddCommand(nodeCmd) +} + +func NewNodeCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "Node", + Short: "Commands related to node management and operations", + Aliases: []string{"node", "n"}, + } + + return cmd +} From 74a58162b4856a793521ea73a2cb1cedc3cfb683 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 27 Jul 2023 09:23:00 +0100 Subject: [PATCH 13/25] [IBC] chore: enable IBC module in k8s validators (#941) --- charts/pocket/README.md | 3 +++ charts/pocket/values.yaml | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/charts/pocket/README.md b/charts/pocket/README.md index 35993ddfd..c12c8d48e 100644 --- a/charts/pocket/README.md +++ b/charts/pocket/README.md @@ -44,6 +44,9 @@ privateKeySecretKeyRef: | config.consensus.pacemaker_config.timeout_msec | int | `10000` | | | config.consensus.private_key | string | `""` | | | config.fisherman.enabled | bool | `false` | | +| config.ibc.enabled | bool | `true` | | +| config.ibc.host.private_key | string | `""` | | +| config.ibc.stores_dir | string | `"/pocket/data/ibc"` | | | config.logger.format | string | `"json"` | | | config.logger.level | string | `"debug"` | | | config.p2p.hostname | string | `""` | | diff --git a/charts/pocket/values.yaml b/charts/pocket/values.yaml index 01d42b52e..31d4b03cc 100644 --- a/charts/pocket/values.yaml +++ b/charts/pocket/values.yaml @@ -117,6 +117,11 @@ config: enabled: false fisherman: enabled: false + ibc: + enabled: true + stores_dir: "/pocket/data/ibc" + host: + private_key: "" # @ignored This value is needed but ignored - use privateKeySecretKeyRef instead genesis: preProvisionedGenesis: From 950ccc378cf1e2996ec0c79fc42144c2af443e16 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 27 Jul 2023 09:36:21 +0100 Subject: [PATCH 14/25] [Utility] Use TreeStore as source of truth (#937) --- ibc/ibc_msg_mempool_test.go | 33 ++++++- persistence/block.go | 22 ++--- persistence/trees/trees.go | 18 ++++ persistence/trees/trees_test.go | 89 ++++++++++++++++++- shared/modules/persistence_module.go | 4 +- shared/modules/treestore_module.go | 3 + utility/transaction.go | 8 +- utility/transaction_test.go | 125 +++++++++++++++++---------- 8 files changed, 235 insertions(+), 67 deletions(-) diff --git a/ibc/ibc_msg_mempool_test.go b/ibc/ibc_msg_mempool_test.go index c82cb4875..b3467ab87 100644 --- a/ibc/ibc_msg_mempool_test.go +++ b/ibc/ibc_msg_mempool_test.go @@ -250,10 +250,39 @@ func TestHandleMessage_ErrorAlreadyInMempool(t *testing.T) { func TestHandleMessage_ErrorAlreadyCommitted(t *testing.T) { // Prepare the environment _, _, utilityMod, persistenceMod, _ := prepareEnvironment(t, 0, 0, 0, 0) - idxTx := prepareIndexedMessage(t, persistenceMod.GetTxIndexer()) + + privKey, err := crypto.GeneratePrivateKey() + require.NoError(t, err) + _, validPruneTx := preparePruneMessage(t, []byte("key")) + require.NoError(t, err) + err = validPruneTx.Sign(privKey) + require.NoError(t, err) + txProtoBytes, err := codec.GetCodec().Marshal(validPruneTx) + require.NoError(t, err) + + idxTx := &coreTypes.IndexedTransaction{ + Tx: txProtoBytes, + Height: 0, + Index: 0, + ResultCode: 0, + Error: "h5law", + SignerAddr: "h5law", + RecipientAddr: "h5law", + MessageType: "h5law", + } + + // Index a test transaction + err = persistenceMod.GetTxIndexer().Index(idxTx) + require.NoError(t, err) + + rwCtx, err := persistenceMod.NewRWContext(0) + require.NoError(t, err) + _, err = rwCtx.ComputeStateHash() + require.NoError(t, err) + rwCtx.Release() // Error on having an indexed transaction - err := utilityMod.HandleTransaction(idxTx.Tx) + err = utilityMod.HandleTransaction(idxTx.Tx) require.Error(t, err) require.EqualError(t, err, coreTypes.ErrTransactionAlreadyCommitted().Error()) } diff --git a/persistence/block.go b/persistence/block.go index e14d2777b..dff671c0b 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -1,11 +1,11 @@ package persistence import ( + "bytes" "encoding/hex" - "errors" "fmt" - "github.com/dgraph-io/badger/v3" + "github.com/pokt-network/pocket/persistence/trees" "github.com/pokt-network/pocket/persistence/types" "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" @@ -13,21 +13,17 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -func (p *persistenceModule) TransactionExists(transactionHash string) (bool, error) { - hash, err := hex.DecodeString(transactionHash) +func (p *persistenceModule) TransactionExists(txHash, txProtoBz []byte) (bool, error) { + exists, err := p.GetBus().GetTreeStore().Prove(trees.TransactionsTreeName, txHash, txProtoBz) if err != nil { return false, err } - res, err := p.txIndexer.GetByHash(hash) - if res == nil { - // check for not found - if err != nil && errors.Is(err, badger.ErrKeyNotFound) { - return false, nil - } - return false, err + // exclusion proof verification + if bytes.Equal(txProtoBz, nil) && exists { + return false, nil } - - return true, nil + // inclusion proof verification + return exists, nil } func (p *PostgresContext) GetMinimumBlockHeight() (latestHeight uint64, err error) { diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index 0ec61b79e..2d47cdc43 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -103,6 +103,24 @@ func (t *treeStore) GetTree(name string) ([]byte, kvstore.KVStore) { return nil, nil } +// Prove generates and verifies a proof against the tree name stored in the TreeStore +// using the given key-value pair. If value == nil this will be an exclusion proof, +// otherwise it will be an inclusion proof. +func (t *treeStore) Prove(name string, key, value []byte) (bool, error) { + st, ok := t.merkleTrees[name] + if !ok { + return false, fmt.Errorf("tree not found: %s", name) + } + proof, err := st.tree.Prove(key) + if err != nil { + return false, fmt.Errorf("error generating proof (%s): %w", name, err) + } + if valid := smt.VerifyProof(proof, st.tree.Root(), key, value, st.tree.Spec()); !valid { + return false, nil + } + return true, nil +} + // GetTreeHashes returns a map of tree names to their root hashes for all // the trees tracked by the treestore, excluding the root tree func (t *treeStore) GetTreeHashes() map[string]string { diff --git a/persistence/trees/trees_test.go b/persistence/trees/trees_test.go index 8acd2a64f..e59e3ba1f 100644 --- a/persistence/trees/trees_test.go +++ b/persistence/trees/trees_test.go @@ -1,6 +1,13 @@ package trees -import "testing" +import ( + "fmt" + "testing" + + "github.com/pokt-network/pocket/persistence/kvstore" + "github.com/pokt-network/smt" + "github.com/stretchr/testify/require" +) // TECHDEBT(#836): Tests added in https://github.com/pokt-network/pocket/pull/836 func TestTreeStore_Update(t *testing.T) { @@ -22,3 +29,83 @@ func TestTreeStore_DebugClearAll(t *testing.T) { func TestTreeStore_GetTreeHashes(t *testing.T) { t.Skip("TODO: Write test case for GetTreeHashes method") // context: https://github.com/pokt-network/pocket/pull/915#discussion_r1267313664 } + +func TestTreeStore_Prove(t *testing.T) { + nodeStore := kvstore.NewMemKVStore() + tree := smt.NewSparseMerkleTree(nodeStore, smtTreeHasher) + testTree := &stateTree{ + name: "test", + tree: tree, + nodeStore: nodeStore, + } + + require.NoError(t, testTree.tree.Update([]byte("key"), []byte("value"))) + require.NoError(t, testTree.tree.Commit()) + + treeStore := &treeStore{ + merkleTrees: make(map[string]*stateTree, 1), + } + treeStore.merkleTrees["test"] = testTree + + testCases := []struct { + name string + treeName string + key []byte + value []byte + valid bool + expectedErr error + }{ + { + name: "valid inclusion proof: key and value in tree", + treeName: "test", + key: []byte("key"), + value: []byte("value"), + valid: true, + expectedErr: nil, + }, + { + name: "valid exclusion proof: key not in tree", + treeName: "test", + key: []byte("key2"), + value: nil, + valid: true, + expectedErr: nil, + }, + { + name: "invalid proof: tree not in store", + treeName: "unstored tree", + key: []byte("key"), + value: []byte("value"), + valid: false, + expectedErr: fmt.Errorf("tree not found: %s", "unstored tree"), + }, + { + name: "invalid inclusion proof: key in tree, wrong value", + treeName: "test", + key: []byte("key"), + value: []byte("wrong value"), + valid: false, + expectedErr: nil, + }, + { + name: "invalid exclusion proof: key in tree", + treeName: "test", + key: []byte("key"), + value: nil, + valid: false, + expectedErr: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + valid, err := treeStore.Prove(tc.treeName, tc.key, tc.value) + require.Equal(t, valid, tc.valid) + if tc.expectedErr == nil { + require.NoError(t, err) + return + } + require.ErrorAs(t, err, &tc.expectedErr) + }) + } +} diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 6ee724d63..b3c1e56e2 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -33,7 +33,9 @@ type PersistenceModule interface { // Indexer operations GetTxIndexer() indexer.TxIndexer - TransactionExists(transactionHash string) (bool, error) + + // TreeStore operations + TransactionExists(txHash, txProtoBz []byte) (bool, error) // Debugging / development only HandleDebugMessage(*messaging.DebugMessage) error diff --git a/shared/modules/treestore_module.go b/shared/modules/treestore_module.go index 117026f05..35b240e51 100644 --- a/shared/modules/treestore_module.go +++ b/shared/modules/treestore_module.go @@ -31,6 +31,9 @@ type TreeStoreModule interface { Update(pgtx pgx.Tx, height uint64) (string, error) // DebugClearAll completely clears the state of the trees. For debugging purposes only. DebugClearAll() error + // Prove generates and verifies a proof against the tree with the matching name using the given + // key and value. If value == nil, it will verify non-membership of the key, otherwise membership. + Prove(treeName string, key, value []byte) (bool, error) // GetTree returns the specified tree's root and nodeStore in order to be imported elsewhere GetTree(name string) ([]byte, kvstore.KVStore) // GetTreeHashes returns a map of tree names to their root hashes diff --git a/utility/transaction.go b/utility/transaction.go index e0bfed939..400e9ad0f 100644 --- a/utility/transaction.go +++ b/utility/transaction.go @@ -7,19 +7,21 @@ import ( "github.com/dgraph-io/badger/v3" "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/crypto" ) // HandleTransaction implements the exposed functionality of the shared utilityModule interface. func (u *utilityModule) HandleTransaction(txProtoBytes []byte) error { - txHash := coreTypes.TxHash(txProtoBytes) + txHash := crypto.SHA3Hash(txProtoBytes) // Is the tx already in the mempool (in memory)? - if u.mempool.Contains(txHash) { + if u.mempool.Contains(hex.EncodeToString(txHash)) { return coreTypes.ErrDuplicateTransaction() } // Is the tx already committed & indexed (on disk)? - if txExists, err := u.GetBus().GetPersistenceModule().TransactionExists(txHash); err != nil { + txExists, err := u.GetBus().GetPersistenceModule().TransactionExists(txHash, txProtoBytes) + if err != nil { return err } else if txExists { return coreTypes.ErrTransactionAlreadyCommitted() diff --git a/utility/transaction_test.go b/utility/transaction_test.go index 4d51702a0..ffc8e234f 100644 --- a/utility/transaction_test.go +++ b/utility/transaction_test.go @@ -3,21 +3,21 @@ package utility import ( "fmt" "strconv" + "strings" "testing" "github.com/pokt-network/pocket/persistence/indexer" "github.com/pokt-network/pocket/shared/codec" - "github.com/pokt-network/pocket/shared/core/types" - coreTypes "github.com/pokt-network/pocket/shared/core/types" + core_types "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" - typesUtil "github.com/pokt-network/pocket/utility/types" + util_types "github.com/pokt-network/pocket/utility/types" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" ) func TestHandleTransaction_ErrorAlreadyInMempool(t *testing.T) { // Prepare test data - emptyTx := types.Transaction{} + emptyTx := core_types.Transaction{} txProtoBytes, err := proto.Marshal(&emptyTx) require.NoError(t, err) @@ -31,18 +31,48 @@ func TestHandleTransaction_ErrorAlreadyInMempool(t *testing.T) { // Error on having a duplciate transaction err = utilityMod.HandleTransaction(txProtoBytes) require.Error(t, err) - require.EqualError(t, err, coreTypes.ErrDuplicateTransaction().Error()) + require.EqualError(t, err, core_types.ErrDuplicateTransaction().Error()) } func TestHandleTransaction_ErrorAlreadyCommitted(t *testing.T) { // Prepare the environment _, utilityMod, persistenceMod := prepareEnvironment(t, 0, 0, 0, 0) - idxTx := prepareEmptyIndexedTransaction(t, persistenceMod.GetTxIndexer()) + + privKey, err := crypto.GeneratePrivateKey() + require.NoError(t, err) + + emptyTx := &core_types.Transaction{} + err = emptyTx.Sign(privKey) + require.NoError(t, err) + txProtoBytes, err := codec.GetCodec().Marshal(emptyTx) + require.NoError(t, err) + + // Test data - Prepare IndexedTransaction + idxTx := &core_types.IndexedTransaction{ + Tx: txProtoBytes, + Height: 0, + Index: 0, + ResultCode: 0, + Error: "Olshansky", + SignerAddr: "Olshansky", + RecipientAddr: "Olshansky", + MessageType: "Olshansky", + } + + // Index a test transaction + err = persistenceMod.GetTxIndexer().Index(idxTx) + require.NoError(t, err) + + rwCtx, err := persistenceMod.NewRWContext(0) + require.NoError(t, err) + _, err = rwCtx.ComputeStateHash() + require.NoError(t, err) + rwCtx.Release() // Error on having an indexed transaction - err := utilityMod.HandleTransaction(idxTx.Tx) + err = utilityMod.HandleTransaction(idxTx.Tx) require.Error(t, err) - require.EqualError(t, err, coreTypes.ErrTransactionAlreadyCommitted().Error()) + require.EqualError(t, err, core_types.ErrTransactionAlreadyCommitted().Error()) } func TestHandleTransaction_BasicValidation(t *testing.T) { @@ -51,7 +81,7 @@ func TestHandleTransaction_BasicValidation(t *testing.T) { pubKey := privKey.PublicKey() - message := &typesUtil.MessageSend{ + message := &util_types.MessageSend{ FromAddress: []byte("from"), ToAddress: []byte("to"), Amount: "10", @@ -59,9 +89,9 @@ func TestHandleTransaction_BasicValidation(t *testing.T) { anyMessage, err := codec.GetCodec().ToAny(message) require.NoError(t, err) - validTx := &types.Transaction{ + validTx := &core_types.Transaction{ Nonce: strconv.Itoa(int(crypto.GetNonce())), - Signature: &types.Signature{ + Signature: &core_types.Signature{ PublicKey: []byte("public key"), Signature: []byte("signature"), }, @@ -72,79 +102,78 @@ func TestHandleTransaction_BasicValidation(t *testing.T) { testCases := []struct { name string - txProto *coreTypes.Transaction + txProto *core_types.Transaction expectedErr error }{ { name: "Invalid transaction: Missing Nonce", - txProto: &types.Transaction{}, - expectedErr: types.ErrEmptyNonce(), + txProto: &core_types.Transaction{}, + expectedErr: core_types.ErrEmptyNonce(), }, { name: "Invalid transaction: Missing Signature Structure", - txProto: &types.Transaction{ + txProto: &core_types.Transaction{ Nonce: strconv.Itoa(int(crypto.GetNonce())), }, - expectedErr: types.ErrEmptySignatureStructure(), + expectedErr: core_types.ErrEmptySignatureStructure(), }, { name: "Invalid transaction: Missing Signature", - txProto: &types.Transaction{ + txProto: &core_types.Transaction{ Nonce: strconv.Itoa(int(crypto.GetNonce())), - Signature: &types.Signature{ + Signature: &core_types.Signature{ PublicKey: nil, Signature: nil, }, }, - expectedErr: types.ErrEmptySignature(), + expectedErr: core_types.ErrEmptySignature(), }, { name: "Invalid transaction: Missing Public Key", - txProto: &types.Transaction{ + txProto: &core_types.Transaction{ Nonce: strconv.Itoa(int(crypto.GetNonce())), - Signature: &types.Signature{ + Signature: &core_types.Signature{ PublicKey: nil, Signature: []byte("bytes in place for signature but not actually valid"), }, }, - expectedErr: types.ErrEmptyPublicKey(), + expectedErr: core_types.ErrEmptyPublicKey(), }, { name: "Invalid transaction: Invalid Public Key", - txProto: &types.Transaction{ + txProto: &core_types.Transaction{ Nonce: strconv.Itoa(int(crypto.GetNonce())), - Signature: &types.Signature{ + Signature: &core_types.Signature{ PublicKey: []byte("invalid pub key"), Signature: []byte("bytes in place for signature but not actually valid"), }, }, - expectedErr: types.ErrNewPublicKeyFromBytes(fmt.Errorf("the public key length is not valid, expected length 32, actual length: 15")), + expectedErr: core_types.ErrNewPublicKeyFromBytes(fmt.Errorf("the public key length is not valid, expected length 32, actual length: 15")), }, // TODO(olshansky): Figure out why sometimes we do and don't need `\u00a0` in the error - // { - // name: "Invalid transaction: Invalid Message", - // txProto: &types.Transaction{ - // Nonce: strconv.Itoa(int(crypto.GetNonce())), - // Signature: &types.Signature{ - // PublicKey: pubKey.Bytes(), - // Signature: []byte("bytes in place for signature but not actually valid"), - // }, - // Msg: nil, - // }, - // expectedErr: types.ErrDecodeMessage(fmt.Errorf("proto: invalid empty type URL")), - // expectedErr: types.ErrDecodeMessage(fmt.Errorf("proto:\u00a0invalid empty type URL")), - // }, + { + name: "Invalid transaction: Invalid Message", + txProto: &core_types.Transaction{ + Nonce: strconv.Itoa(int(crypto.GetNonce())), + Signature: &core_types.Signature{ + PublicKey: pubKey.Bytes(), + Signature: []byte("bytes in place for signature but not actually valid"), + }, + Msg: nil, + }, + expectedErr: core_types.ErrDecodeMessage(fmt.Errorf("proto: invalid empty type URL")), + }, { name: "Invalid transaction: Invalid Signature", - txProto: &types.Transaction{ + txProto: &core_types.Transaction{ Nonce: strconv.Itoa(int(crypto.GetNonce())), - Signature: &types.Signature{ + Signature: &core_types.Signature{ PublicKey: pubKey.Bytes(), Signature: []byte("invalid signature"), }, Msg: anyMessage, }, - expectedErr: types.ErrSignatureVerificationFailed(), + expectedErr: core_types.ErrSignatureVerificationFailed(), }, { name: "Valid well-formatted transaction with valid signature", @@ -158,12 +187,14 @@ func TestHandleTransaction_BasicValidation(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - txProtoBytes, err := proto.Marshal(tc.txProto) + txProtoBytes, err := codec.GetCodec().Marshal(tc.txProto) require.NoError(t, err) err = utilityMod.HandleTransaction(txProtoBytes) if tc.expectedErr != nil { - require.EqualError(t, err, tc.expectedErr.Error()) + errMsg := err.Error() + errMsg = strings.Replace(errMsg, string('\u00a0'), " ", 1) + require.EqualError(t, tc.expectedErr, errMsg) } else { require.NoError(t, err) } @@ -183,7 +214,7 @@ func TestGetIndexedTransaction(t *testing.T) { expectErr error }{ {"returns indexed transaction when it exists", idxTx.Tx, true, nil}, - {"returns error when transaction doesn't exist", []byte("Does not exist"), false, types.ErrTransactionNotCommitted()}, + {"returns error when transaction doesn't exist", []byte("Does not exist"), false, core_types.ErrTransactionNotCommitted()}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -199,16 +230,16 @@ func TestGetIndexedTransaction(t *testing.T) { } } -func prepareEmptyIndexedTransaction(t *testing.T, txIndexer indexer.TxIndexer) *coreTypes.IndexedTransaction { +func prepareEmptyIndexedTransaction(t *testing.T, txIndexer indexer.TxIndexer) *core_types.IndexedTransaction { t.Helper() // Test data - Prepare Transaction - emptyTx := types.Transaction{} + emptyTx := core_types.Transaction{} txProtoBytes, err := proto.Marshal(&emptyTx) require.NoError(t, err) // Test data - Prepare IndexedTransaction - idxTx := &coreTypes.IndexedTransaction{ + idxTx := &core_types.IndexedTransaction{ Tx: txProtoBytes, Height: 0, Index: 0, From d3bf0ad4661dbb43744d847dccc95c4aadafe5ab Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:58:11 +0100 Subject: [PATCH 15/25] [IBC] Enable validators and thus IBC host creation in K8s (#942) --- build/localnet/manifests/cli-client.yaml | 5 +++++ charts/pocket/README.md | 1 + charts/pocket/templates/statefulset.yaml | 5 +++++ charts/pocket/values.yaml | 2 ++ 4 files changed, 13 insertions(+) diff --git a/build/localnet/manifests/cli-client.yaml b/build/localnet/manifests/cli-client.yaml index c173910aa..da85d5a62 100644 --- a/build/localnet/manifests/cli-client.yaml +++ b/build/localnet/manifests/cli-client.yaml @@ -57,6 +57,11 @@ spec: secretKeyRef: name: validators-private-keys key: "001" + - name: POCKET_IBC_HOST_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: validators-private-keys + key: "001" - name: POSTGRES_USER value: "postgres" - name: POSTGRES_PASSWORD diff --git a/charts/pocket/README.md b/charts/pocket/README.md index c12c8d48e..bbac55a3d 100644 --- a/charts/pocket/README.md +++ b/charts/pocket/README.md @@ -77,6 +77,7 @@ privateKeySecretKeyRef: | config.telemetry.endpoint | string | `"/metrics"` | | | config.utility.max_mempool_transaction_bytes | int | `1073741824` | | | config.utility.max_mempool_transactions | int | `9000` | | +| config.validator.enabled | bool | `true` | | | externalPostgresql.database | string | `""` | name of the external database | | externalPostgresql.enabled | bool | `false` | use external postgres database | | externalPostgresql.host | string | `""` | host of the external database | diff --git a/charts/pocket/templates/statefulset.yaml b/charts/pocket/templates/statefulset.yaml index 4be68b50b..ef2d21a30 100644 --- a/charts/pocket/templates/statefulset.yaml +++ b/charts/pocket/templates/statefulset.yaml @@ -81,6 +81,11 @@ spec: secretKeyRef: name: {{ .Values.privateKeySecretKeyRef.name | quote }} key: {{ .Values.privateKeySecretKeyRef.key | quote }} + - name: POCKET_IBC_HOST_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.privateKeySecretKeyRef.name | quote }} + key: {{ .Values.privateKeySecretKeyRef.key | quote }} {{ end }} - name: POSTGRES_USER {{ include "pocket.postgresqlUserValueOrSecretRef" . | nindent 14 }} diff --git a/charts/pocket/values.yaml b/charts/pocket/values.yaml index 31d4b03cc..9587814c7 100644 --- a/charts/pocket/values.yaml +++ b/charts/pocket/values.yaml @@ -113,6 +113,8 @@ config: port: "50832" timeout: 30000 use_cors: false + validator: + enabled: true servicer: enabled: false fisherman: From c903ca1537038834991ac82b0bb1aaa0429e72ea Mon Sep 17 00:00:00 2001 From: Arash <23505281+adshmh@users.noreply.github.com> Date: Mon, 31 Jul 2023 08:53:10 -0400 Subject: [PATCH 16/25] [Utility] Create trustless_relay_validation.md (#938) ## Description Add diagrams to describe validations done on trustless relays. Part of work on #918 ## Issue Fixes #918 ## Type of change Please mark the relevant option(s): - [ ] New feature, functionality or library - [ ] Bug fix - [ ] Code health or cleanup - [ ] Major breaking change - [X] Documentation - [ ] Other ## List of changes - Add a new markdown file to show trustless relay validations. ## Testing - [ ] `make develop_test`; if any code changes were made - [ ] `make test_e2e` on [k8s LocalNet](https://github.com/pokt-network/pocket/blob/main/build/localnet/README.md); if any code changes were made - [ ] `e2e-devnet-test` passes tests on [DevNet](https://pocketnetwork.notion.site/How-to-DevNet-ff1598f27efe44c09f34e2aa0051f0dd); if any code was changed - [ ] [Docker Compose LocalNet](https://github.com/pokt-network/pocket/blob/main/docs/development/README.md); if any major functionality was changed or introduced - [ ] [k8s LocalNet](https://github.com/pokt-network/pocket/blob/main/build/localnet/README.md); if any infrastructure or configuration changes were made ## Required Checklist - [x] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have added, or updated, [`godoc` format comments](https://go.dev/blog/godoc) on touched members (see: [tip.golang.org/doc/comment](https://tip.golang.org/doc/comment)) - [ ] I have tested my changes using the available tooling - [ ] I have updated the corresponding CHANGELOG ### If Applicable Checklist - [ ] I have updated the corresponding README(s); local and/or global - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] I have added, or updated, [mermaid.js](https://mermaid-js.github.io) diagrams in the corresponding README(s) - [ ] I have added, or updated, documentation and [mermaid.js](https://mermaid-js.github.io) diagrams in `shared/docs/*` if I updated `shared/*`README(s) --- utility/doc/README.md | 2 + utility/doc/TRUSTLESS_RELAY_VALIDATION.md | 73 +++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 utility/doc/TRUSTLESS_RELAY_VALIDATION.md diff --git a/utility/doc/README.md b/utility/doc/README.md index aed27a0b0..046e06d13 100644 --- a/utility/doc/README.md +++ b/utility/doc/README.md @@ -27,6 +27,8 @@ And implement the basic transaction functionality: - Pause - Unpause +And implement [the trustless relay validation and execution](TRUSTLESS_RELAY_VALIDATION.md) + Added governance params: - BlocksPerSessionParamName diff --git a/utility/doc/TRUSTLESS_RELAY_VALIDATION.md b/utility/doc/TRUSTLESS_RELAY_VALIDATION.md new file mode 100644 index 000000000..4463ed432 --- /dev/null +++ b/utility/doc/TRUSTLESS_RELAY_VALIDATION.md @@ -0,0 +1,73 @@ +# Validation of Trustless Relays + +- [A. Client-side Relay Validation](#a-client-side-relay-validation) +- [B. Server-side Relay Validation](#b-server-side-relay-validation) + +## A. Client-side Relay Validation + +When an application requests to send a trustless relay, the CLI performs several checks on the relay before sending it to the specified servicer. +The following diagram lists all these checks with links to the corresponding code secion (or an issue if the check is not implemented yet). + +```mermaid +--- +title: Client-side Relay Validation +--- +graph TD + app_key{Validate app key} + session{Validate the Session} + servicer{Validate the Servicer} + payload{Deserialize Payload} + relay{Validate relay contents} + send[Send Trustless Relay to the provided Servicer] + user_err[Return error to user] + + app_key-->|Failure| user_err + session-->|Failure| user_err + servicer-->|Failure| user_err + payload-->|Failure| user_err + relay-->|Failure| user_err + + app_key-->|Success| session + session-->|Success| servicer + servicer-->|Success| payload + payload-->|Success| relay + relay-->|Success| send +``` + +## B. Server-side Relay Validation + +Once a trustless relay has been received on the server side, i.e. by the servicer, several validations are performed on the relay. +The following diagram outlines all these checks along with links to the corresponding section of the code (or to an issue if the check has not been implemented yet) + +```mermaid +--- +title: Server-side Relay Validation +--- +graph TD + deserialize{Deserialize Relay Payload} + meta{Validate Relay Meta} + chain_support{Validate chain support} + session{Validate the Session} + height{Validate Relay Height} + servicer{Validate Servicer} + mine_relay{Validate the app rate limit} + execute[Execute the Relay] + client_err[Return error to client] + + deserialize-->|Failure| client_err + meta-->|Failure| client_err + chain_support-->|Failure| client_err + session-->|Failure| client_err + height-->|Failure| client_err + servicer-->|Failure| client_err + mine_relay-->|Failure| client_err + + deserialize-->|Success| meta + meta-->|Success| chain_support + chain_support-->|Success| session + session-->|Success| height + height-->|Success| servicer + servicer-->|Success| mine_relay + mine_relay-->|Success| execute +``` + From 298b08f625cc95d6da9459b1bd5dd8e4150d87ac Mon Sep 17 00:00:00 2001 From: d7t Date: Mon, 31 Jul 2023 10:04:12 -0600 Subject: [PATCH 17/25] [Persistence] Adds atomic Update for TreeStore (#861) Co-authored-by: Daniel Olshansky --- persistence/context.go | 23 +- persistence/db.go | 1 + persistence/docs/CHANGELOG.md | 4 + persistence/trees/atomic_test.go | 92 ++++++++ persistence/trees/main_test.go | 12 ++ persistence/trees/module_test.go | 1 - persistence/trees/prove_test.go | 90 ++++++++ persistence/trees/trees.go | 92 +++++++- persistence/trees/trees_test.go | 270 ++++++++++++++++-------- shared/modules/persistence_module.go | 11 +- shared/modules/treestore_module.go | 5 +- utility/unit_of_work/block.go | 35 +-- utility/unit_of_work/module.go | 43 ++-- utility/unit_of_work/uow_leader.go | 9 +- utility/unit_of_work/uow_leader_test.go | 129 +++++++++++ 15 files changed, 664 insertions(+), 153 deletions(-) create mode 100644 persistence/trees/atomic_test.go create mode 100644 persistence/trees/main_test.go create mode 100644 persistence/trees/prove_test.go create mode 100644 utility/unit_of_work/uow_leader_test.go diff --git a/persistence/context.go b/persistence/context.go index 58ad0e51d..15a89dfe1 100644 --- a/persistence/context.go +++ b/persistence/context.go @@ -12,6 +12,7 @@ import ( "github.com/pokt-network/pocket/persistence/indexer" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" + "go.uber.org/multierr" ) var _ modules.PersistenceRWContext = &PostgresContext{} @@ -36,29 +37,35 @@ type PostgresContext struct { networkId string } -func (p *PostgresContext) NewSavePoint(bytes []byte) error { - p.logger.Info().Bool("TODO", true).Msg("NewSavePoint not implemented") +// SetSavePoint generates a new Savepoint for this context. +func (p *PostgresContext) SetSavePoint() error { + if err := p.stateTrees.Savepoint(); err != nil { + return err + } return nil } -// TECHDEBT(#327): Guarantee atomicity betweens `prepareBlock`, `insertBlock` and `storeBlock` for save points & rollbacks. -func (p *PostgresContext) RollbackToSavePoint(bytes []byte) error { - p.logger.Info().Bool("TODO", true).Msg("RollbackToSavePoint not fully implemented") - return p.tx.Rollback(context.TODO()) +// RollbackToSavepoint triggers a rollback for the current pgx transaction and the underylying submodule stores. +func (p *PostgresContext) RollbackToSavePoint() error { + ctx, _ := p.getCtxAndTx() + pgErr := p.tx.Rollback(ctx) + treesErr := p.stateTrees.Rollback() + return multierr.Combine(pgErr, treesErr) } -// IMPROVE(#361): Guarantee the integrity of the state // Full details in the thread from the PR review: https://github.com/pokt-network/pocket/pull/285#discussion_r1018471719 func (p *PostgresContext) ComputeStateHash() (string, error) { stateHash, err := p.stateTrees.Update(p.tx, uint64(p.Height)) if err != nil { return "", err } + if err := p.stateTrees.Commit(); err != nil { + return "", err + } p.stateHash = stateHash return p.stateHash, nil } -// TECHDEBT(#327): Make sure these operations are atomic func (p *PostgresContext) Commit(proposerAddr, quorumCert []byte) error { p.logger.Info().Int64("height", p.Height).Msg("About to commit block & context") diff --git a/persistence/db.go b/persistence/db.go index 2a65e7819..73e64cada 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -37,6 +37,7 @@ var protocolActorSchemas = []types.ProtocolActorSchema{ types.ValidatorActor, } +// TECHDEBT(#595): Properly handle context threading and passing for the entire persistence module func (pg *PostgresContext) getCtxAndTx() (context.Context, pgx.Tx) { return context.TODO(), pg.tx } diff --git a/persistence/docs/CHANGELOG.md b/persistence/docs/CHANGELOG.md index 1bdae0d55..7f6dfb0b3 100644 --- a/persistence/docs/CHANGELOG.md +++ b/persistence/docs/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.60] - 2023-07-11 + +- Adds savepoints and rollbacks implementation to TreeStore + ## [0.0.0.60] - 2023-06-26 - Add place-holder for local context and servicer token usage support methods diff --git a/persistence/trees/atomic_test.go b/persistence/trees/atomic_test.go new file mode 100644 index 000000000..06fdbaf8c --- /dev/null +++ b/persistence/trees/atomic_test.go @@ -0,0 +1,92 @@ +package trees + +import ( + "encoding/hex" + "testing" + + "github.com/golang/mock/gomock" + "github.com/pokt-network/pocket/logger" + mock_types "github.com/pokt-network/pocket/persistence/types/mocks" + "github.com/pokt-network/pocket/shared/modules" + mockModules "github.com/pokt-network/pocket/shared/modules/mocks" + + "github.com/stretchr/testify/require" +) + +const ( + // the root hash of a tree store where each tree is empty but present and initialized + h0 = "302f2956c084cc3e0e760cf1b8c2da5de79c45fa542f68a660a5fc494b486972" + // the root hash of a tree store where each tree has has key foo value bar added to it + h1 = "7d5712ea1507915c40e295845fa58773baa405b24b87e9d99761125d826ff915" +) + +func TestTreeStore_AtomicUpdatesWithSuccessfulRollback(t *testing.T) { + ctrl := gomock.NewController(t) + + mockTxIndexer := mock_types.NewMockTxIndexer(ctrl) + mockBus := mockModules.NewMockBus(ctrl) + mockPersistenceMod := mockModules.NewMockPersistenceModule(ctrl) + + mockBus.EXPECT().GetPersistenceModule().AnyTimes().Return(mockPersistenceMod) + mockPersistenceMod.EXPECT().GetTxIndexer().AnyTimes().Return(mockTxIndexer) + + ts := &treeStore{ + logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName), + treeStoreDir: ":memory:", + } + require.NoError(t, ts.setupTrees()) + require.NotEmpty(t, ts.merkleTrees[TransactionsTreeName]) + + hash0 := ts.getStateHash() + require.NotEmpty(t, hash0) + require.Equal(t, hash0, h0) + + require.NoError(t, ts.Savepoint()) + + // insert test data into every tree + for _, treeName := range stateTreeNames { + err := ts.merkleTrees[treeName].tree.Update([]byte("foo"), []byte("bar")) + require.NoError(t, err) + } + + // commit the above changes + require.NoError(t, ts.Commit()) + + // assert state hash is changed + hash1 := ts.getStateHash() + require.NotEmpty(t, hash1) + require.NotEqual(t, hash0, hash1) + require.Equal(t, hash1, h1) + + // set a new savepoint + require.NoError(t, ts.Savepoint()) + require.NotEmpty(t, ts.prevState.merkleTrees) + require.NotEmpty(t, ts.prevState.rootTree) + // assert that savepoint creation doesn't mutate state hash + require.Equal(t, hash1, hex.EncodeToString(ts.prevState.rootTree.tree.Root())) + + // verify that creating a savepoint does not change state hash + hash2 := ts.getStateHash() + require.Equal(t, hash2, hash1) + require.Equal(t, hash2, h1) + + // validate that state tree was updated and a previous savepoint is created + for _, treeName := range stateTreeNames { + require.NotEmpty(t, ts.merkleTrees[treeName]) + require.NotEmpty(t, ts.prevState.merkleTrees[treeName]) + } + + // insert additional test data into all of the trees + for _, treeName := range stateTreeNames { + require.NoError(t, ts.merkleTrees[treeName].tree.Update([]byte("fiz"), []byte("buz"))) + } + + // rollback the changes made to the trees above BEFORE anything was committed + err := ts.Rollback() + require.NoError(t, err) + + // validate that the state hash is unchanged after new data was inserted but rolled back before commitment + hash3 := ts.getStateHash() + require.Equal(t, hash3, hash2) + require.Equal(t, hash3, h1) +} diff --git a/persistence/trees/main_test.go b/persistence/trees/main_test.go new file mode 100644 index 000000000..9d5615ecb --- /dev/null +++ b/persistence/trees/main_test.go @@ -0,0 +1,12 @@ +//go:build test + +package trees + +import ( + "crypto/sha256" + "hash" +) + +type TreeStore = treeStore + +var SMTTreeHasher hash.Hash = sha256.New() diff --git a/persistence/trees/module_test.go b/persistence/trees/module_test.go index 7c7bc660c..91ec5249f 100644 --- a/persistence/trees/module_test.go +++ b/persistence/trees/module_test.go @@ -48,7 +48,6 @@ func TestTreeStore_Create(t *testing.T) { treemod, err := trees.Create(mockBus, trees.WithTreeStoreDirectory(":memory:")) assert.NoError(t, err) - got := treemod.GetBus() assert.Equal(t, got, mockBus) diff --git a/persistence/trees/prove_test.go b/persistence/trees/prove_test.go new file mode 100644 index 000000000..5d6cdb4c3 --- /dev/null +++ b/persistence/trees/prove_test.go @@ -0,0 +1,90 @@ +package trees + +import ( + "fmt" + "testing" + + "github.com/pokt-network/pocket/persistence/kvstore" + "github.com/pokt-network/smt" + "github.com/stretchr/testify/require" +) + +func TestTreeStore_Prove(t *testing.T) { + nodeStore := kvstore.NewMemKVStore() + tree := smt.NewSparseMerkleTree(nodeStore, smtTreeHasher) + testTree := &stateTree{ + name: "test", + tree: tree, + nodeStore: nodeStore, + } + + require.NoError(t, testTree.tree.Update([]byte("key"), []byte("value"))) + require.NoError(t, testTree.tree.Commit()) + + treeStore := &treeStore{ + merkleTrees: make(map[string]*stateTree, 1), + } + treeStore.merkleTrees["test"] = testTree + + testCases := []struct { + name string + treeName string + key []byte + value []byte + valid bool + expectedErr error + }{ + { + name: "valid inclusion proof: key and value in tree", + treeName: "test", + key: []byte("key"), + value: []byte("value"), + valid: true, + expectedErr: nil, + }, + { + name: "valid exclusion proof: key not in tree", + treeName: "test", + key: []byte("key2"), + value: nil, + valid: true, + expectedErr: nil, + }, + { + name: "invalid proof: tree not in store", + treeName: "unstored tree", + key: []byte("key"), + value: []byte("value"), + valid: false, + expectedErr: fmt.Errorf("tree not found: %s", "unstored tree"), + }, + { + name: "invalid inclusion proof: key in tree, wrong value", + treeName: "test", + key: []byte("key"), + value: []byte("wrong value"), + valid: false, + expectedErr: nil, + }, + { + name: "invalid exclusion proof: key in tree", + treeName: "test", + key: []byte("key"), + value: nil, + valid: false, + expectedErr: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + valid, err := treeStore.Prove(tc.treeName, tc.key, tc.value) + require.Equal(t, valid, tc.valid) + if tc.expectedErr == nil { + require.NoError(t, err) + return + } + require.ErrorAs(t, err, &tc.expectedErr) + }) + } +} diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index 2d47cdc43..8fdc43676 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -1,7 +1,15 @@ // package trees maintains a set of sparse merkle trees -// each backed by the KVStore interface. It offers an atomic +// each backed by the `KVStore` interface. It offers an atomic // commit and rollback mechanism for interacting with -// that core resource map of merkle trees. +// its core resource - a set of merkle trees. +// - `Update` is called, which will fetch and apply the contextual changes to the respective trees. +// - `Savepoint` is first called to create a new anchor in time that can be rolled back to +// - `Commit` must be called after any `Update` calls to persist changes applied to disk. +// - If `Rollback` is called at any point before committing, it rolls the TreeStore state back to the +// earlier savepoint. This means that the caller is responsible for correctly managing atomic updates +// of the TreeStore. +// In most contexts, this is from the perspective of the `utility/unit_of_work` package. + package trees import ( @@ -74,6 +82,9 @@ type stateTree struct { var _ modules.TreeStoreModule = &treeStore{} +// ErrFailedRollback is thrown when a rollback fails to reset the TreeStore to a known good state +var ErrFailedRollback = fmt.Errorf("failed to rollback") + // treeStore stores a set of merkle trees that it manages. // It fulfills the modules.treeStore interface // * It is responsible for atomic commit or rollback behavior of the underlying @@ -88,6 +99,18 @@ type treeStore struct { treeStoreDir string rootTree *stateTree merkleTrees map[string]*stateTree + + // prevState holds a previous view of the worldState. + // The tree store rolls back to this view if errors are encountered during block application. + prevState *worldState +} + +// worldState holds a (de)serializable view of the entire tree state. +// TECHDEBT(#566) - Hook this up to node CLI subcommands +type worldState struct { + treeStoreDir string + rootTree *stateTree + merkleTrees map[string]*stateTree } // GetTree returns the root hash and nodeStore for the matching tree stored in the TreeStore. @@ -241,9 +264,6 @@ func (t *treeStore) updateMerkleTrees(pgtx pgx.Tx, txi indexer.TxIndexer, height } } - if err := t.Commit(); err != nil { - return "", fmt.Errorf("failed to commit: %w", err) - } return t.getStateHash(), nil } @@ -279,6 +299,67 @@ func (t *treeStore) getStateHash() string { return hexHash } +//////////////////////////////// +// AtomicStore Implementation // +//////////////////////////////// + +// Savepoint generates a new savepoint (i.e. a worldState) for the tree store and saves it internally. +func (t *treeStore) Savepoint() error { + w, err := t.save() + if err != nil { + return err + } + t.prevState = w + return nil +} + +// Rollback returns the treeStore to the last saved worldState maintained by the treeStore. +// If no worldState has been saved, it returns ErrFailedRollback +func (t *treeStore) Rollback() error { + if t.prevState != nil { + t.merkleTrees = t.prevState.merkleTrees + t.rootTree = t.prevState.rootTree + return nil + } + t.logger.Err(ErrFailedRollback) + return ErrFailedRollback +} + +// save commits any pending changes to the trees and creates a copy of the current worldState, +// then saves that copy as a rollback point for later use if errors are encountered. +// OPTIMIZE: Consider saving only the root hash of each tree and the tree directory here and then +// load the trees up in Rollback instead of setting them up here. +func (t *treeStore) save() (*worldState, error) { + if err := t.Commit(); err != nil { + return nil, err + } + + w := &worldState{ + treeStoreDir: t.treeStoreDir, + merkleTrees: map[string]*stateTree{}, + } + + for treeName := range t.merkleTrees { + root, nodeStore := t.GetTree(treeName) + tree := smt.ImportSparseMerkleTree(nodeStore, smtTreeHasher, root) + w.merkleTrees[treeName] = &stateTree{ + name: treeName, + tree: tree, + nodeStore: nodeStore, + } + } + + root, nodeStore := t.GetTree(RootTreeName) + tree := smt.ImportSparseMerkleTree(nodeStore, smtTreeHasher, root) + w.rootTree = &stateTree{ + name: RootTreeName, + tree: tree, + nodeStore: nodeStore, + } + + return w, nil +} + //////////////////////// // Actor Tree Helpers // //////////////////////// @@ -304,7 +385,6 @@ func (t *treeStore) updateActorsTree(actorType coreTypes.ActorType, actors []*co return err } } - return nil } diff --git a/persistence/trees/trees_test.go b/persistence/trees/trees_test.go index e59e3ba1f..aa8c41ab4 100644 --- a/persistence/trees/trees_test.go +++ b/persistence/trees/trees_test.go @@ -1,111 +1,205 @@ -package trees +package trees_test import ( - "fmt" + "encoding/hex" + "log" + "math/big" "testing" - "github.com/pokt-network/pocket/persistence/kvstore" - "github.com/pokt-network/smt" + "github.com/pokt-network/pocket/persistence" + "github.com/pokt-network/pocket/persistence/trees" + "github.com/pokt-network/pocket/runtime" + "github.com/pokt-network/pocket/runtime/configs" + "github.com/pokt-network/pocket/runtime/test_artifacts" + "github.com/pokt-network/pocket/runtime/test_artifacts/keygen" + core_types "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/crypto" + "github.com/pokt-network/pocket/shared/messaging" + "github.com/pokt-network/pocket/shared/modules" + "github.com/pokt-network/pocket/shared/utils" + "github.com/stretchr/testify/require" ) -// TECHDEBT(#836): Tests added in https://github.com/pokt-network/pocket/pull/836 +var ( + defaultChains = []string{"0001"} + defaultStakeBig = big.NewInt(1000000000000000) + defaultStake = utils.BigIntToString(defaultStakeBig) + defaultStakeStatus = int32(core_types.StakeStatus_Staked) + defaultPauseHeight = int64(-1) // pauseHeight=-1 implies not paused + defaultUnstakingHeight = int64(-1) // unstakingHeight=-1 implies not unstaking + + testSchema = "test_schema" + + genesisStateNumValidators = 5 + genesisStateNumServicers = 1 + genesisStateNumApplications = 1 +) + +const ( + treesHash1 = "5282ee91a3ec0a6f2b30e4780b369bae78c80ef3ea40587fef6ae263bf41f244" +) + func TestTreeStore_Update(t *testing.T) { - // TODO: Write test case for the Update method - t.Skip("TODO: Write test case for Update method") + pool, resource, dbUrl := test_artifacts.SetupPostgresDocker() + t.Cleanup(func() { + require.NoError(t, pool.Purge(resource)) + }) + + t.Run("should update actor trees, commit, and modify the state hash", func(t *testing.T) { + pmod := newTestPersistenceModule(t, dbUrl) + context := newTestPostgresContext(t, 0, pmod) + + require.NoError(t, context.SetSavePoint()) + + hash1, err := context.ComputeStateHash() + require.NoError(t, err) + require.NotEmpty(t, hash1) + require.Equal(t, hash1, treesHash1) + + _, err = createAndInsertDefaultTestApp(t, context) + require.NoError(t, err) + + require.NoError(t, context.SetSavePoint()) + + hash2, err := context.ComputeStateHash() + require.NoError(t, err) + require.NotEmpty(t, hash2) + require.NotEqual(t, hash1, hash2) + }) + + t.Run("should fail to rollback when no treestore savepoint is set", func(t *testing.T) { + pmod := newTestPersistenceModule(t, dbUrl) + context := newTestPostgresContext(t, 0, pmod) + + err := context.RollbackToSavePoint() + require.Error(t, err) + require.ErrorIs(t, err, trees.ErrFailedRollback) + }) } -func TestTreeStore_New(t *testing.T) { - // TODO: Write test case for the NewStateTrees function - t.Skip("TODO: Write test case for NewStateTrees function") +func newTestPersistenceModule(t *testing.T, databaseURL string) modules.PersistenceModule { + t.Helper() + teardownDeterministicKeygen := keygen.GetInstance().SetSeed(42) + defer teardownDeterministicKeygen() + + cfg := newTestDefaultConfig(t, databaseURL) + genesisState, _ := test_artifacts.NewGenesisState( + genesisStateNumValidators, + genesisStateNumServicers, + genesisStateNumApplications, + genesisStateNumServicers, + ) + + runtimeMgr := runtime.NewManager(cfg, genesisState) + + bus, err := runtime.CreateBus(runtimeMgr) + require.NoError(t, err) + + persistenceMod, err := persistence.Create(bus) + require.NoError(t, err) + + return persistenceMod.(modules.PersistenceModule) } -func TestTreeStore_DebugClearAll(t *testing.T) { - // TODO: Write test case for the DebugClearAll method - t.Skip("TODO: Write test case for DebugClearAll method") +// fetches a new default node configuration for testing +func newTestDefaultConfig(t *testing.T, databaseURL string) *configs.Config { + t.Helper() + cfg := &configs.Config{ + Persistence: &configs.PersistenceConfig{ + PostgresUrl: databaseURL, + NodeSchema: testSchema, + BlockStorePath: ":memory:", + TxIndexerPath: ":memory:", + TreesStoreDir: ":memory:", + MaxConnsCount: 5, + MinConnsCount: 1, + MaxConnLifetime: "5m", + MaxConnIdleTime: "1m", + HealthCheckPeriod: "30s", + }, + } + return cfg } +func createAndInsertDefaultTestApp(t *testing.T, db *persistence.PostgresContext) (*core_types.Actor, error) { + t.Helper() + app := newTestApp(t) -// TODO_AFTER(#861): Implement this test with the test suite available in #861 -func TestTreeStore_GetTreeHashes(t *testing.T) { - t.Skip("TODO: Write test case for GetTreeHashes method") // context: https://github.com/pokt-network/pocket/pull/915#discussion_r1267313664 + addrBz, err := hex.DecodeString(app.Address) + require.NoError(t, err) + + pubKeyBz, err := hex.DecodeString(app.PublicKey) + require.NoError(t, err) + + outputBz, err := hex.DecodeString(app.Output) + require.NoError(t, err) + return app, db.InsertApp( + addrBz, + pubKeyBz, + outputBz, + false, + defaultStakeStatus, + defaultStake, + defaultChains, + defaultPauseHeight, + defaultUnstakingHeight) } -func TestTreeStore_Prove(t *testing.T) { - nodeStore := kvstore.NewMemKVStore() - tree := smt.NewSparseMerkleTree(nodeStore, smtTreeHasher) - testTree := &stateTree{ - name: "test", - tree: tree, - nodeStore: nodeStore, - } +// TECHDEBT(#796): Test helpers should be consolidated in a single place +func newTestApp(t *testing.T) *core_types.Actor { + operatorKey, err := crypto.GeneratePublicKey() + require.NoError(t, err) - require.NoError(t, testTree.tree.Update([]byte("key"), []byte("value"))) - require.NoError(t, testTree.tree.Commit()) + outputAddr, err := crypto.GenerateAddress() + require.NoError(t, err) - treeStore := &treeStore{ - merkleTrees: make(map[string]*stateTree, 1), + return &core_types.Actor{ + Address: hex.EncodeToString(operatorKey.Address()), + PublicKey: hex.EncodeToString(operatorKey.Bytes()), + Chains: defaultChains, + StakedAmount: defaultStake, + PausedHeight: defaultPauseHeight, + UnstakingHeight: defaultUnstakingHeight, + Output: hex.EncodeToString(outputAddr), } - treeStore.merkleTrees["test"] = testTree - - testCases := []struct { - name string - treeName string - key []byte - value []byte - valid bool - expectedErr error - }{ - { - name: "valid inclusion proof: key and value in tree", - treeName: "test", - key: []byte("key"), - value: []byte("value"), - valid: true, - expectedErr: nil, - }, - { - name: "valid exclusion proof: key not in tree", - treeName: "test", - key: []byte("key2"), - value: nil, - valid: true, - expectedErr: nil, - }, - { - name: "invalid proof: tree not in store", - treeName: "unstored tree", - key: []byte("key"), - value: []byte("value"), - valid: false, - expectedErr: fmt.Errorf("tree not found: %s", "unstored tree"), - }, - { - name: "invalid inclusion proof: key in tree, wrong value", - treeName: "test", - key: []byte("key"), - value: []byte("wrong value"), - valid: false, - expectedErr: nil, - }, - { - name: "invalid exclusion proof: key in tree", - treeName: "test", - key: []byte("key"), - value: nil, - valid: false, - expectedErr: nil, - }, +} + +// TECHDEBT(#796): Test helpers should be consolidated in a single place +func newTestPostgresContext(t testing.TB, height int64, testPersistenceMod modules.PersistenceModule) *persistence.PostgresContext { + t.Helper() + rwCtx, err := testPersistenceMod.NewRWContext(height) + if err != nil { + log.Fatalf("Error creating new context: %v\n", err) } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - valid, err := treeStore.Prove(tc.treeName, tc.key, tc.value) - require.Equal(t, valid, tc.valid) - if tc.expectedErr == nil { - require.NoError(t, err) - return - } - require.ErrorAs(t, err, &tc.expectedErr) - }) + postgresCtx, ok := rwCtx.(*persistence.PostgresContext) + if !ok { + log.Fatalf("Error casting RW context to Postgres context") } + + // TECHDEBT: This should not be part of `NewTestPostgresContext`. It causes unnecessary resets + // if we call `NewTestPostgresContext` more than once in a single test. + t.Cleanup(func() { + resetStateToGenesis(testPersistenceMod) + }) + + return postgresCtx +} + +// This is necessary for unit tests that are dependant on a baseline genesis state +func resetStateToGenesis(m modules.PersistenceModule) { + if err := m.ReleaseWriteContext(); err != nil { + log.Fatalf("Error releasing write context: %v\n", err) + } + if err := m.HandleDebugMessage(&messaging.DebugMessage{ + Action: messaging.DebugMessageAction_DEBUG_PERSISTENCE_RESET_TO_GENESIS, + Message: nil, + }); err != nil { + log.Fatalf("Error clearing state: %v\n", err) + } +} + +// TODO_AFTER(#861): Implement this test with the test suite available in #861 +func TestTreeStore_GetTreeHashes(t *testing.T) { + t.Skip("TODO: Write test case for GetTreeHashes method") // context: https://github.com/pokt-network/pocket/pull/915#discussion_r1267313664 } diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index b3c1e56e2..38c7681ba 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -46,6 +46,13 @@ type PersistenceModule interface { GetLocalContext() (PersistenceLocalContext, error) } +// AtomicStore defines the interface for stores to implement to guarantee atomic commits to the persistence layer +type AtomicStore interface { + Savepoint() error + Commit() error + Rollback() error +} + // Interface defining the context within which the node can operate with the persistence layer. // Operations in the context of a PersistenceContext are isolated from other operations and // other persistence contexts until committed, enabling parallelizability along other operations. @@ -74,8 +81,8 @@ type PersistenceRWContext interface { // PersistenceWriteContext has no use-case independent of `PersistenceRWContext`, but is a useful abstraction type PersistenceWriteContext interface { // Context Operations - NewSavePoint([]byte) error - RollbackToSavePoint([]byte) error + SetSavePoint() error + RollbackToSavePoint() error Release() // Commits (and releases) the current context to disk (i.e. finality). diff --git a/shared/modules/treestore_module.go b/shared/modules/treestore_module.go index 35b240e51..a79f7a14f 100644 --- a/shared/modules/treestore_module.go +++ b/shared/modules/treestore_module.go @@ -21,13 +21,16 @@ type TreeStoreModule interface { Submodule treeStoreFactory - // Update returns the new state hash for a given height. + AtomicStore + + // Update returns the computed state hash for a given height. // * Height is passed through to the Update function and is used to query the TxIndexer for transactions // to update into the merkle tree set // * Passing a higher height will cause a change but repeatedly calling the same or a lower height will // not incur a change. // * By nature of it taking a pgx transaction at runtime, Update inherits the pgx transaction's read view of the // database. + // * Commit must be called after Update to persist any changes it made to disk. Update(pgtx pgx.Tx, height uint64) (string, error) // DebugClearAll completely clears the state of the trees. For debugging purposes only. DebugClearAll() error diff --git a/utility/unit_of_work/block.go b/utility/unit_of_work/block.go index 6076822b5..914cf1ac9 100644 --- a/utility/unit_of_work/block.go +++ b/utility/unit_of_work/block.go @@ -208,33 +208,18 @@ func (uow *baseUtilityUnitOfWork) prevBlockByzantineValidators() ([][]byte, erro return nil, nil } -// TODO: This has not been tested or investigated in detail -func (uow *baseUtilityUnitOfWork) revertLastSavePoint() coreTypes.Error { - // TODO(@deblasis): Implement this - // if len(u.savePointsSet) == 0 { - // return coreTypes.ErrEmptySavePoints() - // } - // var key []byte - // popIndex := len(u.savePointsList) - 1 - // key, u.savePointsList = u.savePointsList[popIndex], u.savePointsList[:popIndex] - // delete(u.savePointsSet, hex.EncodeToString(key)) - // if err := u.store.RollbackToSavePoint(key); err != nil { - // return coreTypes.ErrRollbackSavePoint(err) - // } +func (uow *baseUtilityUnitOfWork) revertToLastSavepoint() coreTypes.Error { + if err := uow.persistenceRWContext.RollbackToSavePoint(); err != nil { + uow.logger.Err(err).Msgf("failed to rollback to savepoint at height %d", uow.height) + return coreTypes.ErrRollbackSavePoint(err) + } return nil } -//nolint:unused // TODO: This has not been tested or investigated in detail -func (uow *baseUtilityUnitOfWork) newSavePoint(txHashBz []byte) coreTypes.Error { - // TODO(@deblasis): Implement this - // if err := u.store.NewSavePoint(txHashBz); err != nil { - // return coreTypes.ErrNewSavePoint(err) - // } - // txHash := hex.EncodeToString(txHashBz) - // if _, exists := u.savePointsSet[txHash]; exists { - // return coreTypes.ErrDuplicateSavePoint() - // } - // u.savePointsList = append(u.savePointsList, txHashBz) - // u.savePointsSet[txHash] = struct{}{} +func (uow *baseUtilityUnitOfWork) newSavePoint() coreTypes.Error { + if err := uow.persistenceRWContext.SetSavePoint(); err != nil { + uow.logger.Err(err).Msgf("failed to create new savepoint at height %d", uow.height) + return coreTypes.ErrNewSavePoint(err) + } return nil } diff --git a/utility/unit_of_work/module.go b/utility/unit_of_work/module.go index 2624e5bac..22547e090 100644 --- a/utility/unit_of_work/module.go +++ b/utility/unit_of_work/module.go @@ -1,12 +1,10 @@ package unit_of_work import ( - "fmt" - coreTypes "github.com/pokt-network/pocket/shared/core/types" - "github.com/pokt-network/pocket/shared/mempool" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" + "go.uber.org/multierr" ) const ( @@ -48,6 +46,7 @@ func (uow *baseUtilityUnitOfWork) SetProposalBlock(blockHash string, proposerAdd return nil } +// ApplyBlock atomically applies a block to the persistence layer for a given height. func (uow *baseUtilityUnitOfWork) ApplyBlock() error { log := uow.logger.With().Fields(map[string]interface{}{ "source": "ApplyBlock", @@ -58,51 +57,55 @@ func (uow *baseUtilityUnitOfWork) ApplyBlock() error { return coreTypes.ErrProposalBlockNotSet() } + // initialize a new savepoint before applying the block + if err := uow.newSavePoint(); err != nil { + return err + } + // begin block lifecycle phase log.Debug().Msg("calling beginBlock") if err := uow.beginBlock(); err != nil { return err } + // processProposalBlockTransactions indexes the transactions into the TxIndexer. + // If it fails, it returns an error which triggers a rollback below to undo the changes + // that processProposalBlockTransactions could have caused. log.Debug().Msg("processing transactions from proposal block") - txMempool := uow.GetBus().GetUtilityModule().GetMempool() - if err := uow.processProposalBlockTransactions(txMempool); err != nil { - return err + if err := uow.processProposalBlockTransactions(); err != nil { + rollErr := uow.revertToLastSavepoint() + return multierr.Combine(rollErr, err) } - // end block lifecycle phase + // end block lifecycle phase calls endBlock and reverts to the last known savepoint if it encounters any errors log.Debug().Msg("calling endBlock") if err := uow.endBlock(uow.proposalProposerAddr); err != nil { - return err + rollErr := uow.revertToLastSavepoint() + return multierr.Combine(rollErr, err) } + // return the app hash (consensus module will get the validator set directly) - log.Debug().Msg("computing state hash") stateHash, err := uow.persistenceRWContext.ComputeStateHash() if err != nil { - log.Fatal().Err(err).Bool("TODO", true).Msg("Updating the app hash failed. TODO: Look into roll-backing the entire commit...") - return coreTypes.ErrAppHash(err) + rollErr := uow.persistenceRWContext.RollbackToSavePoint() + return coreTypes.ErrAppHash(multierr.Append(err, rollErr)) } // IMPROVE(#655): this acts as a feature flag to allow tests to ignore the check if needed, ideally the tests should have a way to determine // the hash and set it into the proposal block it's currently hard to do because the state is different at every test run (non-determinism) if uow.proposalStateHash != IgnoreProposalBlockCheckHash { if uow.proposalStateHash != stateHash { - log.Fatal().Bool("TODO", true). - Str("proposalStateHash", uow.proposalStateHash). - Str("stateHash", stateHash). - Msg("State hash mismatch. TODO: Look into roll-backing the entire commit...") - return coreTypes.ErrAppHash(fmt.Errorf("state hash mismatch: expected %s from the proposal, got %s", uow.proposalStateHash, stateHash)) + return uow.revertToLastSavepoint() } } - log.Info().Str("state_hash", stateHash).Msgf("ApplyBlock succeeded!") + log.Info().Str("state_hash", stateHash).Msgf("🧱 ApplyBlock succeeded!") uow.stateHash = stateHash return nil } -// TODO(@deblasis): change tracking here func (uow *baseUtilityUnitOfWork) Commit(quorumCert []byte) error { uow.logger.Debug().Msg("committing the rwPersistenceContext...") if err := uow.persistenceRWContext.Commit(uow.proposalProposerAddr, quorumCert); err != nil { @@ -112,7 +115,6 @@ func (uow *baseUtilityUnitOfWork) Commit(quorumCert []byte) error { return nil } -// TODO(@deblasis): change tracking reset here func (uow *baseUtilityUnitOfWork) Release() error { rwCtx := uow.persistenceRWContext if rwCtx != nil { @@ -138,9 +140,10 @@ func (uow *baseUtilityUnitOfWork) isProposalBlockSet() bool { // processProposalBlockTransactions processes the transactions from the proposal block stored in the current // unit of work. It applies the transactions to the persistence context, indexes them, and removes that from // the mempool if they are present. -func (uow *baseUtilityUnitOfWork) processProposalBlockTransactions(txMempool mempool.TXMempool) (err error) { +func (uow *baseUtilityUnitOfWork) processProposalBlockTransactions() (err error) { // CONSIDERATION: should we check that `uow.proposalBlockTxs` is not nil and return an error if so or allow empty blocks? // For reference, see Tendermint: https://docs.tendermint.com/v0.34/tendermint-core/configuration.html#empty-blocks-vs-no-empty-blocks + txMempool := uow.GetBus().GetUtilityModule().GetMempool() for index, txProtoBytes := range uow.proposalBlockTxs { tx, err := coreTypes.TxFromBytes(txProtoBytes) if err != nil { diff --git a/utility/unit_of_work/uow_leader.go b/utility/unit_of_work/uow_leader.go index 2c10d76dc..cfa2e6707 100644 --- a/utility/unit_of_work/uow_leader.go +++ b/utility/unit_of_work/uow_leader.go @@ -2,6 +2,7 @@ package unit_of_work import ( "encoding/hex" + "fmt" "github.com/pokt-network/pocket/logger" coreTypes "github.com/pokt-network/pocket/shared/core/types" @@ -58,7 +59,11 @@ func (uow *leaderUtilityUnitOfWork) CreateProposalBlock(proposer []byte, maxTxBy // Compute & return the new state hash stateHash, err = uow.persistenceRWContext.ComputeStateHash() if err != nil { - log.Fatal().Err(err).Bool("TODO", true).Msg("Updating the app hash failed. TODO: Look into roll-backing the entire commit...") + if err := uow.persistenceRWContext.RollbackToSavePoint(); err != nil { + log.Error().Msgf("failed to recover from rollback at height %+v: %+v", uow.height, err) + return "", nil, err + } + return "", nil, fmt.Errorf("rollback at height %d: failed to compute state hash: %w", uow.height, err) } log.Info().Str("state_hash", stateHash).Msg("Finished successfully") @@ -99,7 +104,7 @@ func (uow *leaderUtilityUnitOfWork) reapMempool(txMempool mempool.TXMempool, max if err != nil { uow.logger.Err(err).Msg("Error handling the transaction") // TODO(#327): Properly implement 'unhappy path' for save points - if err := uow.revertLastSavePoint(); err != nil { + if err := uow.revertToLastSavepoint(); err != nil { return nil, err } txsTotalBz -= txBzSize diff --git a/utility/unit_of_work/uow_leader_test.go b/utility/unit_of_work/uow_leader_test.go new file mode 100644 index 000000000..feef2f37d --- /dev/null +++ b/utility/unit_of_work/uow_leader_test.go @@ -0,0 +1,129 @@ +package unit_of_work + +import ( + "fmt" + "math/big" + "reflect" + "testing" + + "github.com/golang/mock/gomock" + "github.com/pokt-network/pocket/shared/modules" + mockModules "github.com/pokt-network/pocket/shared/modules/mocks" + "github.com/pokt-network/pocket/shared/utils" + "github.com/stretchr/testify/require" +) + +var DefaultStakeBig = big.NewInt(1000000000000000) + +func Test_leaderUtilityUnitOfWork_CreateProposalBlock(t *testing.T) { + t.Helper() + + type fields struct { + leaderUOW func(t *testing.T) *leaderUtilityUnitOfWork + } + type args struct { + proposer []byte + maxTxBytes uint64 + } + tests := []struct { + name string + args args + fields fields + wantStateHash string + wantTxs [][]byte + wantErr bool + }{ + { + name: "should revert a failed block proposal", + args: args{}, + fields: fields{ + leaderUOW: func(t *testing.T) *leaderUtilityUnitOfWork { + ctrl := gomock.NewController(t) + + mockrwcontext := newDefaultMockRWContext(t, ctrl) + mockrwcontext.EXPECT().RollbackToSavePoint().Times(1) + mockrwcontext.EXPECT().ComputeStateHash().Return("", fmt.Errorf("rollback error")) + + mockUtilityMod := newDefaultMockUtilityModule(t, ctrl) + mockbus := mockModules.NewMockBus(ctrl) + mockbus.EXPECT().GetUtilityModule().Return(mockUtilityMod).AnyTimes() + + luow := NewLeaderUOW(0, mockrwcontext, mockrwcontext) + luow.SetBus(mockbus) + + return luow + }, + }, + wantErr: true, + wantTxs: nil, + }, + { + name: "should apply a unit of work", + args: args{}, + fields: fields{ + leaderUOW: func(t *testing.T) *leaderUtilityUnitOfWork { + ctrl := gomock.NewController(t) + + mockrwcontext := newDefaultMockRWContext(t, ctrl) + mockrwcontext.EXPECT().ComputeStateHash().Return("foo", nil).Times(1) + + mockUtilityMod := newDefaultMockUtilityModule(t, ctrl) + mockbus := mockModules.NewMockBus(ctrl) + mockbus.EXPECT().GetUtilityModule().Return(mockUtilityMod).AnyTimes() + + luow := NewLeaderUOW(0, mockrwcontext, mockrwcontext) + luow.SetBus(mockbus) + + return luow + }, + }, + wantErr: false, + wantStateHash: "foo", + wantTxs: [][]byte{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + luow := tt.fields.leaderUOW(t) + gotHash, gotTxs, err := luow.CreateProposalBlock(tt.args.proposer, tt.args.maxTxBytes) + if (err != nil) != tt.wantErr { + t.Errorf("leaderUtilityUnitOfWork.CreateProposalBlock() error = %v, wantErr %v", err, tt.wantErr) + return + } + require.Equal(t, gotHash, tt.wantStateHash) + if !reflect.DeepEqual(gotTxs, tt.wantTxs) { + t.Errorf("leaderUtilityUnitOfWork.CreateProposalBlock() gotTxs = %v, want %v", gotTxs, tt.wantTxs) + } + }) + } +} + +func newDefaultMockRWContext(t *testing.T, ctrl *gomock.Controller) *mockModules.MockPersistenceRWContext { + t.Helper() + + mockrwcontext := mockModules.NewMockPersistenceRWContext(ctrl) + mockrwcontext.EXPECT().SetPoolAmount(gomock.Any(), gomock.Any()).AnyTimes() + mockrwcontext.EXPECT().GetIntParam(gomock.Any(), gomock.Any()).Return(0, nil).AnyTimes() + mockrwcontext.EXPECT().GetPoolAmount(gomock.Any(), gomock.Any()).Return(utils.BigIntToString(DefaultStakeBig), nil).Times(1) + mockrwcontext.EXPECT().AddAccountAmount(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockrwcontext.EXPECT().AddPoolAmount(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockrwcontext.EXPECT().GetAppsReadyToUnstake(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + mockrwcontext.EXPECT().GetServicersReadyToUnstake(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + mockrwcontext.EXPECT().GetValidatorsReadyToUnstake(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + mockrwcontext.EXPECT().GetFishermenReadyToUnstake(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + mockrwcontext.EXPECT().SetServicerStatusAndUnstakingHeightIfPausedBefore(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockrwcontext.EXPECT().SetAppStatusAndUnstakingHeightIfPausedBefore(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockrwcontext.EXPECT().SetValidatorsStatusAndUnstakingHeightIfPausedBefore(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockrwcontext.EXPECT().SetFishermanStatusAndUnstakingHeightIfPausedBefore(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + return mockrwcontext +} + +func newDefaultMockUtilityModule(t *testing.T, ctrl *gomock.Controller) *mockModules.MockUtilityModule { + mockUtilityMod := mockModules.NewMockUtilityModule(ctrl) + testmempool := NewTestingMempool(t) + mockUtilityMod.EXPECT().GetModuleName().Return(modules.UtilityModuleName).AnyTimes() + mockUtilityMod.EXPECT().SetBus(gomock.Any()).Return().AnyTimes() + mockUtilityMod.EXPECT().GetMempool().Return(testmempool).AnyTimes() + return mockUtilityMod +} From a68af5cfd1d6edf018108c972c3cd53559bbaba8 Mon Sep 17 00:00:00 2001 From: d7t Date: Mon, 31 Jul 2023 12:25:09 -0600 Subject: [PATCH 18/25] [chore] Replaces multierr usage with go native errors package (#939) Co-authored-by: Daniel Olshansky --- go.mod | 2 +- p2p/background/router.go | 4 ++-- p2p/config/config.go | 18 +++++++++--------- p2p/module.go | 7 +++---- .../peerstore_provider/peerstore_provider.go | 7 ++++--- p2p/utils/host.go | 6 +++--- persistence/context.go | 3 +-- shared/node.go | 8 +++----- utility/unit_of_work/module.go | 9 +++++---- 9 files changed, 31 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index e9772889b..b5029a9ce 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,6 @@ require ( github.com/rs/zerolog v1.27.0 github.com/spf13/cobra v1.6.0 github.com/spf13/viper v1.13.0 - go.uber.org/multierr v1.9.0 golang.org/x/term v0.5.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.26.1 @@ -197,6 +196,7 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.15.0 // indirect go.uber.org/fx v1.18.2 // indirect + go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect diff --git a/p2p/background/router.go b/p2p/background/router.go index 7899f7817..e79c9c79e 100644 --- a/p2p/background/router.go +++ b/p2p/background/router.go @@ -4,6 +4,7 @@ package background import ( "context" + "errors" "fmt" "time" @@ -11,7 +12,6 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" libp2pHost "github.com/libp2p/go-libp2p/core/host" libp2pPeer "github.com/libp2p/go-libp2p/core/peer" - "go.uber.org/multierr" "google.golang.org/protobuf/proto" "github.com/pokt-network/pocket/logger" @@ -129,7 +129,7 @@ func (rtr *backgroundRouter) Close() error { topicCloseErr = err } - return multierr.Append( + return errors.Join( topicCloseErr, rtr.kadDHT.Close(), ) diff --git a/p2p/config/config.go b/p2p/config/config.go index 90350444b..690a72be0 100644 --- a/p2p/config/config.go +++ b/p2p/config/config.go @@ -1,11 +1,11 @@ package config import ( + "errors" "fmt" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/protocol" - "go.uber.org/multierr" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/shared/crypto" @@ -56,15 +56,15 @@ type RainTreeConfig struct { // IsValid implements the respective member of the `RouterConfig` interface. func (cfg *baseConfig) IsValid() (err error) { if cfg.Addr == nil { - err = multierr.Append(err, fmt.Errorf("pokt address not configured")) + err = errors.Join(err, fmt.Errorf("pokt address not configured")) } if cfg.Host == nil { - err = multierr.Append(err, fmt.Errorf("host not configured")) + err = errors.Join(err, fmt.Errorf("host not configured")) } if cfg.Handler == nil { - err = multierr.Append(err, fmt.Errorf("handler not configured")) + err = errors.Join(err, fmt.Errorf("handler not configured")) } return err } @@ -72,23 +72,23 @@ func (cfg *baseConfig) IsValid() (err error) { // IsValid implements the respective member of the `RouterConfig` interface. func (cfg *UnicastRouterConfig) IsValid() (err error) { if cfg.Logger == nil { - err = multierr.Append(err, fmt.Errorf("logger not configured")) + err = errors.Join(err, fmt.Errorf("logger not configured")) } if cfg.Host == nil { - err = multierr.Append(err, fmt.Errorf("host not configured")) + err = errors.Join(err, fmt.Errorf("host not configured")) } if cfg.ProtocolID == "" { - err = multierr.Append(err, fmt.Errorf("protocol id not configured")) + err = errors.Join(err, fmt.Errorf("protocol id not configured")) } if cfg.MessageHandler == nil { - err = multierr.Append(err, fmt.Errorf("message handler not configured")) + err = errors.Join(err, fmt.Errorf("message handler not configured")) } if cfg.PeerHandler == nil { - err = multierr.Append(err, fmt.Errorf("peer handler not configured")) + err = errors.Join(err, fmt.Errorf("peer handler not configured")) } return err } diff --git a/p2p/module.go b/p2p/module.go index 6bb8f479a..4e130bb10 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -8,7 +8,6 @@ import ( "github.com/libp2p/go-libp2p" libp2pHost "github.com/libp2p/go-libp2p/core/host" "github.com/multiformats/go-multiaddr" - "go.uber.org/multierr" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -182,12 +181,12 @@ func (m *p2pModule) Stop() error { stakedActorRouterCloseErr = m.stakedActorRouter.Close() } - routerCloseErrs := multierr.Append( + routerCloseErrs := errors.Join( m.unstakedActorRouter.Close(), stakedActorRouterCloseErr, ) - err := multierr.Append( + err := errors.Join( routerCloseErrs, m.host.Close(), ) @@ -231,7 +230,7 @@ func (m *p2pModule) Broadcast(msg *anypb.Any) error { unstakedBroadcastErr := m.unstakedActorRouter.Broadcast(poktEnvelopeBz) - return multierr.Append(stakedBroadcastErr, unstakedBroadcastErr) + return errors.Join(stakedBroadcastErr, unstakedBroadcastErr) } func (m *p2pModule) Send(addr cryptoPocket.Address, msg *anypb.Any) error { diff --git a/p2p/providers/peerstore_provider/peerstore_provider.go b/p2p/providers/peerstore_provider/peerstore_provider.go index bbf57746a..e7bb686d7 100644 --- a/p2p/providers/peerstore_provider/peerstore_provider.go +++ b/p2p/providers/peerstore_provider/peerstore_provider.go @@ -3,12 +3,13 @@ package peerstore_provider //go:generate mockgen -package=mock_types -destination=../../types/mocks/peerstore_provider_mock.go github.com/pokt-network/pocket/p2p/providers/peerstore_provider PeerstoreProvider import ( + "errors" + "github.com/pokt-network/pocket/logger" typesP2P "github.com/pokt-network/pocket/p2p/types" coreTypes "github.com/pokt-network/pocket/shared/core/types" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/modules" - "go.uber.org/multierr" ) const PeerstoreProviderSubmoduleName = "peerstore_provider" @@ -35,12 +36,12 @@ func ActorsToPeerstore(abp PeerstoreProvider, actors []*coreTypes.Actor) (pstore logger.Global.Warn().Err(err).Msg("ignoring ErrResolvingAddr - peer unreachable, not adding it to peerstore") continue } else if err != nil { - errs = multierr.Append(errs, err) + errs = errors.Join(errs, err) continue } if err = pstore.AddPeer(networkPeer); err != nil { - errs = multierr.Append(errs, err) + errs = errors.Join(errs, err) } } return pstore, errs diff --git a/p2p/utils/host.go b/p2p/utils/host.go index 3597856b7..e53e0d246 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -2,12 +2,12 @@ package utils import ( "context" + "errors" "fmt" "time" libp2pHost "github.com/libp2p/go-libp2p/core/host" libp2pProtocol "github.com/libp2p/go-libp2p/core/protocol" - "go.uber.org/multierr" "github.com/pokt-network/pocket/logger" typesP2P "github.com/pokt-network/pocket/p2p/types" @@ -26,7 +26,7 @@ const ( func PopulateLibp2pHost(host libp2pHost.Host, pstore typesP2P.Peerstore) (err error) { for _, peer := range pstore.GetPeerList() { if addErr := AddPeerToLibp2pHost(host, peer); addErr != nil { - err = multierr.Append(err, addErr) + err = errors.Join(err, addErr) } } return err @@ -101,7 +101,7 @@ func Libp2pSendToPeer(host libp2pHost.Host, protocolID libp2pProtocol.ID, data [ } if n, err := stream.Write(data); err != nil { - return multierr.Append( + return errors.Join( fmt.Errorf("writing to stream: %w", err), stream.Reset(), ) diff --git a/persistence/context.go b/persistence/context.go index 15a89dfe1..c91560086 100644 --- a/persistence/context.go +++ b/persistence/context.go @@ -12,7 +12,6 @@ import ( "github.com/pokt-network/pocket/persistence/indexer" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" - "go.uber.org/multierr" ) var _ modules.PersistenceRWContext = &PostgresContext{} @@ -50,7 +49,7 @@ func (p *PostgresContext) RollbackToSavePoint() error { ctx, _ := p.getCtxAndTx() pgErr := p.tx.Rollback(ctx) treesErr := p.stateTrees.Rollback() - return multierr.Combine(pgErr, treesErr) + return errors.Join(pgErr, treesErr) } // Full details in the thread from the PR review: https://github.com/pokt-network/pocket/pull/285#discussion_r1018471719 diff --git a/shared/node.go b/shared/node.go index f1e842382..39b905e38 100644 --- a/shared/node.go +++ b/shared/node.go @@ -2,6 +2,7 @@ package shared import ( "context" + "errors" "time" "github.com/pokt-network/pocket/consensus" @@ -17,7 +18,6 @@ import ( "github.com/pokt-network/pocket/state_machine" "github.com/pokt-network/pocket/telemetry" "github.com/pokt-network/pocket/utility" - "go.uber.org/multierr" ) const ( @@ -185,13 +185,11 @@ func (node *Node) handleEvent(message *messaging.PocketEnvelope) error { case messaging.ConsensusNewHeightEventType: err_p2p := node.GetBus().GetP2PModule().HandleEvent(message.Content) err_ibc := node.GetBus().GetIBCModule().HandleEvent(message.Content) - // TODO: Remove this lib once we move to Go 1.2 - return multierr.Combine(err_p2p, err_ibc) + return errors.Join(err_p2p, err_ibc) case messaging.StateMachineTransitionEventType: err_consensus := node.GetBus().GetConsensusModule().HandleEvent(message.Content) err_p2p := node.GetBus().GetP2PModule().HandleEvent(message.Content) - // TODO: Remove this lib once we move to Go 1.2 - return multierr.Combine(err_consensus, err_p2p) + return errors.Join(err_consensus, err_p2p) default: logger.Global.Warn().Msgf("Unsupported message content type: %s", contentType) } diff --git a/utility/unit_of_work/module.go b/utility/unit_of_work/module.go index 22547e090..a654218ac 100644 --- a/utility/unit_of_work/module.go +++ b/utility/unit_of_work/module.go @@ -1,10 +1,11 @@ package unit_of_work import ( + "errors" + coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" - "go.uber.org/multierr" ) const ( @@ -74,21 +75,21 @@ func (uow *baseUtilityUnitOfWork) ApplyBlock() error { log.Debug().Msg("processing transactions from proposal block") if err := uow.processProposalBlockTransactions(); err != nil { rollErr := uow.revertToLastSavepoint() - return multierr.Combine(rollErr, err) + return errors.Join(rollErr, err) } // end block lifecycle phase calls endBlock and reverts to the last known savepoint if it encounters any errors log.Debug().Msg("calling endBlock") if err := uow.endBlock(uow.proposalProposerAddr); err != nil { rollErr := uow.revertToLastSavepoint() - return multierr.Combine(rollErr, err) + return errors.Join(rollErr, err) } // return the app hash (consensus module will get the validator set directly) stateHash, err := uow.persistenceRWContext.ComputeStateHash() if err != nil { rollErr := uow.persistenceRWContext.RollbackToSavePoint() - return coreTypes.ErrAppHash(multierr.Append(err, rollErr)) + return coreTypes.ErrAppHash(errors.Join(err, rollErr)) } // IMPROVE(#655): this acts as a feature flag to allow tests to ignore the check if needed, ideally the tests should have a way to determine From 09415496238f59833c57ba3dab3c9f9f8a3ea708 Mon Sep 17 00:00:00 2001 From: BigBoss Date: Mon, 31 Jul 2023 15:42:02 -0700 Subject: [PATCH 19/25] =?UTF-8?q?hack:=20=F0=9F=98=B4=20sleep=20enough=20f?= =?UTF-8?q?or=20cli=20debug=20subcommands=20to=20broadcast=20(#954)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniel Olshansky --- app/client/cli/debug.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index 99d5b83de..4cc1ea632 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -2,6 +2,7 @@ package cli import ( "os" + "time" "github.com/manifoldco/promptui" "github.com/spf13/cobra" @@ -50,7 +51,10 @@ func newDebugUISubCommands() []*cobra.Command { Use: promptItem, PersistentPreRunE: helpers.P2PDependenciesPreRunE, Run: func(cmd *cobra.Command, _ []string) { + // TECHDEBT(#874): this is a magic number, but an alternative would be to have the p2p module wait until connections are open and to flush the message correctly + time.Sleep(500 * time.Millisecond) // give p2p module time to start handleSelect(cmd, cmd.Use) + time.Sleep(500 * time.Millisecond) // give p2p module time to broadcast }, ValidArgs: items, } @@ -61,7 +65,7 @@ func newDebugUISubCommands() []*cobra.Command { // newDebugUICommand returns the cobra CLI for the Debug UI interface. func newDebugUICommand() *cobra.Command { return &cobra.Command{ - Aliases: []string{"dui"}, + Aliases: []string{"dui", "debug"}, Use: "DebugUI", Short: "Debug selection ui for rapid development", Args: cobra.MaximumNArgs(0), @@ -154,7 +158,7 @@ func handleSelect(cmd *cobra.Command, selection string) { } broadcastDebugMessage(cmd, m) default: - logger.Global.Error().Msg("Selection not yet implemented...") + logger.Global.Error().Str("selection", selection).Msg("Selection not yet implemented...") } } From 50f88462e791841f2456ba0de53c58a33dc9e196 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 31 Jul 2023 18:43:58 -0700 Subject: [PATCH 20/25] DevLog 12 (#957) DevLog12 (iteration 21) update. --- docs/devlog/devlog12.md | 88 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 docs/devlog/devlog12.md diff --git a/docs/devlog/devlog12.md b/docs/devlog/devlog12.md new file mode 100644 index 000000000..66865fab4 --- /dev/null +++ b/docs/devlog/devlog12.md @@ -0,0 +1,88 @@ +# Pocket V1 DevLog #12 + +**Date Published**: July 31st, 2023 + +We have kept the goals and details in this document short, but feel free to reach out to @Olshansk in the [core-dev-chat](https://discord.com/channels/553741558869131266/986789914379186226) for additional details, links & resources. + +## Table of Contents + +- [Iteration 21 Goals \& Results](#iteration-21-goals--results) + - [V0](#v0) + - [V1](#v1) + - [Utility - E2E Trustless Relay Demo](#utility---e2e-trustless-relay-demo) + - [Infrastructure - DevNet Dashboard](#infrastructure---devnet-dashboard) + - [Product - TestNet MVP](#product---testnet-mvp) +- [Screenshots](#screenshots) + - [Iteration 21 - Completed](#iteration-21---completed) + - [V1 Results](#v1-results) + - [Iteration 21 - Planned](#iteration-21---planned) +- [Contribute to V1 🧑‍💻](#contribute-to-v1-) + - [Links \& References](#links--references) + +## Iteration 21 Goals & Results + +**Iterate Dates**: July 15th - July 28th, 2023 + +```bash +# V1 Repo +git diff b55b6f96ca99a1e28ac133689949afa5f7e74c42 --stat +# 98 files changed, 3366 insertions(+), 655 deletions(-) +``` + +### V0 + +We have prepared the [release plan](https://www.notion.so/pocketnetwork/RC-0-10-4-Release-Plan-848c0c329e554a78a2aaf05bcaafb763?pvs=4) for `RC 0.10.4` with the helpful of everyone in the community! + +### V1 + +Our goal, for the second iteration in a row, was **to finalize demos** as much as possible from the [previous iteration](https://github.com/pokt-network/pocket/blob/main/docs/devlog/devlog11.md). + +🟡🟢 Like last week, we made progress on two more demos and give ourselves an overall score of `7/10` as we are tying together lots of lose ends. + +#### Utility - E2E Trustless Relay Demo + +@adshmh presented the first demo of an E2E Trustless Relay Demo + +[Audio](https://drive.google.com/file/d/1bkrIPsDAuZYevJRgyiudml5YWUSUtTto/view?usp=drive_link) + +[![asciicast](https://asciinema.org/a/599295.svg)](https://asciinema.org/a/599295) + +#### Infrastructure - DevNet Dashboard + +[Audio](https://drive.google.com/file/d/1rN2tXJ5qYXxmpnrU1Eo1dj_DRJjYJlDY/view?usp=drive_link] + +Though it may be offline at the time of writing, you can access the DevNet Dashboard [here](https://devnet-first-dashboard.dev-us-east4-1.poktnodes.network:8443/) + +![DevNet Dashboard](https://github.com/pokt-network/pocket/assets/1892194/98a57a86-26a6-4d08-a719-9f40dadcd658) + +#### Product - TestNet MVP + +@mokn himself presented his vision + +[![TestNet MVP](https://github.com/pokt-network/pocket/assets/1892194/97eee0a7-2755-4b56-979b-783ac9c5b0a9)](https://drive.google.com/file/d/1ojkUv6Ds_GTGAdxtMdYujFpOnDgoXYII/view) + +## Screenshots + +Please note that everything that was not `Done` in `iteration21` is moving over to `iteration22`. + +### Iteration 21 - Completed + +#### V1 Results + +![V1 Completed - 1](https://github.com/pokt-network/pocket/assets/1892194/776d5b75-0de6-43d3-800f-c7dddb04dbf3) +![V1 Completed - 2](https://github.com/pokt-network/pocket/assets/1892194/9940893f-b1b5-432a-ae9c-d949e540e739) + +### Iteration 21 - Planned + +![V1 Planned](https://github.com/pokt-network/pocket/assets/1892194/a1ac5624-b4a4-4d94-8812-615d8fe8d0e2) + +## Contribute to V1 🧑‍💻 + +### Links & References + +- [V1 Specifications](https://github.com/pokt-network/pocket-network-protocol) +- [V1 Repo](https://github.com/pokt-network/pocket) +- [V1 Wiki](https://github.com/pokt-network/pocket/wiki) +- [V1 Project Dashboard](https://github.com/pokt-network/pocket/projects?query=is%3Aopen) + + From e0e9fd4a981f9649c3df3caa1d2822231b86f504 Mon Sep 17 00:00:00 2001 From: Arash <23505281+adshmh@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:10:23 -0400 Subject: [PATCH 21/25] [Utility] servicer signs relays (#952) ## Description This PR modifies servicer code to sign all the served relays. It also modified the servicer configuration to a) provide the required private key, and b) remove public key and address fields which are now driven from the private key. ## Issue Fixes #832 ## Type of change - [x] New feature, functionality or library - [ ] Bug fix - [ ] Code health or cleanup - [ ] Major breaking change - [ ] Documentation - [ ] Other ## List of changes - Updaed servicer config proto file. - Updated servicer module - Updated unit tests covering servicer module ## Testing - [x] `make develop_test`; if any code changes were made - [ ] `make test_e2e` on [k8s LocalNet](https://github.com/pokt-network/pocket/blob/main/build/localnet/README.md); if any code changes were made - [ ] `e2e-devnet-test` passes tests on [DevNet](https://pocketnetwork.notion.site/How-to-DevNet-ff1598f27efe44c09f34e2aa0051f0dd); if any code was changed - [ ] [Docker Compose LocalNet](https://github.com/pokt-network/pocket/blob/main/docs/development/README.md); if any major functionality was changed or introduced - [ ] [k8s LocalNet](https://github.com/pokt-network/pocket/blob/main/build/localnet/README.md); if any infrastructure or configuration changes were made ## Required Checklist - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have added, or updated, [`godoc` format comments](https://go.dev/blog/godoc) on touched members (see: [tip.golang.org/doc/comment](https://tip.golang.org/doc/comment)) - [x] I have tested my changes using the available tooling ### If Applicable Checklist - [ ] I have updated the corresponding README(s); local and/or global - [x] I have added tests that prove my fix is effective or that my feature works - [ ] I have added, or updated, [mermaid.js](https://mermaid-js.github.io) diagrams in the corresponding README(s) - [ ] I have added, or updated, documentation and [mermaid.js](https://mermaid-js.github.io) diagrams in `shared/docs/*` if I updated `shared/*`README(s) --- runtime/configs/proto/servicer_config.proto | 7 +- utility/module_enable_actors_test.go | 14 +++- utility/servicer/module.go | 46 +++++++++--- utility/servicer/module_test.go | 82 ++++++++++++++++++--- 4 files changed, 122 insertions(+), 27 deletions(-) diff --git a/runtime/configs/proto/servicer_config.proto b/runtime/configs/proto/servicer_config.proto index 0a3bc65bf..916495fec 100644 --- a/runtime/configs/proto/servicer_config.proto +++ b/runtime/configs/proto/servicer_config.proto @@ -10,16 +10,15 @@ option go_package = "github.com/pokt-network/pocket/runtime/configs"; message ServicerConfig { // Enabled defines whether or not the node is a servicer. bool enabled = 1; - string public_key = 2; - string address = 3; - map services = 4; + string private_key = 2; + map services = 3; // relay_mining_volume_accuracy is a parameter used to adjust the calculated number of service tokens for an application. // It is introduced to minimize the chance of under-utilization of application's tokens, while removing the overhead of // communication between servicers which would be necessary otherwise. // See the following for more details: // https://arxiv.org/abs/2305.10672 - double relay_mining_volume_accuracy = 5; + double relay_mining_volume_accuracy = 4; } // ServiceConfig holds configurations related to where/how the application/client can access the backing RPC service. It is analogous to "ChainConfig" in v0 but can support any RPC service. diff --git a/utility/module_enable_actors_test.go b/utility/module_enable_actors_test.go index b2879bd2e..eadeb8f95 100644 --- a/utility/module_enable_actors_test.go +++ b/utility/module_enable_actors_test.go @@ -6,6 +6,7 @@ import ( "github.com/golang/mock/gomock" "github.com/pokt-network/pocket/runtime" "github.com/pokt-network/pocket/runtime/configs" + "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/modules" mocks "github.com/pokt-network/pocket/shared/modules/mocks" "github.com/stretchr/testify/assert" @@ -13,6 +14,9 @@ import ( ) func TestEnableActorModules(t *testing.T) { + privateKey, err := crypto.GeneratePrivateKey() + require.NoError(t, err) + tests := []struct { name string config *configs.Config @@ -24,7 +28,10 @@ func TestEnableActorModules(t *testing.T) { { name: "servicer only", config: &configs.Config{ - Servicer: &configs.ServicerConfig{Enabled: true}, + Servicer: &configs.ServicerConfig{ + Enabled: true, + PrivateKey: privateKey.String(), + }, }, expectedNames: []string{"servicer"}, }, @@ -46,7 +53,10 @@ func TestEnableActorModules(t *testing.T) { name: "validator and servicer", config: &configs.Config{ Validator: &configs.ValidatorConfig{Enabled: true}, - Servicer: &configs.ServicerConfig{Enabled: true}, + Servicer: &configs.ServicerConfig{ + Enabled: true, + PrivateKey: privateKey.String(), + }, }, expectedNames: []string{"validator", "servicer"}, }, diff --git a/utility/servicer/module.go b/utility/servicer/module.go index dcd4c48f1..18871633d 100644 --- a/utility/servicer/module.go +++ b/utility/servicer/module.go @@ -17,7 +17,7 @@ import ( "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" - "github.com/pokt-network/pocket/shared/crypto" + cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" "github.com/pokt-network/pocket/shared/utils" @@ -60,6 +60,13 @@ type servicer struct { // totalTokens is a mapping from application public keys to session metadata to keep track of session tokens // OPTIMIZE: There is an opportunity to simplify the code through various means such as, but not limited to, avoiding extra math.big operations or excess GetParam calls totalTokens map[string]*sessionTokens + + // private key of the servicer, used to sign the served relays. It is parsed from the private key provided in the servicer's configuration. + privateKey cryptoPocket.Ed25519PrivateKey + // address of the servicer, calculated from the provided private key. + address string + // public key of the servicer, calculated from the provided private key. + publicKey string } var ( @@ -90,6 +97,15 @@ func (*servicer) Create(bus modules.Bus, options ...modules.ModuleOption) (modul cfg := bus.GetRuntimeMgr().GetConfig() s.config = cfg.Servicer + privateKey, err := cryptoPocket.NewPrivateKey(cfg.Servicer.PrivateKey) + if err != nil { + return nil, err + } + + s.privateKey = privateKey.(cryptoPocket.Ed25519PrivateKey) + s.address = s.privateKey.Address().String() + s.publicKey = s.privateKey.PublicKey().String() + return s, nil } @@ -161,8 +177,12 @@ func (s *servicer) isRelayVolumeApplicable(session *coreTypes.Session, relay *co return nil, nil, false, fmt.Errorf("Error marshalling relay and/or response: %w", err) } - relayDigest := crypto.SHA3Hash(relayReqResBytes) - signedDigest := s.sign(relayDigest) + relayDigest := cryptoPocket.SHA3Hash(relayReqResBytes) + signedDigest, err := s.sign(relayDigest) + if err != nil { + return nil, relayReqResBytes, false, fmt.Errorf("Error checking volume applicability for relay in session %s: %w", session.Id, err) + } + response.ServicerSignature = hex.EncodeToString(signedDigest) collision, err := s.isRelayVolumeApplicableOnChain(session, relayDigest) if err != nil { @@ -173,9 +193,13 @@ func (s *servicer) isRelayVolumeApplicable(session *coreTypes.Session, relay *co return signedDigest, relayReqResBytes, collision, nil } -// INCOMPLETE(#832): provide a private key to the servicer and use it to sign all relays -func (s *servicer) sign(bz []byte) []byte { - return bz +// sign uses the servicer's private key, provided through configuration, to sign all relay digests. +func (s *servicer) sign(bz []byte) ([]byte, error) { + signature, err := s.privateKey.Sign(bz) + if err != nil { + return nil, fmt.Errorf("Error signing message: %w", err) + } + return signature, nil } // INCOMPLETE: implement this according to the comment below @@ -218,7 +242,7 @@ func (s *servicer) validateRelayMeta(meta *coreTypes.RelayMeta, currentHeight in func (s *servicer) validateRelayChainSupport(relayChain *coreTypes.Identifiable, currentHeight int64) error { if _, ok := s.config.Services[relayChain.Id]; !ok { - return fmt.Errorf("service %s not supported by servicer %s configuration", relayChain.Id, s.config.Address) + return fmt.Errorf("service %s not supported by servicer %s configuration", relayChain.Id, s.address) } // DISCUSS: either update NewReadContext to take a uint64, or the GetCurrentHeight to return an int64. @@ -229,13 +253,13 @@ func (s *servicer) validateRelayChainSupport(relayChain *coreTypes.Identifiable, defer readCtx.Release() //nolint:errcheck // We only need to make sure the readCtx is released // DISCUSS: should we update the GetServicer signature to take a string instead? - servicer, err := readCtx.GetServicer([]byte(s.config.Address), currentHeight) + servicer, err := readCtx.GetServicer([]byte(s.address), currentHeight) if err != nil { return fmt.Errorf("error reading servicer from persistence: %w", err) } if !slices.Contains(servicer.Chains, relayChain.Id) { - return fmt.Errorf("chain %s not supported by servicer %s configuration fetched from persistence", relayChain.Id, s.config.Address) + return fmt.Errorf("chain %s not supported by servicer %s configuration fetched from persistence", relayChain.Id, s.address) } return nil @@ -322,8 +346,8 @@ func (s *servicer) setAppSessionTokens(session *coreTypes.Session, tokens *sessi // validateServicer makes sure the servicer is A) active in the current session, and B) has not served more than its allocated relays for the session func (s *servicer) validateServicer(meta *coreTypes.RelayMeta, session *coreTypes.Session) error { - if meta.ServicerPublicKey != s.config.PublicKey { - return fmt.Errorf("relay servicer key %s does not match this servicer instance %s", meta.ServicerPublicKey, s.config.PublicKey) + if meta.ServicerPublicKey != s.publicKey { + return fmt.Errorf("relay servicer key %s does not match this servicer instance %s", meta.ServicerPublicKey, s.publicKey) } var found bool diff --git a/utility/servicer/module_test.go b/utility/servicer/module_test.go index 6ab5a5646..1e6f90e0c 100644 --- a/utility/servicer/module_test.go +++ b/utility/servicer/module_test.go @@ -29,7 +29,8 @@ const ( var ( // Initialized in TestMain - testServicer1 *coreTypes.Actor + testServicer1 *coreTypes.Actor + testServicer1PrivateKey crypto.PrivateKey // Initialized in TestMain testApp1 *coreTypes.Actor @@ -50,11 +51,16 @@ func testPublicKey() (publicKey, address string) { // TestMain initialized the test fixtures for all the unit tests in the servicer package func TestMain(m *testing.M) { - servicerPublicKey, servicerAddr := testPublicKey() + privateKey, err := crypto.GeneratePrivateKey() + if err != nil { + log.Fatalf("Error generating private key: %s", err) + } + + testServicer1PrivateKey = privateKey testServicer1 = &coreTypes.Actor{ ActorType: coreTypes.ActorType_ACTOR_TYPE_SERVICER, - Address: servicerAddr, - PublicKey: servicerPublicKey, + Address: privateKey.Address().String(), + PublicKey: privateKey.PublicKey().String(), Chains: []string{"POKT-UnitTestNet"}, StakedAmount: "1000", } @@ -142,7 +148,7 @@ func TestRelay_Admit(t *testing.T) { sessionHeight(testSessionStartingHeight), sessionServicers(testServicer1), ) - mockBus := mockBus(t, &config, uint64(testCurrentHeight), session, testCase.usedSessionTokens) + mockBus := mockBus(t, config, uint64(testCurrentHeight), session, testCase.usedSessionTokens) servicerMod, err := CreateServicer(mockBus) require.NoError(t, err) @@ -189,7 +195,7 @@ func TestRelay_Execute(t *testing.T) { config.Services[svc].Url = ts.URL } - servicer := &servicer{config: &config} + servicer := &servicer{config: config} _, err := servicer.executeRelay(testCase.relay) require.ErrorIs(t, err, testCase.expectedErr) // INCOMPLETE(@adshmh): verify HTTP request properties: payload/headers/user-agent/etc. @@ -197,6 +203,49 @@ func TestRelay_Execute(t *testing.T) { } } +func TestRelay_Sign(t *testing.T) { + testCases := []struct { + name string + privateKey string + expected []byte + expectErr bool + }{ + { + name: "Create fails if private key is missing from config", + expectErr: true, + }, + { + name: "Message is signed using correct private key", + privateKey: testServicer1PrivateKey.String(), + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + config := testServicerConfig(withPrivateKey(testCase.privateKey)) + mockBus := mockBus(t, config, 0, &coreTypes.Session{}, 0) + + servicerMod, err := CreateServicer(mockBus) + if testCase.expectErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + + servicer, ok := servicerMod.(*servicer) + require.True(t, ok) + + message := []byte("message") + signature, err := servicer.sign(message) + require.NoError(t, err) + + isSignatureValid := testServicer1PrivateKey.PublicKey().Verify(message, signature) + require.True(t, isSignatureValid) + }) + } +} + type relayEditor func(*coreTypes.Relay) func testRelayServicer(publicKey string) relayEditor { @@ -258,15 +307,28 @@ func testRelay(editors ...relayEditor) *coreTypes.Relay { return relay } -func testServicerConfig() configs.ServicerConfig { - return configs.ServicerConfig{ - PublicKey: testServicer1.PublicKey, - Address: testServicer1.Address, +type configModifier func(*configs.ServicerConfig) + +func withPrivateKey(key string) func(*configs.ServicerConfig) { + return func(cfg *configs.ServicerConfig) { + cfg.PrivateKey = key + } +} + +func testServicerConfig(editors ...configModifier) *configs.ServicerConfig { + config := configs.ServicerConfig{ + PrivateKey: testServicer1PrivateKey.String(), Services: map[string]*configs.ServiceConfig{ "POKT-UnitTestNet": testServiceConfig1, "ETH-Goerli": testServiceConfig1, }, } + + for _, editor := range editors { + editor(&config) + } + + return &config } type sessionModifier func(*coreTypes.Session) From 2a226ccd106967648f8e9127884b6db849386906 Mon Sep 17 00:00:00 2001 From: Dima Kniazev Date: Tue, 1 Aug 2023 15:08:49 -0700 Subject: [PATCH 22/25] [LocalNet] Fix metrics scraping (#940) ## Description Fixes metrics scraping on LocalNet. After quite a few changes to the LocalNet infrastructure, the metrics collection appeared broken on LocalNet. ### Summary generated by Reviewpad on 26 Jul 23 21:58 UTC This pull request fixes the issue with metrics scraping on LocalNet. It updates the Tiltfile and statefulset.yaml files to add pod annotations for Prometheus scraping and specify the port for scraping. ## List of changes - Changed `podAnnotations` helm chart value to always "quote" values in case they are not strings (K8s API expects strings only in annotations) - Added annotations necessary for scraping of exporter endpoints by Prometheus on LocalNet --- build/localnet/Tiltfile | 8 ++++++++ charts/pocket/templates/statefulset.yaml | 11 ++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/build/localnet/Tiltfile b/build/localnet/Tiltfile index 9f5a36b4c..a88a3d8d5 100644 --- a/build/localnet/Tiltfile +++ b/build/localnet/Tiltfile @@ -190,6 +190,8 @@ for x in range(localnet_config["validators"]["count"]): "genesis.externalConfigMap.name=v1-localnet-genesis", "genesis.externalConfigMap.key=genesis.json", "postgresql.primary.persistence.enabled=false", + "podAnnotations.prometheus\\.io/scrape=true", + "podAnnotations.prometheus\\.io/port=9000", "nodeType=validator", ], values=[chart_dir + "/pocket-validator-overrides.yaml"] if os.path.exists(chart_dir + "/pocket-validator-overrides.yaml") else [],)) @@ -213,6 +215,8 @@ for x in range(localnet_config["servicers"]["count"]): "genesis.externalConfigMap.name=v1-localnet-genesis", "genesis.externalConfigMap.key=genesis.json", "postgresql.primary.persistence.enabled=false", + "podAnnotations.prometheus\\.io/scrape=true", + "podAnnotations.prometheus\\.io/port=9000", "config.servicer.enabled=true", "nodeType=servicer", ], @@ -237,6 +241,8 @@ for x in range(localnet_config["fishermen"]["count"]): "genesis.externalConfigMap.name=v1-localnet-genesis", "genesis.externalConfigMap.key=genesis.json", "postgresql.primary.persistence.enabled=false", + "podAnnotations.prometheus\\.io/scrape=true", + "podAnnotations.prometheus\\.io/port=9000", "config.fisherman.enabled=true", "nodeType=fisherman", ], @@ -261,6 +267,8 @@ for x in range(localnet_config["full_nodes"]["count"]): "genesis.externalConfigMap.name=v1-localnet-genesis", "genesis.externalConfigMap.key=genesis.json", "postgresql.primary.persistence.enabled=false", + "podAnnotations.prometheus\\.io/scrape=true", + "podAnnotations.prometheus\\.io/port=9000", "nodeType=full", ], values=[chart_dir + "/pocket-full-node-overrides.yaml"] if os.path.exists(chart_dir + "/pocket-full-node-overrides.yaml") else [],)) diff --git a/charts/pocket/templates/statefulset.yaml b/charts/pocket/templates/statefulset.yaml index ef2d21a30..7f68f29c5 100644 --- a/charts/pocket/templates/statefulset.yaml +++ b/charts/pocket/templates/statefulset.yaml @@ -2,10 +2,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ include "pocket.fullname" . }} - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} labels: {{- include "pocket.labels" . | nindent 4 }} spec: @@ -17,10 +13,15 @@ spec: metadata: {{- with .Values.podAnnotations }} annotations: - {{- toYaml . | nindent 8 }} + {{- range $key, $value := . }} + {{ $key }}: {{ $value | quote }} + {{- end }} {{- end }} labels: {{- include "pocket.selectorLabels" . | nindent 8 }} + {{- if .Values.podLabels }} + {{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: From 6c7599ee36531f489668ae5858151e755c594125 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Wed, 2 Aug 2023 19:57:43 +0200 Subject: [PATCH 23/25] prevent sending to closed channels --- consensus/hotstuff_leader.go | 6 +++--- consensus/pacemaker/module.go | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index cc067098e..a08dc2b10 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -35,6 +35,9 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *consensusM return } + // Leader should prepare a new block. Introducing a delay based on configurations. + m.paceMaker.StartMinBlockTimeDelay() + if err := m.didReceiveEnoughMessageForStep(NewRound); err != nil { m.logger.Info().Fields(hotstuffMsgToLoggingFields(msg)).Msgf("⏳ Waiting ⏳for more messages; %s", err.Error()) return @@ -62,9 +65,6 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *consensusM // TODO: Add test to make sure same block is not applied twice if round is interrupted after being 'Applied'. // TODO: Add more unit tests for these checks... if m.shouldPrepareNewBlock(highPrepareQC) { - // Leader should prepare a new block. Introducing a delay based on configurations. - m.paceMaker.StartMinBlockTimeDelay() - // This function delays block preparation and returns false if a concurrent preparation request with higher QC is available if shouldPrepareBlock := m.paceMaker.DelayBlockPreparation(); !shouldPrepareBlock { m.logger.Info().Msg("skip prepare new block, a candidate with higher QC is available") diff --git a/consensus/pacemaker/module.go b/consensus/pacemaker/module.go index 6734a471c..5737c277a 100644 --- a/consensus/pacemaker/module.go +++ b/consensus/pacemaker/module.go @@ -39,7 +39,9 @@ type Pacemaker interface { PacemakerDebug ShouldHandleMessage(message *typesCons.HotstuffMessage) (bool, error) + // StartMinBlockTimeDelay configures `prepareStepDelayer` in preparation for delaying block proposal StartMinBlockTimeDelay() + // WARNING: DelayBlockPreparation is a synchronous blocking call that acquires a mutex and prevents block propagation until its complete. DelayBlockPreparation() bool RestartTimer() @@ -296,6 +298,7 @@ func (m *pacemaker) StartMinBlockTimeDelay() { if m.prepareStepDelayer.ch != nil { m.prepareStepDelayer.ch <- true close(m.prepareStepDelayer.ch) + m.prepareStepDelayer.ch = nil m.prepareStepDelayer.shouldProposeBlock = true } @@ -328,6 +331,7 @@ func (m *pacemaker) DelayBlockPreparation() bool { if m.prepareStepDelayer.ch != nil { m.prepareStepDelayer.ch <- false close(m.prepareStepDelayer.ch) + m.prepareStepDelayer.ch = nil } // Deadline has passed, no need to have a channel, propose a block now @@ -343,6 +347,8 @@ func (m *pacemaker) DelayBlockPreparation() bool { // We cannot defer the unlock here because the channel read is blocking m.prepareStepDelayer.m.Unlock() + // We are blocking this function so we cannot defer the unlock. + // We need to unlock before reading from the channel but right after last write to `pacemaker.prepareStepDelayer` return <-ch } From 92ece19085d8fe632a4da0f722099666ea64ab1a Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Fri, 4 Aug 2023 08:12:01 +0200 Subject: [PATCH 24/25] disable block preparation delay when manual mode is on --- consensus/pacemaker/module.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/consensus/pacemaker/module.go b/consensus/pacemaker/module.go index 5737c277a..96cc0f962 100644 --- a/consensus/pacemaker/module.go +++ b/consensus/pacemaker/module.go @@ -270,9 +270,14 @@ func (m *pacemaker) NewHeight() { // StartMinBlockTimeDelay should be called when a delay should be introduced into proposing a new block func (m *pacemaker) StartMinBlockTimeDelay() { + if m.debug.manualMode { + m.logger.Info().Msg("Manual mode is enabled. Not starting block time delay.") + return + } + // Discard any previous timer if one exists if m.prepareStepDelayer.cancelFunc != nil { - m.logger.Warn().Msg("RegisterMinBlockTimeDelay has an existing timer which should not happen. Releasing for now...") + m.logger.Warn().Msg("StartMinBlockTimeDelay has an existing timer which should not happen. Releasing for now...") m.prepareStepDelayer.cancelFunc() } @@ -320,6 +325,11 @@ func (m *pacemaker) StartMinBlockTimeDelay() { // - If a late message is received AFTER the a block is marked as proposed by another call, the late message is discarded // - Reads and assignments to pacemaker.prepareStepDelayer state are protected by a mutex func (m *pacemaker) DelayBlockPreparation() bool { + if m.debug.manualMode { + m.logger.Info().Msg("Manual mode is enabled. Not delaying block preparation.") + return true + } + m.prepareStepDelayer.m.Lock() if m.prepareStepDelayer.shouldProposeBlock { From fef4217e3fb28e24b337a59f6ee8f530386ad166 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Thu, 3 Aug 2023 13:33:56 -0700 Subject: [PATCH 25/25] [E2E Test] Utilities for State Sync Test (#874) ## Description ### Summary generated by Reviewpad on 03 Aug 23 20:11 UTC This pull request introduces several changes across multiple files. Here is a summary of the changes: 1. `go.mod`: - The `golang.org/x/text` dependency is now listed explicitly instead of being an indirect dependency. 2. `scenario_test.go`: - Added a new feature called "State Sync Namespace" that includes various commands and waits for specific amounts of time. - Code improvements and TODO comments have been added. 3. `FAQ.md`: - Updated an issue related to starting LocalNet with SELinux on an operating system. Replaced the command `make compose_and_watch` with `make lightweight_localnet` to avoid permission denied errors. 4. `.gitignore`: - Removed the entry "main" from the list of ignored files. - Removed the entries "rpc/server.gen.go" and "rpc/client.gen.go" from the list of ignored files. - Added the entry "**/gomock_reflect_*/" to ignore mock temporary files. 5. `e2e/README.md`: - Added a new section on `Keywords`. - Modified scenario descriptions and code examples to replace instances of "Validator" with "Node". - Included a flowchart depicting the E2E scenarios with updated terminology. 6. Consensus module files: - Added a new logging statement in the `HandleDebugMessage` function. - Simplified the handling of the `DEBUG_CONSENSUS_RESET_TO_GENESIS` action. 7. `.tiltignore`: - Removed the entry "main" from the list of ignored files. - Removed the entries "rpc/server.gen.go" and "rpc/client.gen.go" from the list of ignored files. 8. `CHANGELOG.md`: - Updated build commands and added a new section on `Keywords`. 9. `persistence/docs/CHANGELOG.md`: - Several changes related to deprecation, addition, and fixing of functions and issues. 10. `validator.feature`: - Renamed file from "valdator.feature" to "validator.feature". - Updated scenario titles and step descriptions to use more descriptive terminology. - Replaced references to "validator" with "node". 11. `tilt_helpers.go`: - Added a new file containing functions related to syncing network configuration and checking package installation. 12. `debug.go`: - Added new debug commands and subcommands. - Updated existing functions and added new functions for debug actions. 13. `account.feature`: - Added a new file containing scenarios for testing node account functionalities. 14. `README.md` files: - Updated sections, titles, dependencies, and instructions in various README.md files. 15. `build/config/README.md`: - Updated usage instructions, changing the command `make compose_and_watch` to `make lightweight_localnet`. 16. `iteration_3_end_to_end_tx.md`: - Updated commands to start LocalNet and consensus debugger. 17. Deleted files: - `validator.go` - `watch_build.sh` 18. New files added: - `debug.feature` - `tilt_helpers.go` - `account.feature` Please let me know if you need more information about any specific change. ## Issue Fixes par of #579 ## Type of change Please mark the relevant option(s): - [x] New feature, functionality or library - [ ] Bug fix - [ ] Code health or cleanup - [ ] Major breaking change - [ ] Documentation - [ ] Other ## List of changes - `s/compose_and_watch/lightweight_localnet` and all related helpers - `s/validator/node` in e2e tests for clarity - Add fire-and-forget `Debug` CLI w/ several useful initial subcommands - Add `keywords` to the `e2e` document - Add an `e2e debug` test to trigger views and track the blockchain increasing - Avoid rebuilding the actors if the CLI changes - Small miscellaneous improvements & code cleanup ## Testing - [x] `make develop_test`; if any code changes were made - [ ] `make test_e2e` on [k8s LocalNet](https://github.com/pokt-network/pocket/blob/main/build/localnet/README.md); if any code changes were made - [ ] `e2e-devnet-test` passes tests on [DevNet](https://pocketnetwork.notion.site/How-to-DevNet-ff1598f27efe44c09f34e2aa0051f0dd); if any code was changed - [ ] [Docker Compose LocalNet](https://github.com/pokt-network/pocket/blob/main/docs/development/README.md); if any major functionality was changed or introduced - [ ] [k8s LocalNet](https://github.com/pokt-network/pocket/blob/main/build/localnet/README.md); if any infrastructure or configuration changes were made ## Required Checklist - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have added, or updated, [`godoc` format comments](https://go.dev/blog/godoc) on touched members (see: [tip.golang.org/doc/comment](https://tip.golang.org/doc/comment)) - [ ] I have tested my changes using the available tooling - [ ] I have updated the corresponding CHANGELOG ### If Applicable Checklist - [x] I have updated the corresponding README(s); local and/or global - [x] I have added tests that prove my fix is effective or that my feature works - [ ] I have added, or updated, [mermaid.js](https://mermaid-js.github.io) diagrams in the corresponding README(s) - [ ] I have added, or updated, documentation and [mermaid.js](https://mermaid-js.github.io) diagrams in `shared/docs/*` if I updated `shared/*`README(s) --- Co-authored-by: d7t --- .github/workflows/main.yml | 2 +- .gitignore | 6 +- .tiltignore | 3 - Makefile | 51 ++-- app/client/cli/debug.go | 137 +++++++++-- build/config/README.md | 2 +- build/docs/CHANGELOG.md | 2 +- build/localnet/README.md | 10 +- build/localnet/Tiltfile | 228 +++++++++++------- build/scripts/watch.sh | 9 +- build/scripts/watch_build.sh | 8 - consensus/module_consensus_debugging.go | 6 +- docs/demos/iteration_3_end_to_end_tx.md | 4 +- docs/development/FAQ.md | 4 +- docs/development/README.md | 7 +- e2e/README.md | 27 ++- e2e/docs/E2E_ADR.md | 10 +- e2e/tests/account.feature | 27 +++ e2e/tests/debug.feature | 18 ++ e2e/tests/node.go | 76 ++++++ e2e/tests/query.feature | 10 +- e2e/tests/root.feature | 4 +- e2e/tests/state_sync.feature | 23 ++ e2e/tests/steps_init_test.go | 206 +++++++++++++--- e2e/tests/tilt_helpers.go | 34 +++ .../{valdator.feature => validator.feature} | 25 +- e2e/tests/validator.go | 67 ----- go.mod | 2 +- persistence/docs/CHANGELOG.md | 2 +- persistence/docs/README.md | 2 +- shared/modules/doc/CHANGELOG.md | 4 +- telemetry/README.md | 2 +- 32 files changed, 715 insertions(+), 303 deletions(-) delete mode 100755 build/scripts/watch_build.sh create mode 100644 e2e/tests/account.feature create mode 100644 e2e/tests/debug.feature create mode 100644 e2e/tests/node.go create mode 100644 e2e/tests/state_sync.feature create mode 100644 e2e/tests/tilt_helpers.go rename e2e/tests/{valdator.feature => validator.feature} (52%) delete mode 100644 e2e/tests/validator.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 24d809706..57ea68386 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -197,4 +197,4 @@ jobs: - id: "run-e2e-tests" run: | - ./argo-linux-amd64 submit --wait --log --namespace devnet-issue-${{ github.event.pull_request.number }} --from 'wftmpl/dev-e2e-tests' --parameter gitsha="${{ github.event.pull_request.head.sha }}" + ./argo-linux-amd64 submit --wait --log --namespace devnet-issue-${{ github.event.pull_request.number }} --from 'wftmpl/dev-e2e-tests' --parameter tags="~@skip_in_ci" --parameter gitsha="${{ github.event.pull_request.head.sha }}" diff --git a/.gitignore b/.gitignore index d049b2729..4996b6c24 100644 --- a/.gitignore +++ b/.gitignore @@ -55,9 +55,6 @@ temp_test.go test_results.json coverage.out -# Output of `make build_and_watch` -main - # generated RPC server and client from openapi.yaml rpc/server.gen.go rpc/client.gen.go @@ -90,3 +87,6 @@ tools/wiki # ggshield .cache_ggshield + +# mock temporary files +**/gomock_reflect_*/ diff --git a/.tiltignore b/.tiltignore index 63afc9698..9a59a3fde 100644 --- a/.tiltignore +++ b/.tiltignore @@ -41,9 +41,6 @@ temp_test.go test_results.json coverage.out -# Output of `make build_and_watch` -main - # generated RPC server and client from openapi.yaml rpc/server.gen.go rpc/client.gen.go diff --git a/Makefile b/Makefile index 9ec1c636a..e268d49d8 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ help: docker_check: { \ if ( ! ( command -v docker >/dev/null && (docker compose version >/dev/null || command -v docker-compose >/dev/null) )); then \ - echo "Seems like you don't have Docker or docker-compose installed. Make sure you review docs/development/README.md before continuing"; \ + echo "Seems like you don't have Docker or docker-compose installed. Make sure you review build/localnet/README.md and docs/development/README.md before continuing"; \ exit 1; \ fi; \ } @@ -47,11 +47,21 @@ docker_check: kubectl_check: { \ if ( ! ( command -v kubectl >/dev/null )); then \ - echo "Seems like you don't have Kubectl installed. Make sure you review docs/development/README.md before continuing"; \ + echo "Seems like you don't have Kubectl installed. Make sure you review build/localnet/README.md and docs/development/README.md before continuing"; \ exit 1; \ fi; \ } +# Internal helper target - check if rsync is installed. +rsync_check: + { \ + if ( ! ( command -v kubectl >/dev/null )); then \ + echo "Seems like you don't have rsync installed. Make sure you review build/localnet/README.md and docs/development/README.md before continuing"; \ + exit 1; \ + fi; \ + } + + .PHONY: trigger_ci trigger_ci: ## Trigger the CI pipeline by submitting an empty commit; See https://github.com/pokt-network/pocket/issues/900 for details git commit --allow-empty -m "Empty commit" @@ -133,6 +143,9 @@ go_imports: ## Group imports using rinchsan/gosimports go_fmt: ## Format all the .go files in the project in place. gofmt -w -s . +# TODO(#964): add `rsync_check`, `kubectl_check`, `docker_check` as a validation in `install_cli_deps`; https://github.com/pokt-network/pocket/assets/1892194/a7a24a11-f54d-46e2-a73e-9e8ea7d06726 +# .PHONY: install_cli_deps +# install_cli_deps: rsync_check kubectl_check docker_check ## Installs `helm`, `tilt` and the underlying `ci_deps` .PHONY: install_cli_deps install_cli_deps: ## Installs `helm`, `tilt` and the underlying `ci_deps` make install_ci_deps @@ -163,33 +176,27 @@ develop_test: docker_check ## Run all of the make commands necessary to develop make develop_start && \ make test_all -.PHONY: client_start -client_start: docker_check ## Run a client daemon which is only used for debugging purposes +.PHONY: lightweight_localnet_client +lightweight_localnet_client: docker_check ## Run a client daemon which is only used for debugging purposes +# Add `--build` to rebuild the client ${docker-compose} up -d client -.PHONY: rebuild_client_start -rebuild_client_start: docker_check ## Rebuild and run a client daemon which is only used for debugging purposes - ${docker-compose} up -d --build client - -.PHONY: client_connect -client_connect: docker_check ## Connect to the running client debugging daemon +.PHONY: lightweight_localnet_client_debug +lightweight_localnet_client_debug: docker_check ## Connect to the running client debugging daemon docker exec -it client /bin/bash -c "go run -tags=debug app/client/*.go DebugUI" -.PHONY: build_and_watch -build_and_watch: ## Continous build Pocket's main entrypoint as files change - /bin/sh ${PWD}/build/scripts/watch_build.sh +# IMPROVE: Avoid building the binary on every shell execution and sync it from local instead +.PHONY: lightweight_localnet_shell +lightweight_localnet_shell: docker_check ## Connect to the running client debugging daemon + docker exec -it client /bin/bash -c "go build -tags=debug -o p1 ./app/client/*.go && chmod +x p1 && mv p1 /usr/bin && echo \"Finished building a new p1 binary\" && /bin/bash" -# TODO(olshansky): Need to think of a Pocket related name for `compose_and_watch`, maybe just `pocket_watch`? -.PHONY: compose_and_watch -compose_and_watch: docker_check db_start monitoring_start ## Run a localnet composed of 4 consensus validators w/ hot reload & debugging +.PHONY: lightweight_localnet +lightweight_localnet: docker_check db_start monitoring_start ## Run a lightweight localnet composed of 4 validators w/ hot reload & debugging +# Add `--build` to rebuild the client ${docker-compose} up --force-recreate validator1 validator2 validator3 validator4 servicer1 fisherman1 -.PHONY: rebuild_and_compose_and_watch -rebuild_and_compose_and_watch: docker_check db_start monitoring_start ## Rebuilds the container from scratch and launches compose_and_watch - ${docker-compose} up --build --force-recreate validator1 validator2 validator3 validator4 servicer1 fisherman1 - .PHONY: db_start -db_start: docker_check ## Start a detached local postgres and admin instance; compose_and_watch is responsible for instantiating the actual schemas +db_start: docker_check ## Start a detached local postgres and admin instance; lightweight_localnet is responsible for instantiating the actual schemas ${docker-compose} up --no-recreate -d db pgadmin .PHONY: db_cli @@ -245,7 +252,7 @@ docker_wipe_nodes: docker_check prompt_user db_drop ## [WARNING] Remove all the docker ps -a -q --filter="name=node*" | xargs -r -I {} docker rm {} .PHONY: monitoring_start -monitoring_start: docker_check ## Start grafana, metrics and logging system (this is auto-triggered by compose_and_watch) +monitoring_start: docker_check ## Start grafana, metrics and logging system (this is auto-triggered by lightweight_localnet) ${docker-compose} up --no-recreate -d grafana loki vm .PHONY: docker_loki_install diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index 4cc1ea632..e2f715289 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -1,11 +1,15 @@ package cli import ( + "fmt" + "log" "os" + "os/exec" "time" "github.com/manifoldco/promptui" "github.com/spf13/cobra" + "golang.org/x/exp/slices" "google.golang.org/protobuf/types/known/anypb" "github.com/pokt-network/pocket/app/client/cli/helpers" @@ -35,31 +39,103 @@ var items = []string{ } func init() { + dbg := newDebugCommand() + dbg.AddCommand(newDebugSubCommands()...) + rootCmd.AddCommand(dbg) + dbgUI := newDebugUICommand() - dbgUI.AddCommand(newDebugUISubCommands()...) rootCmd.AddCommand(dbgUI) } -// newDebugUISubCommands builds out the list of debug subcommands by matching the -// handleSelect dispatch to the appropriate command. -// * To add a debug subcommand, you must add it to the `items` array and then -// write a function handler to match for it in `handleSelect`. -func newDebugUISubCommands() []*cobra.Command { - commands := make([]*cobra.Command, len(items)) - for idx, promptItem := range items { - commands[idx] = &cobra.Command{ - Use: promptItem, +// newDebugCommand returns the cobra CLI for the Debug command. +func newDebugCommand() *cobra.Command { + return &cobra.Command{ + Use: "Debug", + Aliases: []string{"d"}, + Short: "Debug utility for rapid development", + Long: "Debug utility to send fire-and-forget messages to the network for development purposes", + Args: cobra.MaximumNArgs(1), + } +} + +// newDebugSubCommands is a list of commands that can be "fired & forgotten" (no selection necessary) +func newDebugSubCommands() []*cobra.Command { + cmds := []*cobra.Command{ + { + Use: "PrintNodeState", + Aliases: []string{"print", "state"}, + Short: "Prints the node state", + Long: "Sends a message to all visible nodes to log the current state of their consensus", + Args: cobra.ExactArgs(0), PersistentPreRunE: helpers.P2PDependenciesPreRunE, - Run: func(cmd *cobra.Command, _ []string) { - // TECHDEBT(#874): this is a magic number, but an alternative would be to have the p2p module wait until connections are open and to flush the message correctly - time.Sleep(500 * time.Millisecond) // give p2p module time to start - handleSelect(cmd, cmd.Use) - time.Sleep(500 * time.Millisecond) // give p2p module time to broadcast + Run: func(cmd *cobra.Command, args []string) { + runWithSleep(func() { + handleSelect(cmd, PromptPrintNodeState) + }) }, - ValidArgs: items, - } + }, + { + Use: "ResetToGenesis", + Aliases: []string{"reset", "genesis"}, + Short: "Reset to genesis", + Long: "Broadcast a message to all visible nodes to reset the state to genesis", + Args: cobra.ExactArgs(0), + PersistentPreRunE: helpers.P2PDependenciesPreRunE, + Run: func(cmd *cobra.Command, args []string) { + runWithSleep(func() { + handleSelect(cmd, PromptResetToGenesis) + }) + }, + }, + { + Use: "TriggerView", + Aliases: []string{"next", "trigger", "view"}, + Short: "Trigger the next view in consensus", + Long: "Sends a message to all visible nodes on the network to start the next view (height/step/round) in consensus", + Args: cobra.ExactArgs(0), + PersistentPreRunE: helpers.P2PDependenciesPreRunE, + Run: func(cmd *cobra.Command, args []string) { + runWithSleep(func() { + handleSelect(cmd, PromptTriggerNextView) + }) + }, + }, + { + Use: "TogglePacemakerMode", + Aliases: []string{"toggle", "pcm"}, + Short: "Toggle the pacemaker", + Long: "Toggle the consensus pacemaker either on or off so the chain progresses on its own or loses liveness", + Args: cobra.ExactArgs(0), + PersistentPreRunE: helpers.P2PDependenciesPreRunE, + Run: func(cmd *cobra.Command, args []string) { + runWithSleep(func() { + handleSelect(cmd, PromptTogglePacemakerMode) + }) + }, + }, + { + Use: "ScaleActor", + Aliases: []string{"scale"}, + Short: "Scales the number of actors up or down", + Long: "Scales the type of actor specified to the number provided", + Args: cobra.ExactArgs(2), + PersistentPreRunE: helpers.P2PDependenciesPreRunE, + Run: func(cmd *cobra.Command, args []string) { + actor := args[0] + numActors := args[1] + validActors := []string{"fishermen", "full_nodes", "servicers", "validators"} + if !slices.Contains(validActors, actor) { + logger.Global.Fatal().Msg("Invalid actor type provided") + } + sedReplaceCmd := fmt.Sprintf("/%s:/,/count:/ s/count: [0-9]*/count: %s/", actor, numActors) + sedCmd := exec.Command("sed", "-i", sedReplaceCmd, "/usr/local/localnet_config.yaml") + if err := sedCmd.Run(); err != nil { + log.Fatal(err) + } + }, + }, } - return commands + return cmds } // newDebugUICommand returns the cobra CLI for the Debug UI interface. @@ -67,14 +143,19 @@ func newDebugUICommand() *cobra.Command { return &cobra.Command{ Aliases: []string{"dui", "debug"}, Use: "DebugUI", - Short: "Debug selection ui for rapid development", + Short: "Debug utility with an interactive UI for development purposes", + Long: "Opens a shell-driven selection UI to view and select from a list of debug actions for development purposes", Args: cobra.MaximumNArgs(0), PersistentPreRunE: helpers.P2PDependenciesPreRunE, - RunE: runDebug, + RunE: selectDebugCommand, } } -func runDebug(cmd *cobra.Command, _ []string) (err error) { +// selectDebugCommand builds out the list of debug subcommands by matching the +// handleSelect dispatch to the appropriate command. +// - To add a debug subcommand, you must add it to the `items` array and then +// write a function handler to match for it in `handleSelect`. +func selectDebugCommand(cmd *cobra.Command, _ []string) error { for { if selection, err := promptGetInput(); err == nil { handleSelect(cmd, selection) @@ -162,7 +243,17 @@ func handleSelect(cmd *cobra.Command, selection string) { } } -// Broadcast to the entire network. +// HACK: Because of how the p2p module works, we need to surround it with sleep both BEFORE and AFTER the task. +// - Starting the task too early after the debug client initializes results in a lack of visibility of the nodes in the network +// - Ending the task too early before the debug client completes its task results in a lack of propagation of the message or retrieval of the result +// TECHDEBT: There is likely an event based solution to this but it would require a lot more refactoring of the p2p module. +func runWithSleep(task func()) { + time.Sleep(1000 * time.Millisecond) + task() + time.Sleep(1000 * time.Millisecond) +} + +// broadcastDebugMessage broadcasts the debug message to the entire visible network. func broadcastDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { anyProto, err := anypb.New(debugMsg) if err != nil { @@ -178,7 +269,7 @@ func broadcastDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) } } -// Send to just a single (i.e. first) validator in the set +// sendDebugMessage sends the debug message to just a single (i.e. first) node visible func sendDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { anyProto, err := anypb.New(debugMsg) if err != nil { diff --git a/build/config/README.md b/build/config/README.md index 24d8d110c..fffb1883b 100644 --- a/build/config/README.md +++ b/build/config/README.md @@ -12,7 +12,7 @@ It is not recommended at this time to build infrastructure components that rely ## Origin Document -Currently, the Genesis and Configuration generator is necessary to create development `localnet` environments for iterating on V1. A current example (as of 09/2022) of this is the `make compose_and_watch` debug utility that generates a `localnet` using `docker-compose` by injecting the appropriate `config.json` and `genesis.json` files. +Currently, the Genesis and Configuration generator is necessary to create development `localnet` environments for iterating on V1. A current example (as of 09/2022) of this is the `make lightweight_localnet` debug utility that generates a `localnet` using `docker-compose` by injecting the appropriate `config.json` and `genesis.json` files. ## Usage diff --git a/build/docs/CHANGELOG.md b/build/docs/CHANGELOG.md index a23ab1698..aec7e163e 100644 --- a/build/docs/CHANGELOG.md +++ b/build/docs/CHANGELOG.md @@ -233,7 +233,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.0.0.1] - 2022-12-29 - Updated all `config*.json` files with the missing `max_mempool_count` value -- Added `is_client_only` to `config1.json` so Viper knows it can be overridden. The config override is done in the Makefile's `client_connect` target. Setting this can be avoided if we merge the changes in https://github.com/pokt-network/pocket/compare/main...issue/cli-viper-environment-vars-fix +- Added `is_client_only` to `config1.json` so Viper knows it can be overridden. The config override is done in the Makefile's `lightweight_localnet_client_debug` target. Setting this can be avoided if we merge the changes in https://github.com/pokt-network/pocket/compare/main...issue/cli-viper-environment-vars-fix ## [0.0.0.0] - 2022-12-22 diff --git a/build/localnet/README.md b/build/localnet/README.md index ae52c319a..02d9a1494 100644 --- a/build/localnet/README.md +++ b/build/localnet/README.md @@ -2,7 +2,7 @@ This guide shows how to deploy a LocalNet using [pocket-operator](https://github.com/pokt-network/pocket-operator). -- [TLDR](#tldr) +- [TL;DR](#tldr) - [Dependencies](#dependencies) - [Choosing Kubernetes Distribution](#choosing-kubernetes-distribution) - [How to create Kind Kubernetes cluster](#how-to-create-kind-kubernetes-cluster) @@ -16,6 +16,8 @@ This guide shows how to deploy a LocalNet using [pocket-operator](https://github - [Interacting w/ LocalNet](#interacting-w-localnet) - [Make Targets](#make-targets) - [Addresses and keys on LocalNet](#addresses-and-keys-on-localnet) + - [Applications staked on LocalNet](#applications-staked-on-localnet) + - [Servicers staked on LocalNet](#servicers-staked-on-localnet) - [How to change configuration files](#how-to-change-configuration-files) - [Overriding default values for localnet with Tilt](#overriding-default-values-for-localnet-with-tilt) - [How does it work?](#how-does-it-work) @@ -26,7 +28,7 @@ This guide shows how to deploy a LocalNet using [pocket-operator](https://github - [Full Cleanup](#full-cleanup) - [Code Structure](#code-structure) -## TLDR +## TL;DR If you feel adventurous, and you know what you're doing, here is a rapid guide to start LocalNet: @@ -46,6 +48,7 @@ All necessary dependencies, except Docker and Kubernetes cluster, are installed 3. `Kubernetes cluster`: refer to [Choosing Kubernetes Distribution](#choosing-kubernetes-distribution) section for more details. 4. `kubectl`: CLI is required and should be configured to access the cluster. This should happen automatically if using Docker Desktop, Rancher Desktop, k3s, k3d, minikube, etc. 5. [helm](https://helm.sh/docs/intro/install): required to template the YAML manifests for the dependencies (e.g., Postgres, Grafana). Installation instructions available. +6. [rsync](https://www.hostinger.com/tutorials/how-to-use-rsync): required to for some extensions used with `Tilt`; https://github.com/tilt-dev/tilt-extensions/tree/master/syncback#usage ### Choosing Kubernetes Distribution @@ -149,8 +152,8 @@ For example: - `0010297b55fc9278e4be4f1bcfe52bf9bd0443f8` is a servicer #001. - `314019dbb7faf8390c1f0cf4976ef1215c90b7e4` is an application #314. - #### Applications staked on LocalNet + Applications with the following addresses are staked on LocalNet, through the [applications field of the genesis.json in the LocalNet configuration](https://github.com/pokt-network/pocket/blob/main/build/localnet/manifests/configs.yaml#L4088) - `00001fff518b1cdddd74c197d76ba5b5dedc0301` @@ -159,6 +162,7 @@ Applications with the following addresses are staked on LocalNet, through the [a These addresses can be used for e.g. testing the CLI. #### Servicers staked on LocalNet + Servicers with the following addresses are staked on LocalNet, through the [servicers field of the genesis.json in the LocalNet configuration](https://github.com/pokt-network/pocket/blob/main/build/localnet/manifests/configs.yaml#L4120) - `00002b8cea1bcc3dadc72ebecf95564ceb9c2e2a` diff --git a/build/localnet/Tiltfile b/build/localnet/Tiltfile index a88a3d8d5..d4534df35 100644 --- a/build/localnet/Tiltfile +++ b/build/localnet/Tiltfile @@ -2,7 +2,8 @@ load("ext://helm_resource", "helm_resource", "helm_repo") load("ext://namespace", "namespace_create") load("ext://restart_process", "docker_build_with_restart") -load('ext://tests/golang', 'test_go') +load("ext://tests/golang", "test_go") +load("ext://syncback", "syncback") tiltfile_dir = os.path.dirname(config.main_dir) root_dir = os.path.dirname(tiltfile_dir + "/../..") @@ -13,7 +14,7 @@ localnet_config_defaults = { "validators": {"count": 4}, "servicers": {"count": 1}, "fishermen": {"count": 1}, - "full_nodes": {"count": 1} + "full_nodes": {"count": 1}, } localnet_config_file = read_yaml(localnet_config_path, default=localnet_config_defaults) @@ -22,6 +23,7 @@ localnet_config = {} localnet_config.update(localnet_config_defaults) localnet_config.update(localnet_config_file) + # Create a default config file if it does not exist if (localnet_config_file != localnet_config) or ( not os.path.exists(localnet_config_path) @@ -29,6 +31,15 @@ if (localnet_config_file != localnet_config) or ( print("Updating " + localnet_config_path + " with defaults") local("cat - > " + localnet_config_path, stdin=encode_yaml(localnet_config)) +syncback( + name="syncback_localnet_config", + k8s_object="deploy/dev-cli-client", + src_dir="/usr/local/", + paths=["localnet_config.yaml"], + target_dir=root_dir, + labels=["watchers"], +) + # List of directories Tilt watches to trigger a hot-reload on changes. # CONSIDERATION: This can potentially can be replaced with a list of excluded directories. deps = [ @@ -49,6 +60,7 @@ deps = [ deps_full_path = [root_dir + "/" + depdir for depdir in deps] + # Avoid downloading dependencies if no missing/outdated charts are found def check_helm_dependencies_for_chart(path): check_helm_dependencies = local( @@ -58,6 +70,7 @@ def check_helm_dependencies_for_chart(path): if helm_dependencies_not_ok_count > 1: local("helm dependency update " + path) + check_helm_dependencies_for_chart("dependencies") k8s_yaml(helm("dependencies", name="dependencies")) @@ -78,7 +91,7 @@ local_resource( root_dir=root_dir ), deps=deps_full_path, - labels=['watchers'] + labels=["watchers"], ) local_resource( "debug client: Watch & Compile", @@ -86,16 +99,16 @@ local_resource( root_dir=root_dir ), deps=deps_full_path, - labels=['watchers'] + labels=["watchers"], ) # Builds the cluster manager binary local_resource( - 'cluster manager: Watch & Compile', - 'GOOS=linux go build -o {root_dir}/bin/cluster-manager {root_dir}/build/localnet/cluster-manager/*.go'.format( + "cluster manager: Watch & Compile", + "GOOS=linux go build -o {root_dir}/bin/cluster-manager {root_dir}/build/localnet/cluster-manager/*.go".format( root_dir=root_dir ), deps=deps_full_path, - labels=['watchers'] + labels=["watchers"], ) # Builds and maintains the pocket container image after the binary is built on local machine, restarts a process on code change @@ -126,10 +139,14 @@ RUN echo "source /etc/bash_completion" >> ~/.bashrc RUN echo "source <(p1 completion bash | tail -n +2)" >> ~/.bashrc WORKDIR /root COPY bin/p1-linux /usr/local/bin/p1 +COPY localnet_config.yaml /usr/local/localnet_config.yaml """, - only=["bin/p1-linux"], + only=["bin/p1-linux", localnet_config_path], entrypoint=["sleep", "infinity"], - live_update=[sync("bin/p1-linux", "/usr/local/bin/p1")], + live_update=[ + sync("bin/p1-linux", "/usr/local/bin/p1"), + sync(localnet_config_path, "/usr/local/localnet_config.yaml"), + ], ) # Builds and maintains the cluster-manager container image after the binary is built on local machine @@ -141,12 +158,12 @@ WORKDIR / COPY bin/cluster-manager /usr/local/bin/cluster-manager COPY bin/p1-linux /usr/local/bin/p1 """, - only=['bin/cluster-manager', 'bin/p1-linux'], + only=["bin/cluster-manager", "bin/p1-linux"], entrypoint=["/usr/local/bin/cluster-manager"], live_update=[ sync("bin/cluster-manager", "/usr/local/bin/cluster-manager"), sync("bin/p1-linux", "/usr/local/bin/p1"), - ] + ], ) # Pushes localnet manifests to the cluster. @@ -162,9 +179,9 @@ k8s_yaml( ) k8s_yaml(["manifests/cli-client.yaml"]) -k8s_resource('dev-cli-client', labels=['client']) -k8s_yaml(['manifests/cluster-manager.yaml']) -k8s_resource('pocket-v1-cluster-manager', labels=['cluster-manager']) +k8s_resource("dev-cli-client", labels=["client"]) +k8s_yaml(["manifests/cluster-manager.yaml"]) +k8s_resource("pocket-v1-cluster-manager", labels=["cluster-manager"]) chart_dir = root_dir + "/charts/pocket" check_helm_dependencies_for_chart(chart_dir) @@ -173,30 +190,36 @@ check_helm_dependencies_for_chart(chart_dir) def formatted_actor_number(n): return local('printf "%03d" ' + str(n)) + # Provisions validators actor_number = 0 for x in range(localnet_config["validators"]["count"]): actor_number = actor_number + 1 formatted_number = formatted_actor_number(actor_number) - k8s_yaml(helm(chart_dir, - name="validator-%s-pocket" % formatted_number, - set=[ - "global.postgresql.auth.postgresPassword=LocalNetPassword", - "image.repository=pocket-image", - "privateKeySecretKeyRef.name=validators-private-keys", - "privateKeySecretKeyRef.key=%s" % formatted_number, - "genesis.preProvisionedGenesis.enabled=false", - "genesis.externalConfigMap.name=v1-localnet-genesis", - "genesis.externalConfigMap.key=genesis.json", - "postgresql.primary.persistence.enabled=false", - "podAnnotations.prometheus\\.io/scrape=true", - "podAnnotations.prometheus\\.io/port=9000", - "nodeType=validator", - ], - values=[chart_dir + "/pocket-validator-overrides.yaml"] if os.path.exists(chart_dir + "/pocket-validator-overrides.yaml") else [],)) - - k8s_resource("validator-%s-pocket" % formatted_number, labels=['pocket-validators']) + k8s_yaml( + helm( + chart_dir, + name="validator-%s-pocket" % formatted_number, + set=[ + "global.postgresql.auth.postgresPassword=LocalNetPassword", + "image.repository=pocket-image", + "privateKeySecretKeyRef.name=validators-private-keys", + "privateKeySecretKeyRef.key=%s" % formatted_number, + "genesis.preProvisionedGenesis.enabled=false", + "genesis.externalConfigMap.name=v1-localnet-genesis", + "genesis.externalConfigMap.key=genesis.json", + "postgresql.primary.persistence.enabled=false", + "podAnnotations.prometheus\\.io/scrape=true", + "podAnnotations.prometheus\\.io/port=9000", + "nodeType=validator", + ], + values=[chart_dir + "/pocket-validator-overrides.yaml"] + if os.path.exists(chart_dir + "/pocket-validator-overrides.yaml") + else [], + ) + ) + k8s_resource("validator-%s-pocket" % formatted_number, labels=["pocket-validators"]) # Provisions servicer nodes actor_number = 0 @@ -204,25 +227,30 @@ for x in range(localnet_config["servicers"]["count"]): actor_number = actor_number + 1 formatted_number = formatted_actor_number(actor_number) - k8s_yaml(helm(chart_dir, - name="servicer-%s-pocket" % formatted_number, - set=[ - "global.postgresql.auth.postgresPassword=LocalNetPassword", - "image.repository=pocket-image", - "privateKeySecretKeyRef.name=servicers-private-keys", - "privateKeySecretKeyRef.key=%s" % formatted_number, - "genesis.preProvisionedGenesis.enabled=false", - "genesis.externalConfigMap.name=v1-localnet-genesis", - "genesis.externalConfigMap.key=genesis.json", - "postgresql.primary.persistence.enabled=false", - "podAnnotations.prometheus\\.io/scrape=true", - "podAnnotations.prometheus\\.io/port=9000", - "config.servicer.enabled=true", - "nodeType=servicer", - ], - values=[chart_dir + "/pocket-servicer-overrides.yaml"] if os.path.exists(chart_dir + "/pocket-servicer-overrides.yaml") else [],)) - - k8s_resource("servicer-%s-pocket" % formatted_number, labels=['pocket-servicers']) + k8s_yaml( + helm( + chart_dir, + name="servicer-%s-pocket" % formatted_number, + set=[ + "global.postgresql.auth.postgresPassword=LocalNetPassword", + "image.repository=pocket-image", + "privateKeySecretKeyRef.name=servicers-private-keys", + "privateKeySecretKeyRef.key=%s" % formatted_number, + "genesis.preProvisionedGenesis.enabled=false", + "genesis.externalConfigMap.name=v1-localnet-genesis", + "genesis.externalConfigMap.key=genesis.json", + "postgresql.primary.persistence.enabled=false", + "podAnnotations.prometheus\\.io/scrape=true", + "podAnnotations.prometheus\\.io/port=9000", + "config.servicer.enabled=true", + "nodeType=servicer", + ], + values=[chart_dir + "/pocket-servicer-overrides.yaml"] + if os.path.exists(chart_dir + "/pocket-servicer-overrides.yaml") + else [], + ) + ) + k8s_resource("servicer-%s-pocket" % formatted_number, labels=["pocket-servicers"]) # Provisions fishermen nodes actor_number = 0 @@ -230,50 +258,61 @@ for x in range(localnet_config["fishermen"]["count"]): actor_number = actor_number + 1 formatted_number = formatted_actor_number(actor_number) - k8s_yaml(helm(chart_dir, - name="fisherman-%s-pocket" % formatted_number, - set=[ - "global.postgresql.auth.postgresPassword=LocalNetPassword", - "image.repository=pocket-image", - "privateKeySecretKeyRef.name=fishermen-private-keys", - "privateKeySecretKeyRef.key=%s" % formatted_number, - "genesis.preProvisionedGenesis.enabled=false", - "genesis.externalConfigMap.name=v1-localnet-genesis", - "genesis.externalConfigMap.key=genesis.json", - "postgresql.primary.persistence.enabled=false", - "podAnnotations.prometheus\\.io/scrape=true", - "podAnnotations.prometheus\\.io/port=9000", - "config.fisherman.enabled=true", - "nodeType=fisherman", - ], - values=[chart_dir + "/pocket-fisherman-overrides.yaml"] if os.path.exists(chart_dir + "/pocket-fisherman-overrides.yaml") else [],)) - - k8s_resource("fisherman-%s-pocket" % formatted_number, labels=['pocket-fishermen']) + k8s_yaml( + helm( + chart_dir, + name="fisherman-%s-pocket" % formatted_number, + set=[ + "global.postgresql.auth.postgresPassword=LocalNetPassword", + "image.repository=pocket-image", + "privateKeySecretKeyRef.name=fishermen-private-keys", + "privateKeySecretKeyRef.key=%s" % formatted_number, + "genesis.preProvisionedGenesis.enabled=false", + "genesis.externalConfigMap.name=v1-localnet-genesis", + "genesis.externalConfigMap.key=genesis.json", + "postgresql.primary.persistence.enabled=false", + "podAnnotations.prometheus\\.io/scrape=true", + "podAnnotations.prometheus\\.io/port=9000", + "config.fisherman.enabled=true", + "nodeType=fisherman", + ], + values=[chart_dir + "/pocket-fisherman-overrides.yaml"] + if os.path.exists(chart_dir + "/pocket-fisherman-overrides.yaml") + else [], + ) + ) + + k8s_resource("fisherman-%s-pocket" % formatted_number, labels=["pocket-fishermen"]) # Provisions full nodes actor_number = 0 for x in range(localnet_config["full_nodes"]["count"]): actor_number = actor_number + 1 formatted_number = formatted_actor_number(actor_number) + k8s_yaml( + helm( + root_dir + "/charts/pocket", + name="full-node-%s-pocket" % formatted_number, + set=[ + "global.postgresql.auth.postgresPassword=LocalNetPassword", + "image.repository=pocket-image", + "privateKeySecretKeyRef.name=misc-private-keys", + "privateKeySecretKeyRef.key=%s" % formatted_number, + "genesis.preProvisionedGenesis.enabled=false", + "genesis.externalConfigMap.name=v1-localnet-genesis", + "genesis.externalConfigMap.key=genesis.json", + "postgresql.primary.persistence.enabled=false", + "podAnnotations.prometheus\\.io/scrape=true", + "podAnnotations.prometheus\\.io/port=9000", + "nodeType=full", + ], + values=[chart_dir + "/pocket-full-node-overrides.yaml"] + if os.path.exists(chart_dir + "/pocket-full-node-overrides.yaml") + else [], + ) + ) - k8s_yaml(helm(root_dir + "/charts/pocket", - name="full-node-%s-pocket" % formatted_number, - set=[ - "global.postgresql.auth.postgresPassword=LocalNetPassword", - "image.repository=pocket-image", - "privateKeySecretKeyRef.name=misc-private-keys", - "privateKeySecretKeyRef.key=%s" % formatted_number, - "genesis.preProvisionedGenesis.enabled=false", - "genesis.externalConfigMap.name=v1-localnet-genesis", - "genesis.externalConfigMap.key=genesis.json", - "postgresql.primary.persistence.enabled=false", - "podAnnotations.prometheus\\.io/scrape=true", - "podAnnotations.prometheus\\.io/port=9000", - "nodeType=full", - ], - values=[chart_dir + "/pocket-full-node-overrides.yaml"] if os.path.exists(chart_dir + "/pocket-full-node-overrides.yaml") else [],)) - - k8s_resource("full-node-%s-pocket" % formatted_number, labels=['pocket-full-nodes']) + k8s_resource("full-node-%s-pocket" % formatted_number, labels=["pocket-full-nodes"]) # Exposes grafana k8s_resource( @@ -281,12 +320,15 @@ k8s_resource( workload="dependencies-grafana", extra_pod_selectors=[{"app.kubernetes.io/name": "grafana"}], port_forwards=["42000:3000"], - labels=["monitoring"] + labels=["monitoring"], ) # E2E test button -test_go('e2e-tests', '{root_dir}/e2e/tests'.format(root_dir=root_dir), '.', - extra_args=["-v", "-count=1", "-tags=e2e"], - labels=['e2e-tests'], - trigger_mode=TRIGGER_MODE_MANUAL, +test_go( + "e2e-tests", + "{root_dir}/e2e/tests".format(root_dir=root_dir), + ".", + extra_args=["-v", "-count=1", "-tags=e2e"], + labels=["e2e-tests"], + trigger_mode=TRIGGER_MODE_MANUAL, ) diff --git a/build/scripts/watch.sh b/build/scripts/watch.sh index 01d55d544..b2fbdd892 100755 --- a/build/scripts/watch.sh +++ b/build/scripts/watch.sh @@ -19,7 +19,8 @@ else fi reflex \ - --start-service \ - -r '\.go' \ - --decoration="none" \ - -s -- sh -c "$command"; + --start-service \ + -R '^app/client' \ + -r '\.go' \ + --decoration="none" \ + -s -- sh -c "$command" diff --git a/build/scripts/watch_build.sh b/build/scripts/watch_build.sh deleted file mode 100755 index 5f5e5b920..000000000 --- a/build/scripts/watch_build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -if command -v reflex >/dev/null -then - reflex -r '\.go$' -s --decoration="none" -- sh -c "go build -v app/pocket/main.go" -else - echo "reflex not found. Install with `go install github.com/cespare/reflex@latest`" -fi diff --git a/consensus/module_consensus_debugging.go b/consensus/module_consensus_debugging.go index a302f1444..137643df6 100644 --- a/consensus/module_consensus_debugging.go +++ b/consensus/module_consensus_debugging.go @@ -13,11 +13,11 @@ func (m *consensusModule) HandleDebugMessage(debugMessage *messaging.DebugMessag m.m.Lock() defer m.m.Unlock() + m.logger.Debug().Msgf("Consensus module handling debug message: %s", debugMessage.Action) + switch debugMessage.Action { case messaging.DebugMessageAction_DEBUG_CONSENSUS_RESET_TO_GENESIS: - if err := m.resetToGenesis(debugMessage); err != nil { - return err - } + return m.resetToGenesis(debugMessage) case messaging.DebugMessageAction_DEBUG_CONSENSUS_PRINT_NODE_STATE: m.printNodeState(debugMessage) case messaging.DebugMessageAction_DEBUG_CONSENSUS_TRIGGER_NEXT_VIEW: diff --git a/docs/demos/iteration_3_end_to_end_tx.md b/docs/demos/iteration_3_end_to_end_tx.md index 1ace95335..86d2dab19 100644 --- a/docs/demos/iteration_3_end_to_end_tx.md +++ b/docs/demos/iteration_3_end_to_end_tx.md @@ -43,13 +43,13 @@ make protogen_local # generate the protobuf files make generate_rpc_openapi # generate the OpenAPI spec make docker_wipe_nodes # clear all the 4 validator nodes make db_drop # clear the existing database -make compose_and_watch # Start 4 validator node LocalNet +make lightweight_localnet # Start 4 validator node LocalNet ``` ## Shell #2: Setup Consensus debugger ```bash -make client_start && make client_connect # start the consensus debugger +make lightweight_localnet_client && make lightweight_localnet_client_debug # start the consensus debugger ``` Use `TriggerNextView` and `PrintNodeState` to increment and inspect each node's `height/round/step`. diff --git a/docs/development/FAQ.md b/docs/development/FAQ.md index 4d4eaf1ff..8b7de79a7 100644 --- a/docs/development/FAQ.md +++ b/docs/development/FAQ.md @@ -11,9 +11,9 @@ _NOTE: Consider turning off the `gofmt` in your IDE to prevent unexpected format ## Unable to start LocalNet - permission denied -- **Issue**: when trying to run `make compose_and_watch` on an operating system with SELinux, the command gives the error: +- **Issue**: when trying to run `make lightweight_localnet` on an operating system with SELinux, the command gives the error: -``` +```bash Recreating validator2 ... done Recreating validator4 ... done Recreating validator1 ... done diff --git a/docs/development/README.md b/docs/development/README.md index 3f62d8a6b..84dfad020 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -86,6 +86,7 @@ Optionally activate changelog pre-commit hook cp .githooks/pre-commit .git/hooks/pre-commit chmod +x .git/hooks/pre-commit ``` + _**NOTE**: The pre-commit changelog verification has been disabled during the developement of V1 as of 2023-05-16 to unblock development velocity; see more details [here](https://github.com/pokt-network/pocket/assets/1892194/394fdb09-e388-44aa-820d-e9d5a23578cf). This check is no longer done in the CI and is not recommended for local development either currently._ ### Pocket Network CLI @@ -167,7 +168,7 @@ Note that there are a few tests in the library that are prone to race conditions ### Running LocalNet -At the time of writing, we have two basic approaches to running a LocalNet. We suggest getting started with the `Docker Compose` approach outlined below before moving to the advanced Kubernetes configuration. +At the time of writing, we have two basic approaches to running a LocalNet. We suggest getting started with the `Docker Compose` (aka `lightweight LocalNet`) approach outlined below before moving to the advanced Kubernetes (aka LocalNet) configuration. #### [Advanced] Kubernetes @@ -186,13 +187,13 @@ make docker_wipe 2. In one shell, run the 4 nodes setup: ```bash -make compose_and_watch +make lightweight_localnet ``` 4. In another shell, run the development client: ```bash -make client_start && make client_connect +make lightweight_localnet_client && make lightweight_localnet_client_debug ``` 4. Check the state of each node: diff --git a/e2e/README.md b/e2e/README.md index a87c4fcf2..5e3ee41c9 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -7,6 +7,7 @@ - [Build Tags](#build-tags) - [Issue templates](#issue-templates) - [Implementation](#implementation) +- [Keywords](#keywords) > tl; dr - `make localnet_up` and then `make test_e2e` @@ -35,8 +36,8 @@ Issues can formally define features by attaching an erroring `feature` file to b ```gherkin Feature: Example Namespace - Scenario: User Needs Example - Given the user has a validator + Scenario: User Needs Example + Given the user has a node When the user runs the command "example" Then the user should be able to see standard output containing "Example Output" And the pocket client should have exited without error @@ -46,7 +47,7 @@ Feature: Example Namespace The test suite is located in `e2e/tests` and it contains a set of Cucumber feature files and the associated Go tests to run them. `make test_e2e` sees any files named with the pattern `*.feature` in `e2e/tests` and runs them with [godog](https://github.com/cucumber/godog), the Go test runner for Cucumber tests. The LocalNet must be up and running for the E2E test suite to run. -The Validator issues RPC commands on the container by calling `kubectl exec` and targeting the pod in the cluster by name. It records the results of the command including stdout and stderr, allowing for assertions about the results of the command. +The Node issues RPC commands on the container by calling `kubectl exec` and targeting the pod in the cluster by name. It records the results of the command including stdout and stderr, allowing for assertions about the results of the command. ```mermaid --- @@ -60,10 +61,26 @@ flowchart TD Kubeconfig --> Kubectl Kubeconfig --> DevNet subgraph E2E [E2E scenarios] - Kubectl -- commandResult --> Validator - Validator -- args --> Kubectl + Kubectl -- commandResult --> Node + Node -- args --> Kubectl end subgraph DevNet [DevNet] Runner[E2E Test Runner] end ``` + +## Keywords + +The keywords below are a summary of the source documentation available [here](https://cucumber.io/docs/gherkin/reference/#keywords). + +- **Feature**: This keyword, followed by the name and optional description, is used to describe a feature of the system that you're testing. It should provide a high-level description of a software feature, and to group related scenarios. +- **Scenario**: This keyword, followed by the name and optional description, is used to describe a particular behavior of the system that you're testing. A feature can have multiple scenarios, and each scenario should follow the 'Given-When-Then' structure. +- **Given**: This keyword is used to set up a situation or a context. It puts the system in a known state before the user interacts with the system. +- **When**: This keyword is used to describe an action or event. This is something the user does or the system does. +- **Then**: This keyword is used to describe an expected outcome or result. +- **And**, But: These keywords are used when you have more than one Given, When, or Then step. They help to make the specifications more readable. +- **Background**: This keyword provides the context for the following scenarios. It allows you to add some context to the scenarios in a single place. +- **Scenario Outline**: This keyword can be used when the same test is performed multiple times with a different combination of values. +- **Examples**: This keyword is used in conjunction with **Scenario Outline** to provide the values for the test. +- **Rule**: This keyword is used to represent one business rule that should be implemented. It provides additional information for a feature. +- **Tags**: This is not a Gherkin keyword but an integral part of organizing your Cucumber features. They are preceded by '@' symbol and can be used before Feature, Scenario, Scenario Outline, or Examples. diff --git a/e2e/docs/E2E_ADR.md b/e2e/docs/E2E_ADR.md index d3e7dee53..ecefeda62 100644 --- a/e2e/docs/E2E_ADR.md +++ b/e2e/docs/E2E_ADR.md @@ -79,7 +79,7 @@ Below is an example of testing the `help` command of the Pocket binary. Feature: Root Namespace Scenario: User Needs Help - Given the user has a validator + Given the user has a node When the user runs the command "help" Then the user should be able to see standard output containing "Available Commands" And the pocket client should have exited without error @@ -124,16 +124,16 @@ type PocketClient interface { ``` - The `PocketClient` interface is included in the test suite and defines a single function interface with the `RunCommand` method. -- The `validatorPod` adapter fulfills the `PocketClient` interface and lets us call commands through Kubernetes. This is the main way that tests assemble the environment for later assertions. +- The `nodePod` adapter fulfills the `PocketClient` interface and lets us call commands through Kubernetes. This is the main way that tests assemble the environment for later assertions. ```go -// validatorPod holds the connection information to pod validator-001 for testing -type validatorPod struct { +// nodePod holds the connection information to pod validator-001 for testing +type nodePod struct { result *commandResult // stores the result of the last command that was run } // RunCommand runs a command on the pocket binary -func (v *validatorPod) RunCommand(args ...string) (*commandResult, error) { +func (v *nodePod) RunCommand(args ...string) (*commandResult, error) { base := []string{ "exec", "-i", "deploy/pocket-v1-cli-client", "--container", "pocket", diff --git a/e2e/tests/account.feature b/e2e/tests/account.feature new file mode 100644 index 000000000..8e793dcd9 --- /dev/null +++ b/e2e/tests/account.feature @@ -0,0 +1,27 @@ +Feature: Node Namespace + + Scenario: User Wants Help Using The Node Command + Given the user has a node + When the user runs the command "Validator help" + Then the user should be able to see standard output containing "Available Commands" + And the node should have exited without error + + Scenario: User Can Stake A Validator + Given the user has a node + When the user stakes their validator with amount 150000000001 uPOKT + Then the user should be able to see standard output containing "" + And the node should have exited without error + + Scenario: User Can Unstake A Validator + Given the user has a node + When the user stakes their validator with amount 150000000001 uPOKT + Then the user should be able to see standard output containing "" + Then the user should be able to unstake their validator + Then the user should be able to see standard output containing "" + And the node should have exited without error + + Scenario: User Can Send To An Address + Given the user has a node + When the user sends 150000000 uPOKT to another address + Then the user should be able to see standard output containing "" + And the node should have exited without error diff --git a/e2e/tests/debug.feature b/e2e/tests/debug.feature new file mode 100644 index 000000000..38026897c --- /dev/null +++ b/e2e/tests/debug.feature @@ -0,0 +1,18 @@ +Feature: Debug Namespace + + # IMPROVE(#959): Remove time-based waits from tests + + # Since the configuration for consensus is optimistically responsive, we need to be in manual + # Pacemaker mode and call TriggerView to further the blockchain. + # 1 second was chosen arbitrarily for the time for block propagation. + Scenario: 4 Validator blockchain from genesis reaches block 2 when TriggerView is executed twice + Given the network is at genesis + And the network has "4" actors of type "Validator" + When the developer runs the command "TriggerView" + And the developer waits for "1000" milliseconds + Then "validator-001" should be at height "1" + And "validator-004" should be at height "1" + When the developer runs the command "TriggerView" + And the developer waits for "1000" milliseconds + Then "validator-001" should be at height "2" + And "validator-004" should be at height "2" \ No newline at end of file diff --git a/e2e/tests/node.go b/e2e/tests/node.go new file mode 100644 index 000000000..422e6e009 --- /dev/null +++ b/e2e/tests/node.go @@ -0,0 +1,76 @@ +//go:build e2e + +package e2e + +import ( + "fmt" + "os/exec" + + "github.com/pokt-network/pocket/runtime" + "github.com/pokt-network/pocket/runtime/defaults" +) + +// cliPath is the path of the binary installed and is set by the Tiltfile +const cliPath = "/usr/local/bin/p1" + +var ( + // defaultRPCURL used by targetPod to build commands + defaultRPCURL string + // targetDevClientPod is the kube pod that executes calls to the pocket binary under test + targetDevClientPod = "deploy/dev-cli-client" +) + +func init() { + defaultRPCHost := runtime.GetEnv("RPC_HOST", defaults.RandomValidatorEndpointK8SHostname) + defaultRPCURL = fmt.Sprintf("http://%s:%s", defaultRPCHost, defaults.DefaultRPCPort) +} + +// commandResult combines the stdout, stderr, and err of an operation +type commandResult struct { + Stdout string + Stderr string + Err error +} + +// PocketClient is a single function interface for interacting with a node +type PocketClient interface { + RunCommand(...string) (*commandResult, error) + RunCommandOnHost(string, ...string) (*commandResult, error) +} + +// Ensure that Validator fulfills PocketClient +var _ PocketClient = &nodePod{} + +// nodePod holds the connection information to a specific pod in between different instructions during testing +type nodePod struct { + targetPodName string + result *commandResult // stores the result of the last command that was run +} + +// RunCommand runs a command on a pre-configured kube pod with the given args +func (n *nodePod) RunCommand(args ...string) (*commandResult, error) { + return n.RunCommandOnHost(defaultRPCURL, args...) +} + +// RunCommandOnHost runs a command on specified kube pod with the given args +func (n *nodePod) RunCommandOnHost(rpcUrl string, args ...string) (*commandResult, error) { + base := []string{ + "exec", "-i", targetDevClientPod, + "--container", "pocket", + "--", cliPath, + "--non_interactive=true", + "--remote_cli_url=" + rpcUrl, + } + args = append(base, args...) + cmd := exec.Command("kubectl", args...) + r := &commandResult{} + out, err := cmd.Output() + if err != nil { + return nil, err + } + r.Stdout = string(out) + n.result = r + // IMPROVE: make targetPodName configurable + n.targetPodName = targetDevClientPod + return r, nil +} diff --git a/e2e/tests/query.feature b/e2e/tests/query.feature index 91e3e4eb9..74cc60180 100644 --- a/e2e/tests/query.feature +++ b/e2e/tests/query.feature @@ -1,14 +1,14 @@ Feature: Query Namespace - Scenario: User Wants Help Using The Query Command - Given the user has a validator + Scenario: User Wants Help Using The Query Command + Given the user has a node When the user runs the command "Query help" Then the user should be able to see standard output containing "Available Commands" - And the validator should have exited without error + And the node should have exited without error Scenario: User Wants To See The Block At Current Height - Given the user has a validator + Given the user has a node When the user runs the command "Query Block" Then the user should be able to see standard output containing "state_hash" - And the validator should have exited without error \ No newline at end of file + And the node should have exited without error \ No newline at end of file diff --git a/e2e/tests/root.feature b/e2e/tests/root.feature index 754534f2e..b9d6225d4 100644 --- a/e2e/tests/root.feature +++ b/e2e/tests/root.feature @@ -1,7 +1,7 @@ Feature: Root Namespace Scenario: User Needs Help - Given the user has a validator + Given the user has a node When the user runs the command "help" Then the user should be able to see standard output containing "Available Commands" - And the validator should have exited without error \ No newline at end of file + And the node should have exited without error \ No newline at end of file diff --git a/e2e/tests/state_sync.feature b/e2e/tests/state_sync.feature new file mode 100644 index 000000000..1aa85fe0e --- /dev/null +++ b/e2e/tests/state_sync.feature @@ -0,0 +1,23 @@ +Feature: State Sync Namespace + + # IMPROVE(#959): Remove time-based waits from tests + # TODO(#964): Remove the `skip_in_ci` tag for these tests + @skip_in_ci + Scenario: New FullNode does not sync to Blockchain at height 2 + Given the network is at genesis + And the network has "4" actors of type "Validator" + When the developer runs the command "ScaleActor full_nodes 1" + And the developer waits for "3000" milliseconds + Then "full-node-002" should be unreachable + When the developer runs the command "TriggerView" + And the developer waits for "1000" milliseconds + And the developer runs the command "TriggerView" + And the developer waits for "1000" milliseconds + Then "validator-001" should be at height "2" + And "validator-004" should be at height "2" + # full_nodes is the key used in `localnet_config.yaml` + When the developer runs the command "ScaleActor full_nodes 2" + # IMPROVE: Figure out if there's something better to do then waiting for a node to spin up + And the developer waits for "40000" milliseconds + # TODO(#812): The full node should be at height "2" after state sync is implemented + Then "full-node-002" should be at height "0" \ No newline at end of file diff --git a/e2e/tests/steps_init_test.go b/e2e/tests/steps_init_test.go index ee680cd82..1f83171f1 100644 --- a/e2e/tests/steps_init_test.go +++ b/e2e/tests/steps_init_test.go @@ -3,11 +3,13 @@ package e2e import ( + "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" + "time" pocketLogger "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/runtime/defaults" @@ -15,6 +17,8 @@ import ( pocketk8s "github.com/pokt-network/pocket/shared/k8s" "github.com/regen-network/gocuke" "github.com/stretchr/testify/require" + "golang.org/x/text/cases" + "golang.org/x/text/language" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -25,13 +29,12 @@ var e2eLogger = pocketLogger.Global.CreateLoggerForModule("e2e") const ( // Each actor is represented e.g. validator-001-pocket:42069 thru validator-999-pocket:42069. // Defines the host & port scheme that LocalNet uses for naming actors. - validatorServiceURLTmpl = "validator-%s-pocket:%d" - // validatorA maps to suffix ID 001 and is also used by the cluster-manager - // though it has no special permissions. + validatorServiceURLTemplate = "validator-%s-pocket:%d" + // Mapping from validators to suffix IDs as convienece for some of the tests validatorA = "001" - // validatorB maps to suffix ID 002 and receives in the Send test. validatorB = "002" - chainId = "0001" + // Placeholder chainID + chainId = "0001" ) type rootSuite struct { @@ -42,53 +45,164 @@ type rootSuite struct { validatorKeys map[string]string // clientset is the kubernetes API we acquire from the user's $HOME/.kube/config clientset *kubernetes.Clientset - // validator holds command results between runs and reports errors to the test suite - // TECHDEBT: Rename `validator` to something more appropriate - validator *validatorPod - // validatorA maps to suffix ID 001 of the kube pod that we use as our control agent + // node holds command results between runs and reports errors to the test suite + node *nodePod } func (s *rootSuite) Before() { clientSet, err := getClientset(s) require.NoErrorf(s, err, "failed to get clientset") - vkmap, err := pocketk8s.FetchValidatorPrivateKeys(clientSet) + validatorKeyMap, err := pocketk8s.FetchValidatorPrivateKeys(clientSet) if err != nil { e2eLogger.Fatal().Err(err).Msg("failed to get validator key map") } - s.validator = new(validatorPod) + s.node = new(nodePod) s.clientset = clientSet - s.validatorKeys = vkmap + s.validatorKeys = validatorKeyMap } // TestFeatures runs the e2e tests specified in any .features files in this directory // * This test suite assumes that a LocalNet is running that can be accessed by `kubectl` func TestFeatures(t *testing.T) { - gocuke.NewRunner(t, &rootSuite{}).Path("*.feature").Run() + e2eTestTags := os.Getenv("POCKET_E2E_TEST_TAGS") + gocuke.NewRunner(t, &rootSuite{}).Path("*.feature").Tags(e2eTestTags).Run() } // InitializeScenario registers step regexes to function handlers -func (s *rootSuite) TheUserHasAValidator() { - res, err := s.validator.RunCommand("help") +func (s *rootSuite) TheUserHasANode() { + res, err := s.node.RunCommand("help") require.NoErrorf(s, err, res.Stderr) - s.validator.result = res + s.node.result = res } -func (s *rootSuite) TheValidatorShouldHaveExitedWithoutError() { - require.NoError(s, s.validator.result.Err) +func (s *rootSuite) TheNodeShouldHaveExitedWithoutError() { + require.NoError(s, s.node.result.Err) } func (s *rootSuite) TheUserRunsTheCommand(cmd string) { cmds := strings.Split(cmd, " ") - res, err := s.validator.RunCommand(cmds...) + res, err := s.node.RunCommand(cmds...) require.NoError(s, err) - s.validator.result = res + s.node.result = res +} + +// TheDeveloperRunsTheCommand is similar to TheUserRunsTheCommand but exclusive to `Debug` commands +func (s *rootSuite) TheDeveloperRunsTheCommand(cmd string) { + cmds := strings.Split(cmd, " ") + cmds = append([]string{"Debug"}, cmds...) + res, err := s.node.RunCommand(cmds...) + require.NoError(s, err, fmt.Sprintf("failed to run command: '%s' due to error: %s", cmd, err)) + s.node.result = res + e2eLogger.Debug().Msgf("TheDeveloperRunsTheCommand: '%s' with result: %s", cmd, res.Stdout) + + // Special case for managing LocalNet config when scaling actors + if cmds[1] == "ScaleActor" { + s.syncLocalNetConfigFromHostToLocalFS() + } +} + +func (s *rootSuite) TheNetworkIsAtGenesis() { + s.TheDeveloperRunsTheCommand("ResetToGenesis") +} + +func (s *rootSuite) TheDeveloperWaitsForMilliseconds(millis int64) { + time.Sleep(time.Duration(millis) * time.Millisecond) +} + +func (s *rootSuite) TheNetworkHasActorsOfType(num int64, actor string) { + // normalize actor to Title case and plural + caser := cases.Title(language.AmericanEnglish) + actor = caser.String(strings.ToLower(actor)) + if len(actor) > 0 && actor[len(actor)-1] != 's' { + actor += "s" + } + args := []string{ + "Query", + actor, + } + + // Depending on the type of `actor` we're querying, we'll have a different set of expected responses + // so not all of these fields will be populated, but at least one will be. + type expectedResponse struct { + NumValidators *int64 `json:"total_validators"` + NumApps *int64 `json:"total_apps"` + NumFishermen *int64 `json:"total_fishermen"` + NumServicers *int64 `json:"total_servicers"` + NumAccounts *int64 `json:"total_accounts"` + } + validate := func(res *expectedResponse) bool { + return res != nil && ((res.NumValidators != nil && *res.NumValidators > 0) || + (res.NumApps != nil && *res.NumApps > 0) || + (res.NumFishermen != nil && *res.NumFishermen > 0) || + (res.NumServicers != nil && *res.NumServicers > 0) || + (res.NumAccounts != nil && *res.NumAccounts > 0)) + } + + resRaw, err := s.node.RunCommand(args...) + require.NoError(s, err) + + res := getResponseFromStdout[expectedResponse](s, resRaw.Stdout, validate) + require.NotNil(s, res) + + // Validate that at least one of the fields that is populated has the right number of actors + if res.NumValidators != nil { + require.Equal(s, num, *res.NumValidators) + } else if res.NumApps != nil { + require.Equal(s, num, *res.NumApps) + } else if res.NumFishermen != nil { + require.Equal(s, num, *res.NumFishermen) + } else if res.NumServicers != nil { + require.Equal(s, num, *res.NumServicers) + } else if res.NumAccounts != nil { + require.Equal(s, num, *res.NumAccounts) + } +} + +func (s *rootSuite) ShouldBeUnreachable(pod string) { + validate := func(res string) bool { + return strings.Contains(res, "Unable to connect to the RPC") + } + args := []string{ + "Query", + "Height", + } + rpcURL := fmt.Sprintf("http://%s-pocket:%s", pod, defaults.DefaultRPCPort) + resRaw, err := s.node.RunCommandOnHost(rpcURL, args...) + require.NoError(s, err) + + res := getStrFromStdout(s, resRaw.Stdout, validate) + require.NotNil(s, res) + + require.Equal(s, fmt.Sprintf("❌ Unable to connect to the RPC @ \x1b[1mhttp://%s-pocket:%s\x1b[0m", pod, defaults.DefaultRPCPort), *res) +} + +func (s *rootSuite) ShouldBeAtHeight(pod string, height int64) { + args := []string{ + "Query", + "Height", + } + type expectedResponse struct { + Height *int64 `json:"Height"` + } + validate := func(res *expectedResponse) bool { + return res != nil && res.Height != nil + } + + rpcURL := fmt.Sprintf("http://%s-pocket:%s", pod, defaults.DefaultRPCPort) + resRaw, err := s.node.RunCommandOnHost(rpcURL, args...) + require.NoError(s, err) + + res := getResponseFromStdout[expectedResponse](s, resRaw.Stdout, validate) + require.NotNil(s, res) + + require.Equal(s, height, *res.Height) } func (s *rootSuite) TheUserShouldBeAbleToSeeStandardOutputContaining(arg1 string) { - require.Contains(s, s.validator.result.Stdout, arg1) + require.Contains(s, s.node.result.Stdout, arg1) } func (s *rootSuite) TheUserStakesTheirValidatorWithAmountUpokt(amount int64) { @@ -111,15 +225,15 @@ func (s *rootSuite) TheUserSendsUpoktToAnotherAddress(amount int64) { valB.Address().String(), fmt.Sprintf("%d", amount), } - res, err := s.validator.RunCommand(args...) + res, err := s.node.RunCommand(args...) require.NoError(s, err) - s.validator.result = res + s.node.result = res } // stakeValidator runs Validator stake command with the address, amount, chains..., and serviceURL provided func (s *rootSuite) stakeValidator(privKey cryptoPocket.PrivateKey, amount string) { - validatorServiceUrl := fmt.Sprintf(validatorServiceURLTmpl, validatorA, defaults.DefaultP2PPort) + validatorServiceUrl := fmt.Sprintf(validatorServiceURLTemplate, validatorA, defaults.DefaultP2PPort) args := []string{ "Validator", "Stake", @@ -128,10 +242,10 @@ func (s *rootSuite) stakeValidator(privKey cryptoPocket.PrivateKey, amount strin chainId, validatorServiceUrl, } - res, err := s.validator.RunCommand(args...) + res, err := s.node.RunCommand(args...) require.NoError(s, err) - s.validator.result = res + s.node.result = res } // unstakeValidator unstakes the Validator at the same address that stakeValidator uses @@ -142,10 +256,10 @@ func (s *rootSuite) unstakeValidator() { "Unstake", privKey.Address().String(), } - res, err := s.validator.RunCommand(args...) + res, err := s.node.RunCommand(args...) require.NoError(s, err) - s.validator.result = res + s.node.result = res } // getPrivateKey generates a new keypair from the private hex key that we get from the clientset @@ -190,3 +304,39 @@ func inClusterConfig(t gocuke.TestingT) *rest.Config { return config } + +// getResponseFromStdout returns the first output from stdout that passes the validate function provided. +// For example, when running `p1 Query Height`, the output is: +// +// {"level":"info","module":"e2e","time":"2023-07-11T15:46:07-07:00","message":"..."} +// {"height":3} +// +// And will return the following map so it can be used by the caller: +// +// map[height:3] +func getResponseFromStdout[T any](t gocuke.TestingT, stdout string, validate func(res *T) bool) *T { + t.Helper() + + for _, s := range strings.Split(stdout, "\n") { + var m T + if err := json.Unmarshal([]byte(s), &m); err != nil { + continue + } + if !validate(&m) { + continue + } + return &m + } + return nil +} + +func getStrFromStdout(t gocuke.TestingT, stdout string, validate func(res string) bool) *string { + t.Helper() + for _, s := range strings.Split(stdout, "\n") { + if !validate(s) { + continue + } + return &s + } + return nil +} diff --git a/e2e/tests/tilt_helpers.go b/e2e/tests/tilt_helpers.go new file mode 100644 index 000000000..a605ee22e --- /dev/null +++ b/e2e/tests/tilt_helpers.go @@ -0,0 +1,34 @@ +//go:build e2e + +package e2e + +import ( + "log" + "os/exec" +) + +// HACK: Dynamic scaling actors using `p1` and the `e2e test framework` is still a WIP so this is a +// functional interim solution until there's a need for a proper design. +func (s *rootSuite) syncLocalNetConfigFromHostToLocalFS() { + if !isPackageInstalled("tilt") { + e2eLogger.Debug().Msgf("syncLocalNetConfigFromHostToLocalFS: 'tilt' is not installed, skipping...") + return + } + tiltLocalnetConfigSyncbackTrigger := exec.Command("tilt", "trigger", "syncback_localnet_config") + if err := tiltLocalnetConfigSyncbackTrigger.Run(); err != nil { + e2eLogger.Err(err).Msgf("syncLocalNetConfigFromHostToLocalFS: failed to run command: '%s'", tiltLocalnetConfigSyncbackTrigger.String()) + log.Fatal(err) + } +} + +func isPackageInstalled(pkg string) bool { + if _, err := exec.LookPath(pkg); err != nil { + // the executable is not found, return false + if execErr, ok := err.(*exec.Error); ok && execErr.Err == exec.ErrNotFound { + return false + } + // another kind of error happened, let's log and exit + log.Fatal(err) + } + return true +} diff --git a/e2e/tests/valdator.feature b/e2e/tests/validator.feature similarity index 52% rename from e2e/tests/valdator.feature rename to e2e/tests/validator.feature index ec8a2ca47..e1bd22c4f 100644 --- a/e2e/tests/valdator.feature +++ b/e2e/tests/validator.feature @@ -1,28 +1,27 @@ -# TECHDEBT: Validator should eventually be changed to full node or just node. Feature: Validator Namespace - Scenario: User Wants Help Using The Validator Command - Given the user has a validator + Scenario: User Wants Help Using The Validator Command + Given the user has a node When the user runs the command "Validator help" Then the user should be able to see standard output containing "Available Commands" - And the validator should have exited without error + And the node should have exited without error - Scenario: User Can Stake An Address - Given the user has a validator + Scenario: User Can Stake A Validator + Given the user has a node When the user stakes their validator with amount 150000000001 uPOKT Then the user should be able to see standard output containing "" - And the validator should have exited without error + And the node should have exited without error - Scenario: User Can Unstake An Address - Given the user has a validator + Scenario: User Can Unstake A Validator + Given the user has a node When the user stakes their validator with amount 150000000001 uPOKT Then the user should be able to see standard output containing "" - Then the user should be able to unstake their validator + Then the user should be able to unstake their validator Then the user should be able to see standard output containing "" - And the validator should have exited without error + And the node should have exited without error Scenario: User Can Send To An Address - Given the user has a validator + Given the user has a node When the user sends 150000000 uPOKT to another address Then the user should be able to see standard output containing "" - And the validator should have exited without error + And the node should have exited without error diff --git a/e2e/tests/validator.go b/e2e/tests/validator.go deleted file mode 100644 index 04b27bf7f..000000000 --- a/e2e/tests/validator.go +++ /dev/null @@ -1,67 +0,0 @@ -//go:build e2e - -package e2e - -import ( - "fmt" - "os/exec" - - "github.com/pokt-network/pocket/runtime" - "github.com/pokt-network/pocket/runtime/defaults" -) - -var ( - // rpcURL used by targetPod to build commands - rpcURL string - // targetPod is the kube pod that executes calls to the pocket binary under test - targetPod = "deploy/dev-cli-client" -) - -func init() { - rpcHost := runtime.GetEnv("RPC_HOST", defaults.RandomValidatorEndpointK8SHostname) - rpcURL = fmt.Sprintf("http://%s:%s", rpcHost, defaults.DefaultRPCPort) -} - -// cliPath is the path of the binary installed and is set by the Tiltfile -const cliPath = "/usr/local/bin/p1" - -// commandResult combines the stdout, stderr, and err of an operation -type commandResult struct { - Stdout string - Stderr string - Err error -} - -// PocketClient is a single function interface for interacting with a node -type PocketClient interface { - RunCommand(...string) (*commandResult, error) -} - -// Ensure that Validator fulfills PocketClient -var _ PocketClient = &validatorPod{} - -// validatorPod holds the connection information to pod validator-001 for testing -type validatorPod struct { - result *commandResult // stores the result of the last command that was run -} - -// RunCommand runs a command on a target kube pod -func (v *validatorPod) RunCommand(args ...string) (*commandResult, error) { - base := []string{ - "exec", "-i", targetPod, - "--container", "pocket", - "--", cliPath, - "--non_interactive=true", - "--remote_cli_url=" + rpcURL, - } - args = append(base, args...) - cmd := exec.Command("kubectl", args...) - r := &commandResult{} - out, err := cmd.Output() - r.Stdout = string(out) - v.result = r - if err != nil { - return r, err - } - return r, nil -} diff --git a/go.mod b/go.mod index b5029a9ce..bb68ad331 100644 --- a/go.mod +++ b/go.mod @@ -251,7 +251,7 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/text v0.7.0 golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect golang.org/x/tools v0.3.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/persistence/docs/CHANGELOG.md b/persistence/docs/CHANGELOG.md index 7f6dfb0b3..1016cf5eb 100644 --- a/persistence/docs/CHANGELOG.md +++ b/persistence/docs/CHANGELOG.md @@ -426,7 +426,7 @@ Deprecate PrePersistence - Added PopulateGenesisState function to persistence module - Fixed the stake status iota issue - Discovered and documented (with TODO) double setting parameters issue -- Attached to the Utility Module and using in `make compose_and_watch` +- Attached to the Utility Module and using in `make lightweight_localnet` ## [0.0.0.1] - 2022-07-05 diff --git a/persistence/docs/README.md b/persistence/docs/README.md index d29b5fab0..08d8398e1 100644 --- a/persistence/docs/README.md +++ b/persistence/docs/README.md @@ -99,7 +99,7 @@ A subset of these are explained below. Any targets or helpers to configure and launch the database instances do not populate the actual database. -A LocalNet (see `make compose_and_watch`) must have been executed in order to trigger creation of schemas and hydration of the relevant tables. +A LocalNet (see `make lightweight_localnet`) must have been executed in order to trigger creation of schemas and hydration of the relevant tables. #### CLI Access - db_cli_node diff --git a/shared/modules/doc/CHANGELOG.md b/shared/modules/doc/CHANGELOG.md index d6d965cce..207e7a92a 100644 --- a/shared/modules/doc/CHANGELOG.md +++ b/shared/modules/doc/CHANGELOG.md @@ -125,7 +125,7 @@ UtilityModule - Opened followup issue #163 - Added config and genesis generator to build package - Deprecated old build files -- Use new config and genesis files for make compose_and_watch -- Use new config and genesis files for make client_start && make client_connect +- Use new config and genesis files for make lightweight_localnet +- Use new config and genesis files for make lightweight_localnet_client && make lightweight_localnet_client_debug diff --git a/telemetry/README.md b/telemetry/README.md index eed3bb8b9..fd61c911a 100644 --- a/telemetry/README.md +++ b/telemetry/README.md @@ -158,7 +158,7 @@ make docker_loki_install 1. Spin up the stack ```bash -make compose_and_watch +make lightweight_localnet ``` 2. Wait a few seconds and **Voila!**