From 4e78f236142b293382c6b0dfffee54d40d0a8d5c Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Fri, 2 Aug 2024 16:31:34 +0300 Subject: [PATCH 1/2] tweak waitForCommit to check seq nrs --- .../ccip_integration_tests/ocr3_node_test.go | 52 +++++++++++++++---- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go b/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go index a373ecb994..4ac56f9fec 100644 --- a/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go +++ b/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go @@ -169,13 +169,17 @@ func TestIntegration_OCR3Nodes(t *testing.T) { } // Wait for the commit reports to be generated and reported on all chains. - numUnis := len(universes) + // Expected min seq nr is numChains-1 because we fire off numChains-1 messages in the first batch + // from each chain to the other chains. + // i.e each chain is connected to numChains-1 other chains. + // We add 1 because the minSeqNr the latest committed sequence number + 1. + expectedMinSeqNr := uint64(len(universes)-1) + 1 var wg sync.WaitGroup for _, uni := range universes { wg.Add(1) go func(uni onchainUniverse) { defer wg.Done() - waitForCommit(t, uni, numUnis, nil) + waitForCommit(t, uni, maps.Keys(universes), nil, expectedMinSeqNr) }(uni) } @@ -192,12 +196,15 @@ func TestIntegration_OCR3Nodes(t *testing.T) { t.Log("Sending ping pong from each chain to each other again for a second time") sendPingPong(t, universes, pingPongs, messageIDs, replayBlocks, 2) + // Expected min seq nr is multiplied by two because we're again firing numChains-1 + // messages from each chain to the other chains. + expectedMinSeqNr *= 2 for _, uni := range universes { startBlock := preRequestBlocks[uni.chainID] wg.Add(1) go func(uni onchainUniverse, startBlock *uint64) { defer wg.Done() - waitForCommit(t, uni, numUnis, startBlock) + waitForCommit(t, uni, maps.Keys(universes), startBlock, expectedMinSeqNr) }(uni, &startBlock) } @@ -287,7 +294,13 @@ func sendPingPong( } } -func waitForCommit(t *testing.T, uni onchainUniverse, numUnis int, startBlock *uint64) { +func waitForCommit( + t *testing.T, + uni onchainUniverse, + allChainIDs []uint64, + startBlock *uint64, + expectedMinSeqNr uint64, +) { sink := make(chan *evm_2_evm_multi_offramp.EVM2EVMMultiOffRampCommitReportAccepted) subscription, err := uni.offramp.WatchCommitReportAccepted(&bind.WatchOpts{ Start: startBlock, @@ -298,19 +311,38 @@ func waitForCommit(t *testing.T, uni onchainUniverse, numUnis int, startBlock *u for { select { case <-time.After(5 * time.Second): - t.Logf("Waiting for commit report on chain id %d (selector %d)", uni.chainID, getSelector(uni.chainID)) + t.Logf("Waiting for commit report on chain id %d (selector %d), expectedMinSeqNr %d", uni.chainID, getSelector(uni.chainID), expectedMinSeqNr) case subErr := <-subscription.Err(): t.Fatalf("Subscription error: %+v", subErr) case report := <-sink: if len(report.Report.MerkleRoots) > 0 { - if len(report.Report.MerkleRoots) == numUnis-1 { - t.Logf("Received commit report with %d merkle roots on chain id %d (selector %d): %+v", - len(report.Report.MerkleRoots), uni.chainID, getSelector(uni.chainID), report) + t.Logf("Received commit report with %d merkle roots on chain id %d (selector %d): %+v", + len(report.Report.MerkleRoots), uni.chainID, getSelector(uni.chainID), func() []string { + var res []string + for _, root := range report.Report.MerkleRoots { + res = append(res, fmt.Sprintf("MerkleRoot{Root: %s, ChainSelector: %d, Interval: [Min:%d, Max: %d]}", + hexutil.Encode(root.MerkleRoot[:]), root.SourceChainSelector, root.Interval.Min, root.Interval.Max)) + } + return res + }()) + gotAll := true + for _, remoteChainID := range allChainIDs { + if remoteChainID == uni.chainID { + continue + } + scc, err := uni.offramp.GetSourceChainConfig(&bind.CallOpts{ + Context: testutils.Context(t), + }, getSelector(remoteChainID)) + require.NoError(t, err) + gotAll = gotAll && scc.MinSeqNr == expectedMinSeqNr + } + if gotAll { + t.Logf("Expected min seq nr reached on chain id %d (selector %d)", + uni.chainID, getSelector(uni.chainID)) return } - t.Fatalf("Received commit report with %d merkle roots, expected %d", len(report.Report.MerkleRoots), numUnis) } else { - t.Logf("Received commit report without merkle roots on chain id %d (selector %d): %+v", uni.chainID, getSelector(uni.chainID), report) + t.Logf("Received commit report without merkle roots on chain id %d (selector %d)", uni.chainID, getSelector(uni.chainID)) } } } From 812da72d1107fed79f013100b97393976f7bdc95 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Fri, 2 Aug 2024 18:52:50 +0300 Subject: [PATCH 2/2] major simplification of integration test * don't use ping pongs, send a single message directly * change assertions to assert for specific (source chain, sequence number) pairs * don't use Watch for ExecutionStateChanged, for some reason it doesn't work consistently --- .../ccip/ccip_integration_tests/helpers.go | 65 ++++- .../ccip_integration_tests/ocr3_node_test.go | 270 ++++++------------ .../ccip_integration_tests/ocr_node_helper.go | 1 - 3 files changed, 144 insertions(+), 192 deletions(-) diff --git a/core/capabilities/ccip/ccip_integration_tests/helpers.go b/core/capabilities/ccip/ccip_integration_tests/helpers.go index 5fb623713f..057e48b3e7 100644 --- a/core/capabilities/ccip/ccip_integration_tests/helpers.go +++ b/core/capabilities/ccip/ccip_integration_tests/helpers.go @@ -25,6 +25,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/evm_2_evm_multi_offramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/evm_2_evm_multi_onramp" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/maybe_revert_message_receiver" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/mock_arm_contract" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/nonce_manager" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ocr3_config_encoder" @@ -42,7 +43,10 @@ import ( ) var ( - homeChainID = chainsel.GETH_TESTNET.EvmChainID + homeChainID = chainsel.GETH_TESTNET.EvmChainID + ccipSendRequestedTopic = evm_2_evm_multi_onramp.EVM2EVMMultiOnRampCCIPSendRequested{}.Topic() + commitReportAcceptedTopic = evm_2_evm_multi_offramp.EVM2EVMMultiOffRampCommitReportAccepted{}.Topic() + executionStateChangedTopic = evm_2_evm_multi_offramp.EVM2EVMMultiOffRampExecutionStateChanged{}.Topic() ) const ( @@ -84,6 +88,43 @@ type onchainUniverse struct { priceRegistry *price_registry.PriceRegistry tokenAdminRegistry *token_admin_registry.TokenAdminRegistry nonceManager *nonce_manager.NonceManager + receiver *maybe_revert_message_receiver.MaybeRevertMessageReceiver +} + +type requestData struct { + destChainSelector uint64 + receiverAddress common.Address + data []byte +} + +func (u *onchainUniverse) SendCCIPRequests(t *testing.T, requestDatas []requestData) { + for _, reqData := range requestDatas { + msg := router.ClientEVM2AnyMessage{ + Receiver: common.LeftPadBytes(reqData.receiverAddress.Bytes(), 32), + Data: reqData.data, + TokenAmounts: nil, // TODO: no tokens for now + FeeToken: u.weth.Address(), + ExtraArgs: nil, // TODO: no extra args for now, falls back to default + } + fee, err := u.router.GetFee(&bind.CallOpts{Context: testutils.Context(t)}, reqData.destChainSelector, msg) + require.NoError(t, err) + _, err = u.weth.Deposit(&bind.TransactOpts{ + From: u.owner.From, + Signer: u.owner.Signer, + Value: fee, + }) + require.NoError(t, err) + u.backend.Commit() + _, err = u.weth.Approve(u.owner, u.router.Address(), fee) + require.NoError(t, err) + u.backend.Commit() + + t.Logf("Sending CCIP request from chain %d (selector %d) to chain selector %d", + u.chainID, getSelector(u.chainID), reqData.destChainSelector) + _, err = u.router.CcipSend(u.owner, reqData.destChainSelector, msg) + require.NoError(t, err) + u.backend.Commit() + } } type chainBase struct { @@ -174,6 +215,16 @@ func createUniverses( offramp, err := evm_2_evm_multi_offramp.NewEVM2EVMMultiOffRamp(offrampAddr, backend) require.NoError(t, err) + receiverAddress, _, _, err := maybe_revert_message_receiver.DeployMaybeRevertMessageReceiver( + owner, + backend, + false, + ) + require.NoError(t, err, "failed to deploy MaybeRevertMessageReceiver on chain id %d", chainID) + backend.Commit() + receiver, err := maybe_revert_message_receiver.NewMaybeRevertMessageReceiver(receiverAddress, backend) + require.NoError(t, err) + universe := onchainUniverse{ backend: backend, owner: owner, @@ -188,6 +239,7 @@ func createUniverses( priceRegistry: priceRegistry, tokenAdminRegistry: tokenAdminRegistry, nonceManager: nonceManager, + receiver: receiver, } // Set up the initial configurations for the contracts setupUniverseBasics(t, universe) @@ -217,9 +269,9 @@ func createUniverses( } // print out topic hashes of relevant events for debugging purposes - t.Logf("Topic hash of CommitReportAccepted: %s", evm_2_evm_multi_offramp.EVM2EVMMultiOffRampCommitReportAccepted{}.Topic().Hex()) - t.Logf("Topic hash of ExecutionStateChanged: %s", evm_2_evm_multi_offramp.EVM2EVMMultiOffRampExecutionStateChanged{}.Topic().Hex()) - t.Logf("Topic hash of SkippedAlreadyExecutedMessage: %s", evm_2_evm_multi_offramp.EVM2EVMMultiOffRampSkippedAlreadyExecutedMessage{}.Topic().Hex()) + t.Logf("Topic hash of CommitReportAccepted: %s", commitReportAcceptedTopic.Hex()) + t.Logf("Topic hash of ExecutionStateChanged: %s", executionStateChangedTopic.Hex()) + t.Logf("Topic hash of CCIPSendRequested: %s", ccipSendRequestedTopic.Hex()) return homeChainUniverse, universes } @@ -268,7 +320,6 @@ func tweakChainTimestamp(t *testing.T, backend *backends.SimulatedBackend, tweak blockTime := time.Unix(int64(backend.Blockchain().CurrentHeader().Time), 0) sinceBlockTime := time.Since(blockTime) diff := sinceBlockTime - tweak - t.Logf("block timestamp before tweaking is %s, sinceBlocktime %s, diff %s", blockTime, sinceBlockTime, diff) err := backend.AdjustTime(diff) require.NoError(t, err, "unable to adjust time on simulated chain") backend.Commit() @@ -391,8 +442,8 @@ func (h *homeChain) AddDON( 10*time.Second, // deltaResend 20*time.Second, // deltaInitial 2*time.Second, // deltaRound - 20*time.Second, // deltaGrace - 10*time.Second, // deltaCertifiedCommitRequest + 2*time.Second, // deltaGrace + 10*time.Second, // deltaCertifiedCommitRequest TODO: whats a good value for this? 10*time.Second, // deltaStage 3, // rmax schedule, diff --git a/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go b/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go index 4ac56f9fec..8f5f2b5434 100644 --- a/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go +++ b/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go @@ -8,13 +8,10 @@ import ( "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/hashicorp/consul/sdk/freeport" "go.uber.org/zap/zapcore" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ping_pong_demo" - + "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/evm_2_evm_multi_offramp" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" @@ -26,6 +23,8 @@ import ( "github.com/stretchr/testify/require" ) +const STATE_SUCCESS = uint8(2) + /* * If you want to debug, set log level to info and use the following commands for easier logs filtering. * @@ -43,7 +42,7 @@ func TestIntegration_OCR3Nodes(t *testing.T) { simulatedBackendBlockTime = 900 * time.Millisecond // Simulated backend blocks committing interval oraclesBootWaitTime = 30 * time.Second // Time to wait for oracles to come up (HACK) fChain = 1 // fChain value for all the chains - oracleLogLevel = zapcore.ErrorLevel // Log level for the oracle / plugins. + oracleLogLevel = zapcore.InfoLevel // Log level for the oracle / plugins. ) t.Logf("creating %d universes", numChains) @@ -114,7 +113,6 @@ func TestIntegration_OCR3Nodes(t *testing.T) { cfgs, err := homeChainUni.ccipConfig.GetAllChainConfigs(callCtx) require.NoError(t, err) - t.Logf("Got all homechain configs %#v", cfgs) require.Len(t, cfgs, numChains) // Create a DON for each chain @@ -145,15 +143,11 @@ func TestIntegration_OCR3Nodes(t *testing.T) { require.NoErrorf(t, tApp.AddJobV2(ctx, &jb), "Wasn't able to create ccip job for node %d", i) } - t.Logf("Initializing PingPong contracts") - pingPongs := initializePingPongContracts(t, universes) - - // NOTE: messageIDs are populated in the sendPingPong function - var messageIDs = make(map[uint64]map[uint64][32]byte) // sourceChain->destChain->messageID - var replayBlocks = make(map[uint64]uint64) // chainID -> blocksToReplay - - t.Logf("Sending ping pong from each chain to each other") - sendPingPong(t, universes, pingPongs, messageIDs, replayBlocks, 1) + t.Logf("Sending ccip requests from each chain to all other chains") + for _, uni := range universes { + requests := genRequestData(uni.chainID, universes) + uni.SendCCIPRequests(t, requests) + } // Wait for the oracles to come up. // TODO: We need some data driven way to do this e.g. wait until LP filters to be registered. @@ -162,218 +156,126 @@ func TestIntegration_OCR3Nodes(t *testing.T) { // Replay the log poller on all the chains so that the logs are in the db. // otherwise the plugins won't pick them up. for _, node := range nodes { - for chainID, replayBlock := range replayBlocks { - t.Logf("Replaying logs for chain %d from block %d", chainID, replayBlock) - require.NoError(t, node.app.ReplayFromBlock(big.NewInt(int64(chainID)), replayBlock, false), "failed to replay logs") + for chainID := range universes { + t.Logf("Replaying logs for chain %d from block %d", chainID, 1) + require.NoError(t, node.app.ReplayFromBlock(big.NewInt(int64(chainID)), 1, false), "failed to replay logs") } } - // Wait for the commit reports to be generated and reported on all chains. - // Expected min seq nr is numChains-1 because we fire off numChains-1 messages in the first batch - // from each chain to the other chains. - // i.e each chain is connected to numChains-1 other chains. - // We add 1 because the minSeqNr the latest committed sequence number + 1. - expectedMinSeqNr := uint64(len(universes)-1) + 1 + // with only one request sent from each chain to each other chain, + // and with sequence numbers on incrementing by 1 on a per-dest chain + // basis, we expect the min sequence number to be 1 on all chains. + expectedSeqNrRange := ccipocr3.NewSeqNumRange(1, 1) var wg sync.WaitGroup for _, uni := range universes { - wg.Add(1) - go func(uni onchainUniverse) { - defer wg.Done() - waitForCommit(t, uni, maps.Keys(universes), nil, expectedMinSeqNr) - }(uni) - } - - tStart := time.Now() - t.Log("Waiting for commit reports") - wg.Wait() - t.Logf("Commit reports received after %s", time.Since(tStart)) - - var preRequestBlocks = make(map[uint64]uint64) - for _, uni := range universes { - preRequestBlocks[uni.chainID] = uni.backend.Blockchain().CurrentBlock().Number.Uint64() - } - - t.Log("Sending ping pong from each chain to each other again for a second time") - sendPingPong(t, universes, pingPongs, messageIDs, replayBlocks, 2) - - // Expected min seq nr is multiplied by two because we're again firing numChains-1 - // messages from each chain to the other chains. - expectedMinSeqNr *= 2 - for _, uni := range universes { - startBlock := preRequestBlocks[uni.chainID] - wg.Add(1) - go func(uni onchainUniverse, startBlock *uint64) { - defer wg.Done() - waitForCommit(t, uni, maps.Keys(universes), startBlock, expectedMinSeqNr) - }(uni, &startBlock) + for remoteSelector := range universes { + if remoteSelector == uni.chainID { + continue + } + wg.Add(1) + go func(uni onchainUniverse, remoteSelector uint64) { + defer wg.Done() + waitForCommitWithInterval(t, uni, getSelector(remoteSelector), expectedSeqNrRange) + }(uni, remoteSelector) + } } - tStart = time.Now() - t.Log("Waiting for second batch of commit reports") + start := time.Now() wg.Wait() - t.Logf("Second batch of commit reports received after %s", time.Since(tStart)) + t.Logf("All chains received the expected commit report in %s", time.Since(start)) - expectedMessagesPerChain := (numChains - 1) * 2 + // with only one request sent from each chain to each other chain, + // all ExecutionStateChanged events should have the sequence number 1. + expectedSeqNr := uint64(1) for _, uni := range universes { - wg.Add(1) - go func(uni onchainUniverse) { - defer wg.Done() - waitForExec(t, uni, expectedMessagesPerChain) - }(uni) + for remoteSelector := range universes { + if remoteSelector == uni.chainID { + continue + } + wg.Add(1) + go func(uni onchainUniverse, remoteSelector uint64) { + defer wg.Done() + waitForExecWithSeqNr(t, uni, getSelector(remoteSelector), expectedSeqNr) + }(uni, remoteSelector) + } } - tStart = time.Now() + start = time.Now() wg.Wait() - t.Logf("Execution state changes received after %s", time.Since(tStart)) + t.Logf("All chains received the expected ExecutionStateChanged event in %s", time.Since(start)) } -func sendPingPong( - t *testing.T, - universes map[uint64]onchainUniverse, - pingPongs map[uint64]map[uint64]*ping_pong_demo.PingPongDemo, - messageIDs map[uint64]map[uint64][32]byte, - replayBlocks map[uint64]uint64, - expectedSeqNum uint64, -) { - for chainID, uni := range universes { - var replayBlock uint64 - for otherChain, pingPong := range pingPongs[chainID] { - t.Log("PingPong From: ", chainID, " To: ", otherChain) - - expNextSeqNr, err1 := uni.onramp.GetExpectedNextSequenceNumber(&bind.CallOpts{}, getSelector(otherChain)) - require.NoError(t, err1) - require.Equal(t, expectedSeqNum, expNextSeqNr, "expected next sequence number should be 1") - - uni.backend.Commit() - - _, err2 := pingPong.StartPingPong(uni.owner) - require.NoError(t, err2) - uni.backend.Commit() - - endBlock := uni.backend.Blockchain().CurrentBlock().Number.Uint64() - logIter, err3 := uni.onramp.FilterCCIPSendRequested(&bind.FilterOpts{ - Start: endBlock - 1, - End: &endBlock, - }, []uint64{getSelector(otherChain)}) - require.NoError(t, err3) - // Iterate until latest event - var count int - for logIter.Next() { - count++ - } - require.Equal(t, 1, count, "expected 1 CCIPSendRequested log only") - - log := logIter.Event - require.Equal(t, getSelector(otherChain), log.DestChainSelector) - require.Equal(t, pingPong.Address(), log.Message.Sender) - chainPingPongAddr := pingPongs[otherChain][chainID].Address().Bytes() - - // Receiver address is abi-encoded if destination is EVM. - paddedAddr := common.LeftPadBytes(chainPingPongAddr, len(log.Message.Receiver)) - require.Equal(t, paddedAddr, log.Message.Receiver) - - // check that sequence number is equal to the expected next sequence number. - // and that the sequence number is bumped in the onramp. - require.Equalf(t, log.Message.Header.SequenceNumber, expNextSeqNr, "incorrect sequence number in CCIPSendRequested event on chain %d", log.DestChainSelector) - newExpNextSeqNr, err := uni.onramp.GetExpectedNextSequenceNumber(&bind.CallOpts{}, getSelector(otherChain)) - require.NoError(t, err) - require.Equal(t, expNextSeqNr+1, newExpNextSeqNr, "expected next sequence number should be bumped by 1") - - _, ok := messageIDs[chainID] - if !ok { - messageIDs[chainID] = make(map[uint64][32]byte) - } - messageIDs[chainID][otherChain] = log.Message.Header.MessageId - - // replay block should be the earliest block that has a ccip message. - if replayBlock == 0 { - replayBlock = endBlock - } +func genRequestData(chainID uint64, universes map[uint64]onchainUniverse) []requestData { + var res []requestData + for destChainID, destUni := range universes { + if destChainID == chainID { + continue } - replayBlocks[chainID] = replayBlock + res = append(res, requestData{ + destChainSelector: getSelector(destChainID), + receiverAddress: destUni.receiver.Address(), + data: []byte(fmt.Sprintf("msg from chain %d to chain %d", chainID, destChainID)), + }) } + return res } -func waitForCommit( +func waitForCommitWithInterval( t *testing.T, uni onchainUniverse, - allChainIDs []uint64, - startBlock *uint64, - expectedMinSeqNr uint64, + expectedSourceChainSelector uint64, + expectedSeqNumRange ccipocr3.SeqNumRange, ) { sink := make(chan *evm_2_evm_multi_offramp.EVM2EVMMultiOffRampCommitReportAccepted) subscription, err := uni.offramp.WatchCommitReportAccepted(&bind.WatchOpts{ - Start: startBlock, Context: testutils.Context(t), }, sink) require.NoError(t, err) for { select { - case <-time.After(5 * time.Second): - t.Logf("Waiting for commit report on chain id %d (selector %d), expectedMinSeqNr %d", uni.chainID, getSelector(uni.chainID), expectedMinSeqNr) + case <-time.After(10 * time.Second): + t.Logf("Waiting for commit report on chain id %d (selector %d) from source selector %d expected seq nr range %s", + uni.chainID, getSelector(uni.chainID), expectedSourceChainSelector, expectedSeqNumRange.String()) case subErr := <-subscription.Err(): t.Fatalf("Subscription error: %+v", subErr) case report := <-sink: if len(report.Report.MerkleRoots) > 0 { - t.Logf("Received commit report with %d merkle roots on chain id %d (selector %d): %+v", - len(report.Report.MerkleRoots), uni.chainID, getSelector(uni.chainID), func() []string { - var res []string - for _, root := range report.Report.MerkleRoots { - res = append(res, fmt.Sprintf("MerkleRoot{Root: %s, ChainSelector: %d, Interval: [Min:%d, Max: %d]}", - hexutil.Encode(root.MerkleRoot[:]), root.SourceChainSelector, root.Interval.Min, root.Interval.Max)) - } - return res - }()) - gotAll := true - for _, remoteChainID := range allChainIDs { - if remoteChainID == uni.chainID { - continue + // Check the interval of sequence numbers and make sure it matches + // the expected range. + for _, mr := range report.Report.MerkleRoots { + if mr.SourceChainSelector == expectedSourceChainSelector && + uint64(expectedSeqNumRange.Start()) == mr.Interval.Min && + uint64(expectedSeqNumRange.End()) == mr.Interval.Max { + t.Logf("Received commit report on chain id %d (selector %d) from source selector %d expected seq nr range %s", + uni.chainID, getSelector(uni.chainID), expectedSourceChainSelector, expectedSeqNumRange.String()) + return } - scc, err := uni.offramp.GetSourceChainConfig(&bind.CallOpts{ - Context: testutils.Context(t), - }, getSelector(remoteChainID)) - require.NoError(t, err) - gotAll = gotAll && scc.MinSeqNr == expectedMinSeqNr } - if gotAll { - t.Logf("Expected min seq nr reached on chain id %d (selector %d)", - uni.chainID, getSelector(uni.chainID)) - return - } - } else { - t.Logf("Received commit report without merkle roots on chain id %d (selector %d)", uni.chainID, getSelector(uni.chainID)) } } } } -func waitForExec(t *testing.T, uni onchainUniverse, numExpected int) { - const STATE_SUCCESS = uint8(2) - - sink := make(chan *evm_2_evm_multi_offramp.EVM2EVMMultiOffRampExecutionStateChanged) - subscription, err := uni.offramp.WatchExecutionStateChanged(&bind.WatchOpts{ - Context: testutils.Context(t), - }, sink, nil, nil, nil) - require.NoError(t, err) - - numGot := 0 +func waitForExecWithSeqNr(t *testing.T, uni onchainUniverse, expectedSourceChainSelector, expectedSeqNr uint64) { for { - select { - case <-time.After(10 * time.Second): - t.Logf("Waiting for %d execution state changes on chain %d (selector %d)", numExpected, uni.chainID, getSelector(uni.chainID)) - case <-subscription.Err(): - t.Fatalf("Subscription error") - case event := <-sink: - t.Logf("Received execution state change for msg id %s, sequence number %d, source chain %d", - hexutil.Encode(event.MessageId[:]), event.SequenceNumber, event.SourceChainSelector) - require.Equal(t, STATE_SUCCESS, event.State) - require.Greater(t, event.SequenceNumber, uint64(0)) - numGot++ - if numGot == numExpected { - t.Logf("Received all %d execution state changes on chain id %d (selector %d)", numExpected, uni.chainID, getSelector(uni.chainID)) - return + scc, err := uni.offramp.GetSourceChainConfig(nil, expectedSourceChainSelector) + require.NoError(t, err) + t.Logf("Waiting for ExecutionStateChanged on chain %d (selector %d) from chain %d with expected sequence number %d, current onchain minSeqNr: %d", + uni.chainID, getSelector(uni.chainID), expectedSourceChainSelector, expectedSeqNr, scc.MinSeqNr) + iter, err := uni.offramp.FilterExecutionStateChanged(nil, []uint64{expectedSourceChainSelector}, []uint64{expectedSeqNr}, nil) + require.NoError(t, err) + var count int + for iter.Next() { + if iter.Event.SequenceNumber == expectedSeqNr && iter.Event.SourceChainSelector == expectedSourceChainSelector { + count++ } } + if count == 1 { + t.Logf("Received ExecutionStateChanged on chain %d (selector %d) from chain %d with expected sequence number %d", + uni.chainID, getSelector(uni.chainID), expectedSourceChainSelector, expectedSeqNr) + return + } + time.Sleep(5 * time.Second) } } diff --git a/core/capabilities/ccip/ccip_integration_tests/ocr_node_helper.go b/core/capabilities/ccip/ccip_integration_tests/ocr_node_helper.go index 757fdd7b58..ec35621bf8 100644 --- a/core/capabilities/ccip/ccip_integration_tests/ocr_node_helper.go +++ b/core/capabilities/ccip/ccip_integration_tests/ocr_node_helper.go @@ -119,7 +119,6 @@ func setupNodeOCR3( ChainOpts: legacyevm.ChainOpts{ AppConfig: cfg, GenEthClient: func(i *big.Int) client.Client { - t.Log("genning eth client for chain id:", i.String()) client, ok := clients[i.Uint64()] if !ok { t.Fatal("no backend for chainID", i)