Refactoring (NethermindEth#6635)

* Resolved conflicts

* max body size for http json rpc (NethermindEth#36)

* Format

* Self-recovery of TxPool max capacity

* fix ProcessedTransactionsDbCleanerTests

* Update txPool worst value if removal unsuccessful (NethermindEth#6702)

(cherry picked from commit f262cb2)

* Clearer error message for missing starting block state (NethermindEth#6672)

* Update txPool worst value if removal unsuccessful (NethermindEth#6702)

* Add Cancun hard-fork settings for Gnosis (NethermindEth#6709)

* Cancun spec and config for mainnet (NethermindEth#6679)

* Add Paris release spec class for completeness (NethermindEth#6633)

* 0.0.0.0 to +

---------

Co-authored-by: smartprogrammer <[email protected]>
Co-authored-by: Marcin Sobczak <[email protected]>
Co-authored-by: Ben Adams <[email protected]>
Co-authored-by: Ruben Buniatyan <[email protected]>
Co-authored-by: Ahmad Bitar <[email protected]>
Co-authored-by: Kamil Chodoła <[email protected]>
Co-authored-by: Lukasz Rozmej <[email protected]>

src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs

@@ -181,15 +181,13 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
             stopwatch?.Start();
             List<(Block Block, string ExpectedException)> correctRlp = DecodeRlps(test, failOnInvalidRlp);
 
-            if (test.GenesisRlp == null)
-            {
-                test.GenesisRlp = Rlp.Encode(new Block(JsonToEthereumTest.Convert(test.GenesisBlockHeader)));
-            }
+            test.GenesisRlp ??= Rlp.Encode(new Block(JsonToEthereumTest.Convert(test.GenesisBlockHeader)));
 
             Block genesisBlock = Rlp.Decode<Block>(test.GenesisRlp.Bytes);
             Assert.That(genesisBlock.Header.Hash, Is.EqualTo(new Hash256(test.GenesisBlockHeader.Hash)));
 
             ManualResetEvent genesisProcessed = new(false);
+
             blockTree.NewHeadBlock += (_, args) =>
             {
                 if (args.Block.Number == 0)
@@ -271,7 +269,6 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
                     {
                         Assert.That(suggestedBlock.Header.Hash, Is.EqualTo(new Hash256(testBlockJson.BlockHeader.Hash)));
 
-
                         for (int uncleIndex = 0; uncleIndex < suggestedBlock.Uncles.Length; uncleIndex++)
                         {
                             Assert.That(suggestedBlock.Uncles[uncleIndex].Hash, Is.EqualTo(new Hash256(testBlockJson.UncleHeaders[uncleIndex].Hash)));

src/Nethermind/Nethermind.AuRa.Test/AuRaSealValidatorTests.cs

@@ -144,7 +144,7 @@ TestCaseData GetTestCaseData(
         public (bool, object) validate_params(BlockHeader parentBlock, BlockHeader block, Action<AuRaParameters> modifyParameters, Repeat repeat, bool parentIsHead, bool isValidSealer)
         {
             _blockTree.Head.Returns(parentIsHead ? new Block(parentBlock) : new Block(Build.A.BlockHeader.WithNumber(parentBlock.Number - 1).TestObject));
-            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), block.Beneficiary, block.AuRaStep.Value).Returns(isValidSealer);
+            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), block.Beneficiary, block.AuRaStep.Value, out _).Returns(isValidSealer);
 
             object cause = null;
 

src/Nethermind/Nethermind.AuRa.Test/AuRaSealerTests.cs

@@ -60,15 +60,15 @@ public void Setup()
         public bool can_seal(long auRaStep, bool validSealer)
         {
             _auRaStepCalculator.CurrentStep.Returns(auRaStep);
-            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), _address, auRaStep).Returns(validSealer);
+            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), _address, auRaStep, out _).Returns(validSealer);
             return _auRaSealer.CanSeal(10, _blockTree.Head.Hash);
         }
 
         [Test]
         public async Task seal_can_recover_address()
         {
             _auRaStepCalculator.CurrentStep.Returns(11);
-            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), _address, 11).Returns(true);
+            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), _address, 11, out _).Returns(true);
             Block block = Build.A.Block.WithHeader(Build.A.BlockHeader.WithBeneficiary(_address).WithAura(11, null).TestObject).TestObject;
 
             block = await _auRaSealer.SealBlock(block, CancellationToken.None);

src/Nethermind/Nethermind.AuRa.Test/Validators/ListValidatorTests.cs

@@ -48,7 +48,7 @@ private static IEnumerable ValidateTestCases
 
         [TestCaseSource(nameof(ValidateTestCases))]
         public bool should_validate_correctly(Address address, long index) =>
-            _validSealerStrategy.IsValidSealer(GetListValidator(TestItem.AddressA, TestItem.AddressB).Validators, address, index);
+            _validSealerStrategy.IsValidSealer(GetListValidator(TestItem.AddressA, TestItem.AddressB).Validators, address, index, out _);
 
         [TestCase(1)]
         [TestCase(2)]

src/Nethermind/Nethermind.Blockchain.Test/BlockchainProcessorTests.cs

@@ -78,7 +78,7 @@ public Block[] Process(Hash256 newBranchStateRoot, List<Block> suggestedBlocks,
                     if (blockTracer != NullBlockTracer.Instance)
                     {
                         // this is for block reruns on failure for diag tracing
-                        throw new InvalidBlockException(suggestedBlocks[0]);
+                        throw new InvalidBlockException(suggestedBlocks[0], "wrong tracer");
                     }
 
                     _logger.Info($"Processing {suggestedBlocks.Last().ToString(Block.Format.Short)}");
@@ -96,7 +96,7 @@ public Block[] Process(Hash256 newBranchStateRoot, List<Block> suggestedBlocks,
                                 {
                                     _allowedToFail.Remove(hash);
                                     BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(suggestedBlocks.Last(), Array.Empty<TxReceipt>()));
-                                    throw new InvalidBlockException(suggestedBlock);
+                                    throw new InvalidBlockException(suggestedBlock, "allowed to fail");
                                 }
 
                                 notYet = true;

src/Nethermind/Nethermind.Blockchain/InvalidBlockException.cs

@@ -8,8 +8,11 @@ namespace Nethermind.Blockchain;
 
 public class InvalidBlockException : BlockchainException
 {
-    public InvalidBlockException(Block block, Exception? innerException = null)
-        : base($"Invalid block: {block}", innerException) => InvalidBlock = block;
+    public InvalidBlockException(Block block, string message, Exception? innerException = null)
+        : base($"Invalid block: {block} : {message}", innerException) => InvalidBlock = block.Header;
 
-    public Block InvalidBlock { get; }
+    public InvalidBlockException(BlockHeader block, string message, Exception? innerException = null)
+        : base($"Invalid block: {block} : {message}", innerException) => InvalidBlock = block;
+
+    public BlockHeader InvalidBlock { get; }
 }

src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProcessor.cs

@@ -108,8 +108,9 @@ private void ValidateGasLimit(Block block)
             BlockHeader parentHeader = GetParentHeader(block);
             if (_gasLimitOverride?.IsGasLimitValid(parentHeader, block.GasLimit, out long? expectedGasLimit) == false)
             {
-                if (_logger.IsWarn) _logger.Warn($"Invalid gas limit for block {block.Number}, hash {block.Hash}, expected value from contract {expectedGasLimit}, but found {block.GasLimit}.");
-                throw new InvalidBlockException(block);
+                string reason = $"Invalid gas limit, expected value from contract {expectedGasLimit}, but found {block.GasLimit}";
+                if (_logger.IsWarn) _logger.Warn($"Proposed block is not valid {block.ToString(Block.Format.FullHashAndNumber)}. {reason}.");
+                throw new InvalidBlockException(block, reason);
             }
         }
 
@@ -121,8 +122,9 @@ private void ValidateTxs(Block block)
                 AddingTxEventArgs args = CheckTxPosdaoRules(new AddingTxEventArgs(i, tx, block, block.Transactions));
                 if (args.Action != TxAction.Add)
                 {
-                    if (_logger.IsWarn) _logger.Warn($"Proposed block is not valid {block.ToString(Block.Format.FullHashAndNumber)}. {tx.ToShortString()} doesn't have required permissions. Reason: {args.Reason}.");
-                    throw new InvalidBlockException(block);
+                    string reason = $"{tx.ToShortString()} doesn't have required permissions: {args.Reason}";
+                    if (_logger.IsWarn) _logger.Warn($"Proposed block is not valid {block.ToString(Block.Format.FullHashAndNumber)}. {reason}.");
+                    throw new InvalidBlockException(block, reason);
                 }
             }
         }

src/Nethermind/Nethermind.Consensus.AuRa/AuRaSealValidator.cs

@@ -75,9 +75,9 @@ public bool ValidateParams(BlockHeader parent, BlockHeader header, bool isUncle
                 // no worries we do this validation later before processing the block
                 if (parent.Hash == _blockTree.Head?.Hash)
                 {
-                    if (!_validSealerStrategy.IsValidSealer(_validatorStore.GetValidators(), header.Beneficiary, step))
+                    if (!_validSealerStrategy.IsValidSealer(_validatorStore.GetValidators(), header.Beneficiary, step, out Address expectedAddress))
                     {
-                        if (_logger.IsError) _logger.Error($"Block from incorrect proposer at block {header.ToString(BlockHeader.Format.FullHashAndNumber)}, step {step} from author {header.Beneficiary}.");
+                        if (_logger.IsError) _logger.Error($"Proposed block is not valid {header.ToString(BlockHeader.Format.FullHashAndNumber)}. Incorrect proposer at step {step}, expected {expectedAddress}, but found {header.Beneficiary}.");
                         return false;
                     }
                 }

src/Nethermind/Nethermind.Consensus.AuRa/AuRaSealer.cs

@@ -74,8 +74,8 @@ bool StepNotYetProduced(long step) => !_blockTree.Head.Header.AuRaStep.HasValue
 
             bool IsThisNodeTurn(long step)
             {
-                var validators = _validatorStore.GetValidators();
-                return _validSealerStrategy.IsValidSealer(validators, _signer.Address, step);
+                Address[] validators = _validatorStore.GetValidators();
+                return _validSealerStrategy.IsValidSealer(validators, _signer.Address, step, out _);
             }
 
             long currentStep = _auRaStepCalculator.CurrentStep;

src/Nethermind/Nethermind.Consensus.AuRa/Validators/AuRaValidatorBase.cs

@@ -49,11 +49,12 @@ public virtual void OnBlockProcessingStart(Block block, ProcessingOptions option
             if (!options.ContainsFlag(ProcessingOptions.ProducingBlock) && !block.IsGenesis)
             {
                 var auRaStep = block.Header.AuRaStep.Value;
-                if (!_validSealerStrategy.IsValidSealer(Validators, block.Beneficiary, auRaStep))
+                if (!_validSealerStrategy.IsValidSealer(Validators, block.Beneficiary, auRaStep, out Address expectedAddress))
                 {
-                    if (_logger.IsError) _logger.Error($"Block from incorrect proposer at block {block.ToString(Block.Format.FullHashAndNumber)}, step {auRaStep} from author {block.Beneficiary}.");
+                    string reason = $"Incorrect proposer at step {auRaStep}, expected {expectedAddress}, but found {block.Beneficiary}";
+                    if (_logger.IsError) _logger.Error($"Proposed block is not valid {block.ToString(Block.Format.FullHashAndNumber)}. {reason}.");
                     this.GetReportingValidator().ReportBenign(block.Beneficiary, block.Number, IReportingValidator.BenignCause.IncorrectProposer);
-                    throw new InvalidBlockException(block);
+                    throw new InvalidBlockException(block, reason);
                 }
             }
         }

src/Nethermind/Nethermind.Consensus.AuRa/Validators/IValidSealerStrategy.cs

@@ -14,7 +14,8 @@ public interface IValidSealerStrategy
         /// <param name="validators">Validators at given step.</param>
         /// <param name="address">Address to be checked if its a sealer at this step.</param>
         /// <param name="step">Step to be checked.</param>
+        /// <param name="expectedAddress">Address which should create block in current step.</param>
         /// <returns>'true' if <see cref="address"/> should seal a block at <see cref="step"/> for supplied <see cref="validators"/> collection. Otherwise 'false'.</returns>
-        bool IsValidSealer(IList<Address> validators, Address address, long step);
+        bool IsValidSealer(IList<Address> validators, Address address, long step, out Address expectedAddress);
     }
 }

src/Nethermind/Nethermind.Consensus.AuRa/Validators/ValidSealerStrategy.cs

@@ -9,6 +9,7 @@ namespace Nethermind.Consensus.AuRa.Validators
 {
     public class ValidSealerStrategy : IValidSealerStrategy
     {
-        public bool IsValidSealer(IList<Address> validators, Address address, long step) => validators.GetItemRoundRobin(step) == address;
+        public bool IsValidSealer(IList<Address> validators, Address address, long step, out Address expectedAddress) =>
+            (expectedAddress = validators.GetItemRoundRobin(step)) == address;
     }
 }

src/Nethermind/Nethermind.Consensus/Processing/BlockExtensions.cs

@@ -39,16 +39,11 @@ public static bool TrySetTransactions(this Block block, Transaction[] transactio
             return false;
         }
 
-        public static bool IsByNethermindNode(this Block block)
-        {
-            try
-            {
-                return Encoding.ASCII.GetString(block.ExtraData).Contains(BlocksConfig.DefaultExtraData, StringComparison.InvariantCultureIgnoreCase);
-            }
-            catch (Exception)
-            {
-                return false;
-            }
-        }
+        public static bool IsByNethermindNode(this Block block) => block.Header.IsByNethermindNode();
+
+        public static bool IsByNethermindNode(this BlockHeader block) =>
+            Ascii.IsValid(block.ExtraData)
+            && Encoding.ASCII.GetString(block.ExtraData ?? Array.Empty<byte>())
+                .Contains(BlocksConfig.DefaultExtraData, StringComparison.InvariantCultureIgnoreCase);
     }
 }

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionsExecutor.cs

@@ -109,13 +109,20 @@ protected TxAction ProcessTransaction(
                 }
                 else
                 {
-                    _transactionProcessor.ProcessTransaction(blkCtx, currentTx, receiptsTracer, processingOptions, _stateProvider);
+                    TransactionResult result = _transactionProcessor.ProcessTransaction(blkCtx, currentTx, receiptsTracer, processingOptions, _stateProvider);
 
-                    if (addToBlock)
+                    if (result)
                     {
-                        transactionsInBlock.Add(currentTx);
-                        _transactionProcessed?.Invoke(this,
-                            new TxProcessedEventArgs(index, currentTx, receiptsTracer.TxReceipts[index]));
+                        if (addToBlock)
+                        {
+                            transactionsInBlock.Add(currentTx);
+                            _transactionProcessed?.Invoke(this,
+                                new TxProcessedEventArgs(index, currentTx, receiptsTracer.TxReceipts[index]));
+                        }
+                    }
+                    else
+                    {
+                        args.Set(TxAction.Skip, result.Error!);
                     }
                 }
 

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs

@@ -2,13 +2,17 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
+using Nethermind.Blockchain;
 using Nethermind.Core;
 using Nethermind.Core.Specs;
 using Nethermind.Evm;
 using Nethermind.Evm.Tracing;
 using Nethermind.Evm.TransactionProcessing;
 using Nethermind.State;
+using Metrics = Nethermind.Evm.Metrics;
 
 namespace Nethermind.Consensus.Processing
 {
@@ -34,7 +38,7 @@ public BlockValidationTransactionsExecutor(ITransactionProcessorAdapter transact
 
             public TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processingOptions, BlockReceiptsTracer receiptsTracer, IReleaseSpec spec)
             {
-                Evm.Metrics.ResetBlockStats();
+                Metrics.ResetBlockStats();
                 BlockExecutionContext blkCtx = new(block.Header);
                 for (int i = 0; i < block.Transactions.Length; i++)
                 {
@@ -46,9 +50,17 @@ public TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processing
 
             private void ProcessTransaction(BlockExecutionContext blkCtx, Transaction currentTx, int index, BlockReceiptsTracer receiptsTracer, ProcessingOptions processingOptions)
             {
-                _transactionProcessor.ProcessTransaction(blkCtx, currentTx, receiptsTracer, processingOptions, _stateProvider);
+                TransactionResult result = _transactionProcessor.ProcessTransaction(blkCtx, currentTx, receiptsTracer, processingOptions, _stateProvider);
+                if (!result) ThrowInvalidBlockException(result, blkCtx.Header, currentTx, index);
                 TransactionProcessed?.Invoke(this, new TxProcessedEventArgs(index, currentTx, receiptsTracer.TxReceipts[index]));
             }
+
+            [DoesNotReturn]
+            [StackTraceHidden]
+            private void ThrowInvalidBlockException(TransactionResult result, BlockHeader header, Transaction currentTx, int index)
+            {
+                throw new InvalidBlockException(header, $"Transaction {currentTx.Hash} at index {index} failed with error {result.Error}");
+            }
         }
     }
 }

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs

@@ -214,7 +214,7 @@ private void ValidateProcessedBlock(Block suggestedBlock, ProcessingOptions opti
         {
             if (_logger.IsError) _logger.Error($"Processed block is not valid {suggestedBlock.ToString(Block.Format.FullHashAndNumber)}");
             if (_logger.IsError) _logger.Error($"Suggested block TD: {suggestedBlock.TotalDifficulty}, Suggested block IsPostMerge {suggestedBlock.IsPostMerge}, Block TD: {block.TotalDifficulty}, Block IsPostMerge {block.IsPostMerge}");
-            throw new InvalidBlockException(suggestedBlock);
+            throw new InvalidBlockException(suggestedBlock, "invalid hash after block processing");
         }
     }
 

src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs

@@ -6,6 +6,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Text;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Nethermind.Blockchain;
@@ -497,32 +498,38 @@ void DeleteInvalidBlocks(in ProcessingBranch processingBranch, Hash256 invalidBl
         }
         catch (InvalidBlockException ex)
         {
-            InvalidBlock?.Invoke(this, new IBlockchainProcessor.InvalidBlockEventArgs { InvalidBlock = ex.InvalidBlock, });
-
             invalidBlockHash = ex.InvalidBlock.Hash;
+            Block? invalidBlock = processingBranch.BlocksToProcess.FirstOrDefault(b => b.Hash == invalidBlockHash);
+            if (invalidBlock is not null)
+            {
+                InvalidBlock?.Invoke(this, new IBlockchainProcessor.InvalidBlockEventArgs { InvalidBlock = invalidBlock, });
 
+                BlockTraceDumper.LogDiagnosticRlp(invalidBlock, _logger,
+                    (_options.DumpOptions & DumpOptions.Rlp) != 0,
+                    (_options.DumpOptions & DumpOptions.RlpLog) != 0);
 
-            BlockTraceDumper.LogDiagnosticRlp(ex.InvalidBlock, _logger,
-                (_options.DumpOptions & DumpOptions.Rlp) != 0,
-                (_options.DumpOptions & DumpOptions.RlpLog) != 0);
-
-            TraceFailingBranch(
-                processingBranch,
-                options,
-                new BlockReceiptsTracer(),
-                DumpOptions.Receipts);
+                TraceFailingBranch(
+                    processingBranch,
+                    options,
+                    new BlockReceiptsTracer(),
+                    DumpOptions.Receipts);
 
-            TraceFailingBranch(
-                processingBranch,
-                options,
-                new ParityLikeBlockTracer(ParityTraceTypes.StateDiff | ParityTraceTypes.Trace),
-                DumpOptions.Parity);
+                TraceFailingBranch(
+                    processingBranch,
+                    options,
+                    new ParityLikeBlockTracer(ParityTraceTypes.StateDiff | ParityTraceTypes.Trace),
+                    DumpOptions.Parity);
 
-            TraceFailingBranch(
-                processingBranch,
-                options,
-                new GethLikeBlockMemoryTracer(GethTraceOptions.Default),
-                DumpOptions.Geth);
+                TraceFailingBranch(
+                    processingBranch,
+                    options,
+                    new GethLikeBlockMemoryTracer(GethTraceOptions.Default),
+                    DumpOptions.Geth);
+            }
+            else
+            {
+                if (_logger.IsError) _logger.Error($"Unexpected situation occurred during the handling of an invalid block {ex.InvalidBlock}", ex);
+            }
 
             processedBlocks = null;
         }
@@ -566,7 +573,7 @@ private void PrepareBlocksToProcess(Block suggestedBlock, ProcessingOptions opti
             if (!blocksToProcess[0].IsGenesis)
             {
                 BlockHeader? parentOfFirstBlock = _blockTree.FindHeader(blocksToProcess[0].ParentHash!, BlockTreeLookupOptions.None);
-                if (parentOfFirstBlock == null)
+                if (parentOfFirstBlock is null)
                 {
                     throw new InvalidOperationException("Attempted to process a disconnected blockchain");
                 }