From d025d8282ac7c18e3cead5202ce0fdd142bf5345 Mon Sep 17 00:00:00 2001 From: allnil Date: Mon, 23 Oct 2023 17:25:54 +0100 Subject: [PATCH 01/32] add structures which implements cache for base_fee_per_gas and used_ratio --- crates/rpc/rpc/src/eth/api/fees.rs | 35 +++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 08822b58403b..6022600d2a95 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -6,12 +6,15 @@ use crate::{ }; use reth_network_api::NetworkInfo; use reth_primitives::{ - basefee::calculate_next_block_base_fee, BlockNumberOrTag, SealedHeader, U256, + basefee::calculate_next_block_base_fee, BlockNumberOrTag, SealedHeader, U256, B256 }; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::{FeeHistory, TxGasAndReward}; use reth_transaction_pool::TransactionPool; use tracing::debug; +use derive_more::{Deref, DerefMut}; +use schnellru::{ByLength, LruMap}; +use std::fmt::{self, Debug, Formatter}; impl EthApi where @@ -48,6 +51,7 @@ where return Ok(FeeHistory::default()) } + // See https://github.com/ethereum/go-ethereum/blob/2754b197c935ee63101cbbca2752338246384fec/eth/gasprice/feehistory.go#L218C8-L225 let max_fee_history = if reward_percentiles.is_none() { self.gas_oracle().config().max_header_history @@ -197,3 +201,32 @@ where Ok(rewards_in_block) } } + + + +/// Wrapper struct for LruMap +#[derive(Deref, DerefMut)] +struct BaseFeePerGasLruCache(LruMap), ByLength>); + +impl Debug for BaseFeePerGasLruCache { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("BaseFeePerGasLruCache") + .field("cache_length", &self.len()) + .field("cache_memory_usage", &self.memory_usage()) + .finish() + } +} + + +/// Wrapper struct for LruMap +#[derive(Deref, DerefMut)] +struct GasUsedRatioLruCache(LruMap), ByLength>); + +impl Debug for GasUsedRatioLruCache { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("GasUsedRatioLruCache") + .field("cache_length", &self.len()) + .field("cache_memory_usage", &self.memory_usage()) + .finish() + } +} From a4b9195e3b0f11f3cc799c1761701ce8e8a8dd31 Mon Sep 17 00:00:00 2001 From: allnil Date: Tue, 24 Oct 2023 22:46:09 +0100 Subject: [PATCH 02/32] prepare blueprint for cache integration for fees values --- crates/rpc/rpc/src/eth/api/fees.rs | 44 +++++++++++++++++++++-------- crates/rpc/rpc/src/eth/cache/mod.rs | 35 +++++++++++++++++++++-- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 6022600d2a95..e152831c05b7 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -1,9 +1,9 @@ -//! Contains RPC handler implementations for fee history. - use crate::{ eth::error::{EthApiError, EthResult}, EthApi, }; +use alloy_primitives::FixedBytes; +use reth_interfaces::RethError; use reth_network_api::NetworkInfo; use reth_primitives::{ basefee::calculate_next_block_base_fee, BlockNumberOrTag, SealedHeader, U256, B256 @@ -51,7 +51,6 @@ where return Ok(FeeHistory::default()) } - // See https://github.com/ethereum/go-ethereum/blob/2754b197c935ee63101cbbca2752338246384fec/eth/gasprice/feehistory.go#L218C8-L225 let max_fee_history = if reward_percentiles.is_none() { self.gas_oracle().config().max_header_history @@ -67,6 +66,7 @@ where ); block_count = max_fee_history } + let Some(end_block) = self.provider().block_number_for_id(newest_block.into())? else { return Err(EthApiError::UnknownBlockNumber) @@ -79,7 +79,8 @@ where block_count = end_block_plus; } - // If reward percentiles were specified, we need to validate that they are monotonically + // If reward percentiles were specified, we + schnellru::Limiterneed to validate that they are monotonically // increasing and 0 <= p <= 100 // // Note: The types used ensure that the percentiles are never < 0 @@ -94,31 +95,50 @@ where // Treat a request for 1 block as a request for `newest_block..=newest_block`, // otherwise `newest_block - 2 // SAFETY: We ensured that block count is capped + let start_block = end_block_plus - block_count; - let headers = self.provider().sealed_headers_range(start_block..=end_block)?; - if headers.len() != block_count as usize { - return Err(EthApiError::InvalidBlockRange) - } + // TO-DO: first we support only in-cache queries + // let headers = self.provider().sealed_headers_range(start_block..=end_block)?; - // Collect base fees, gas usage ratios and (optionally) reward percentile data + // begin let mut base_fee_per_gas: Vec = Vec::new(); let mut gas_used_ratio: Vec = Vec::new(); let mut rewards: Vec> = Vec::new(); - for header in &headers { + + let mut headers: Vec = Vec::new(); + + for n in start_block..end_block_plus { + let header = self.cache() + .get_fee_history(n) + .await? + .ok_or(EthApiError::InternalEthError)?; + + if n == end_block_plus { + headers.push(header.clone()); + } + base_fee_per_gas .push(U256::try_from(header.base_fee_per_gas.unwrap_or_default()).unwrap()); gas_used_ratio.push(header.gas_used as f64 / header.gas_limit as f64); // Percentiles were specified, so we need to collect reward percentile ino if let Some(percentiles) = &reward_percentiles { - rewards.push(self.calculate_reward_percentiles(percentiles, header).await?); + rewards.push(self.calculate_reward_percentiles(percentiles, &header).await?); } - } + }; + +// if headers.len() != block_count as usize { + // return Err(EthApiError::InvalidBlockRange) + // } + + // Collect base fees, gas usage ratios and (optionally) reward percentile data + // The spec states that `base_fee_per_gas` "[..] includes the next block after the newest of // the returned range, because this value can be derived from the newest block" // // The unwrap is safe since we checked earlier that we got at least 1 header. + // let last_header = headers.last().unwrap(); let chain_spec = self.provider().chain_spec(); base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( diff --git a/crates/rpc/rpc/src/eth/cache/mod.rs b/crates/rpc/rpc/src/eth/cache/mod.rs index f5c061b7143f..04a49e95db79 100644 --- a/crates/rpc/rpc/src/eth/cache/mod.rs +++ b/crates/rpc/rpc/src/eth/cache/mod.rs @@ -1,8 +1,9 @@ //! Async caching support for eth RPC +use alloy_primitives::BlockNumber; use futures::{future::Either, Stream, StreamExt}; use reth_interfaces::{provider::ProviderError, RethResult}; -use reth_primitives::{Block, Receipt, SealedBlock, TransactionSigned, B256}; +use reth_primitives::{Block, Receipt, SealedBlock, TransactionSigned, B256, Header, SealedHeader}; use reth_provider::{ BlockReader, BlockSource, CanonStateNotification, EvmEnvProvider, StateProviderFactory, }; @@ -41,6 +42,9 @@ type ReceiptsResponseSender = oneshot::Sender>>>; /// The type that can send the response to a requested env type EnvResponseSender = oneshot::Sender>; +/// The type that can send the response to a requested list of block headers +type BlockFeeResponseSender = oneshot::Sender>>; + type BlockLruCache = MultiConsumerLruCache< B256, Block, @@ -52,6 +56,8 @@ type ReceiptsLruCache = MultiConsumerLruCache, L, Receipts type EnvLruCache = MultiConsumerLruCache; +type FeeHistoryLruCache = MultiConsumerLruCache; + /// Provides async access to cached eth data /// /// This is the frontend for the async caching service which manages cached data on a different @@ -77,6 +83,7 @@ impl EthStateCache { full_block_cache: BlockLruCache::new(max_blocks, "blocks"), receipts_cache: ReceiptsLruCache::new(max_receipts, "receipts"), evm_env_cache: EnvLruCache::new(max_envs, "evm_env"), + fee_history_cache: FeeHistoryLruCache::new(max_blocks, "fee_history"), action_tx: to_service.clone(), action_rx: UnboundedReceiverStream::new(rx), action_task_spawner, @@ -199,6 +206,17 @@ impl EthStateCache { let _ = self.to_service.send(CacheAction::GetEnv { block_hash, response_tx }); rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? } + + + /// Requests available fee history based on cached block headers + pub(crate) async fn get_fee_history( + &self, + block_number: u64, + ) -> RethResult>{ + let (response_tx, rx) = oneshot::channel(); + let _ = self.to_service.send(CacheAction::GetBlockFee { block_number, response_tx}); + rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? + } } /// A task than manages caches for data required by the `eth` rpc implementation. @@ -224,7 +242,7 @@ pub(crate) struct EthStateCacheService< LimitReceipts = ByLength, LimitEnvs = ByLength, > where - LimitBlocks: Limiter, + LimitBlocks: Limiter + Limiter, LimitReceipts: Limiter>, LimitEnvs: Limiter, { @@ -236,6 +254,8 @@ pub(crate) struct EthStateCacheService< receipts_cache: ReceiptsLruCache, /// The LRU cache for revm environments evm_env_cache: EnvLruCache, + /// The LRU cache for blocks fees history + fee_history_cache: FeeHistoryLruCache, /// Sender half of the action channel. action_tx: UnboundedSender, /// Receiver half of the action channel. @@ -270,7 +290,8 @@ where // cache good block if let Ok(Some(block)) = res { - self.full_block_cache.insert(block_hash, block); + self.full_block_cache.insert(block_hash, block.clone()); + self.fee_history_cache.insert(block.number, block.header.clone().seal_slow()); } } @@ -292,6 +313,7 @@ where self.full_block_cache.update_cached_metrics(); self.receipts_cache.update_cached_metrics(); self.evm_env_cache.update_cached_metrics(); + self.fee_history_cache.update_cached_metrics(); } } @@ -409,6 +431,12 @@ where })); } } + CacheAction::GetBlockFee { block_number, response_tx } => { + if let Some(fee) = this.fee_history_cache.get(&block_number).cloned() { + let _ = response_tx.send(Ok(Some(fee))); + continue + } + } CacheAction::BlockResult { block_hash, res } => { this.on_new_block(block_hash, res); } @@ -461,6 +489,7 @@ enum CacheAction { ReceiptsResult { block_hash: B256, res: RethResult>> }, EnvResult { block_hash: B256, res: Box> }, CacheNewCanonicalChain { blocks: Vec, receipts: Vec }, + GetBlockFee{ block_number:u64, response_tx: BlockFeeResponseSender }, } struct BlockReceipts { From 558b3c4bf1796f55f157aaa72e8dda0d2e34409a Mon Sep 17 00:00:00 2001 From: allnil Date: Wed, 25 Oct 2023 17:48:37 +0100 Subject: [PATCH 03/32] add BlockFees type to collect fees data for particular block in cache, refactor caclulations --- crates/rpc/rpc/src/eth/api/fees.rs | 55 +++++++++++++---------------- crates/rpc/rpc/src/eth/cache/mod.rs | 38 ++++++++++++++++---- 2 files changed, 56 insertions(+), 37 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index e152831c05b7..d649efd0ba28 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -2,11 +2,10 @@ use crate::{ eth::error::{EthApiError, EthResult}, EthApi, }; -use alloy_primitives::FixedBytes; -use reth_interfaces::RethError; + use reth_network_api::NetworkInfo; use reth_primitives::{ - basefee::calculate_next_block_base_fee, BlockNumberOrTag, SealedHeader, U256, B256 + basefee::calculate_next_block_base_fee, BlockNumberOrTag, U256, B256 }; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::{FeeHistory, TxGasAndReward}; @@ -15,6 +14,7 @@ use tracing::debug; use derive_more::{Deref, DerefMut}; use schnellru::{ByLength, LruMap}; use std::fmt::{self, Debug, Formatter}; +use crate::eth::cache::BlockFees; impl EthApi where @@ -66,7 +66,6 @@ where ); block_count = max_fee_history } - let Some(end_block) = self.provider().block_number_for_id(newest_block.into())? else { return Err(EthApiError::UnknownBlockNumber) @@ -97,54 +96,50 @@ where // SAFETY: We ensured that block count is capped let start_block = end_block_plus - block_count; - // TO-DO: first we support only in-cache queries - // let headers = self.provider().sealed_headers_range(start_block..=end_block)?; - // begin let mut base_fee_per_gas: Vec = Vec::new(); let mut gas_used_ratio: Vec = Vec::new(); let mut rewards: Vec> = Vec::new(); - let mut headers: Vec = Vec::new(); + struct LastBlock { + gas_used: u64, + gas_limit: u64, + base_fee_per_gas: u64, + } + let mut last_block = LastBlock{gas_used: 0, gas_limit: 0, base_fee_per_gas: 0}; for n in start_block..end_block_plus { - let header = self.cache() + let block_fees = self.cache() .get_fee_history(n) .await? .ok_or(EthApiError::InternalEthError)?; if n == end_block_plus { - headers.push(header.clone()); - } - - base_fee_per_gas - .push(U256::try_from(header.base_fee_per_gas.unwrap_or_default()).unwrap()); - gas_used_ratio.push(header.gas_used as f64 / header.gas_limit as f64); + last_block.gas_used = block_fees.gas_used; + last_block.gas_limit = block_fees.gas_limit; + last_block.base_fee_per_gas = block_fees.base_fee_per_gas; + } + + base_fee_per_gas.push(U256::try_from(block_fees.base_fee_per_gas).unwrap()); + gas_used_ratio.push(block_fees.gas_used_ratio); // Percentiles were specified, so we need to collect reward percentile ino if let Some(percentiles) = &reward_percentiles { - rewards.push(self.calculate_reward_percentiles(percentiles, &header).await?); + rewards.push(self.calculate_reward_percentiles(percentiles, &block_fees).await?); } }; -// if headers.len() != block_count as usize { - // return Err(EthApiError::InvalidBlockRange) - // } - // Collect base fees, gas usage ratios and (optionally) reward percentile data - // The spec states that `base_fee_per_gas` "[..] includes the next block after the newest of // the returned range, because this value can be derived from the newest block" // // The unwrap is safe since we checked earlier that we got at least 1 header. - // - let last_header = headers.last().unwrap(); let chain_spec = self.provider().chain_spec(); base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( - last_header.gas_used, - last_header.gas_limit, - last_header.base_fee_per_gas.unwrap_or_default(), + last_block.gas_used, + last_block.gas_limit, + last_block.base_fee_per_gas, chain_spec.base_fee_params, ))); @@ -164,11 +159,11 @@ where async fn calculate_reward_percentiles( &self, percentiles: &[f64], - header: &SealedHeader, + block_fees: &BlockFees, ) -> Result, EthApiError> { let (transactions, receipts) = self .cache() - .get_transactions_and_receipts(header.hash) + .get_transactions_and_receipts(block_fees.block_hash) .await? .ok_or(EthApiError::InvalidBlockRange)?; @@ -187,7 +182,7 @@ where Some(TxGasAndReward { gas_used, - reward: tx.effective_gas_tip(header.base_fee_per_gas).unwrap_or_default(), + reward: tx.effective_gas_tip(Some(block_fees.base_fee_per_gas)).unwrap_or_default(), }) }) .collect::>(); @@ -210,7 +205,7 @@ where continue } - let threshold = (header.gas_used as f64 * percentile / 100.) as u64; + let threshold = (block_fees.gas_used as f64 * percentile / 100.) as u64; while cumulative_gas_used < threshold && tx_index < transactions.len() - 1 { tx_index += 1; cumulative_gas_used += transactions[tx_index].gas_used; diff --git a/crates/rpc/rpc/src/eth/cache/mod.rs b/crates/rpc/rpc/src/eth/cache/mod.rs index 04a49e95db79..5b6b53a7818f 100644 --- a/crates/rpc/rpc/src/eth/cache/mod.rs +++ b/crates/rpc/rpc/src/eth/cache/mod.rs @@ -1,9 +1,9 @@ //! Async caching support for eth RPC -use alloy_primitives::BlockNumber; +use alloy_primitives::{BlockNumber}; use futures::{future::Either, Stream, StreamExt}; use reth_interfaces::{provider::ProviderError, RethResult}; -use reth_primitives::{Block, Receipt, SealedBlock, TransactionSigned, B256, Header, SealedHeader}; +use reth_primitives::{Block, Receipt, SealedBlock, TransactionSigned, B256}; use reth_provider::{ BlockReader, BlockSource, CanonStateNotification, EvmEnvProvider, StateProviderFactory, }; @@ -43,7 +43,23 @@ type ReceiptsResponseSender = oneshot::Sender>>>; type EnvResponseSender = oneshot::Sender>; /// The type that can send the response to a requested list of block headers -type BlockFeeResponseSender = oneshot::Sender>>; +type BlockFeeResponseSender = oneshot::Sender>>; + +/// The type that contains fees data for associated block in cache +#[derive(Clone, Debug)] +pub struct BlockFees { + /// Hash of the block + pub block_hash: B256, + /// Block data on bas_fee_per_gas + pub base_fee_per_gas: u64, + /// Precalculated ratio + pub gas_used_ratio: f64, + /// Block data on gas_used + pub gas_used: u64, + /// Block data on gas_limit + pub gas_limit: u64, +} + type BlockLruCache = MultiConsumerLruCache< B256, @@ -56,7 +72,7 @@ type ReceiptsLruCache = MultiConsumerLruCache, L, Receipts type EnvLruCache = MultiConsumerLruCache; -type FeeHistoryLruCache = MultiConsumerLruCache; +type FeeHistoryLruCache = MultiConsumerLruCache; /// Provides async access to cached eth data /// @@ -212,7 +228,7 @@ impl EthStateCache { pub(crate) async fn get_fee_history( &self, block_number: u64, - ) -> RethResult>{ + ) -> RethResult>{ let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetBlockFee { block_number, response_tx}); rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? @@ -242,7 +258,7 @@ pub(crate) struct EthStateCacheService< LimitReceipts = ByLength, LimitEnvs = ByLength, > where - LimitBlocks: Limiter + Limiter, + LimitBlocks: Limiter + Limiter, LimitReceipts: Limiter>, LimitEnvs: Limiter, { @@ -291,7 +307,15 @@ where // cache good block if let Ok(Some(block)) = res { self.full_block_cache.insert(block_hash, block.clone()); - self.fee_history_cache.insert(block.number, block.header.clone().seal_slow()); + self.fee_history_cache.insert( + block.number, + BlockFees { + base_fee_per_gas: block.header.base_fee_per_gas.unwrap_or_default(), + gas_used_ratio: block.header.gas_used as f64 / block.header.gas_limit as f64, + gas_used: block.header.gas_used, + gas_limit: block.header.gas_limit, + block_hash: block_hash, + }); } } From 54204b4100e16e482ec827f0c4ce180d76484a92 Mon Sep 17 00:00:00 2001 From: allnil Date: Wed, 25 Oct 2023 18:01:30 +0100 Subject: [PATCH 04/32] format and clean --- crates/rpc/rpc/src/eth/api/fees.rs | 35 +++++++++++++---------------- crates/rpc/rpc/src/eth/cache/mod.rs | 33 +++++++++++++-------------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index d649efd0ba28..b751ba440919 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -3,18 +3,16 @@ use crate::{ EthApi, }; +use crate::eth::cache::BlockFees; +use derive_more::{Deref, DerefMut}; use reth_network_api::NetworkInfo; -use reth_primitives::{ - basefee::calculate_next_block_base_fee, BlockNumberOrTag, U256, B256 -}; +use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, B256, U256}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::{FeeHistory, TxGasAndReward}; use reth_transaction_pool::TransactionPool; -use tracing::debug; -use derive_more::{Deref, DerefMut}; use schnellru::{ByLength, LruMap}; use std::fmt::{self, Debug, Formatter}; -use crate::eth::cache::BlockFees; +use tracing::debug; impl EthApi where @@ -102,17 +100,15 @@ where let mut rewards: Vec> = Vec::new(); struct LastBlock { - gas_used: u64, - gas_limit: u64, - base_fee_per_gas: u64, + gas_used: u64, + gas_limit: u64, + base_fee_per_gas: u64, } - let mut last_block = LastBlock{gas_used: 0, gas_limit: 0, base_fee_per_gas: 0}; + let mut last_block = LastBlock { gas_used: 0, gas_limit: 0, base_fee_per_gas: 0 }; for n in start_block..end_block_plus { - let block_fees = self.cache() - .get_fee_history(n) - .await? - .ok_or(EthApiError::InternalEthError)?; + let block_fees = + self.cache().get_fee_history(n).await?.ok_or(EthApiError::InternalEthError)?; if n == end_block_plus { last_block.gas_used = block_fees.gas_used; @@ -127,10 +123,10 @@ where if let Some(percentiles) = &reward_percentiles { rewards.push(self.calculate_reward_percentiles(percentiles, &block_fees).await?); } - }; + } // Collect base fees, gas usage ratios and (optionally) reward percentile data - + // The spec states that `base_fee_per_gas` "[..] includes the next block after the newest of // the returned range, because this value can be derived from the newest block" // @@ -182,7 +178,9 @@ where Some(TxGasAndReward { gas_used, - reward: tx.effective_gas_tip(Some(block_fees.base_fee_per_gas)).unwrap_or_default(), + reward: tx + .effective_gas_tip(Some(block_fees.base_fee_per_gas)) + .unwrap_or_default(), }) }) .collect::>(); @@ -217,8 +215,6 @@ where } } - - /// Wrapper struct for LruMap #[derive(Deref, DerefMut)] struct BaseFeePerGasLruCache(LruMap), ByLength>); @@ -232,7 +228,6 @@ impl Debug for BaseFeePerGasLruCache { } } - /// Wrapper struct for LruMap #[derive(Deref, DerefMut)] struct GasUsedRatioLruCache(LruMap), ByLength>); diff --git a/crates/rpc/rpc/src/eth/cache/mod.rs b/crates/rpc/rpc/src/eth/cache/mod.rs index 5b6b53a7818f..bdc553757d5b 100644 --- a/crates/rpc/rpc/src/eth/cache/mod.rs +++ b/crates/rpc/rpc/src/eth/cache/mod.rs @@ -1,6 +1,6 @@ //! Async caching support for eth RPC -use alloy_primitives::{BlockNumber}; +use alloy_primitives::BlockNumber; use futures::{future::Either, Stream, StreamExt}; use reth_interfaces::{provider::ProviderError, RethResult}; use reth_primitives::{Block, Receipt, SealedBlock, TransactionSigned, B256}; @@ -42,7 +42,7 @@ type ReceiptsResponseSender = oneshot::Sender>>>; /// The type that can send the response to a requested env type EnvResponseSender = oneshot::Sender>; -/// The type that can send the response to a requested list of block headers +/// The type that can send the response to a requested list of block headers type BlockFeeResponseSender = oneshot::Sender>>; /// The type that contains fees data for associated block in cache @@ -50,17 +50,16 @@ type BlockFeeResponseSender = oneshot::Sender>>; pub struct BlockFees { /// Hash of the block pub block_hash: B256, - /// Block data on bas_fee_per_gas + /// Block data on bas_fee_per_gas pub base_fee_per_gas: u64, /// Precalculated ratio - pub gas_used_ratio: f64, + pub gas_used_ratio: f64, /// Block data on gas_used pub gas_used: u64, /// Block data on gas_limit pub gas_limit: u64, } - type BlockLruCache = MultiConsumerLruCache< B256, Block, @@ -72,7 +71,8 @@ type ReceiptsLruCache = MultiConsumerLruCache, L, Receipts type EnvLruCache = MultiConsumerLruCache; -type FeeHistoryLruCache = MultiConsumerLruCache; +type FeeHistoryLruCache = + MultiConsumerLruCache; /// Provides async access to cached eth data /// @@ -223,14 +223,10 @@ impl EthStateCache { rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? } - /// Requests available fee history based on cached block headers - pub(crate) async fn get_fee_history( - &self, - block_number: u64, - ) -> RethResult>{ + pub(crate) async fn get_fee_history(&self, block_number: u64) -> RethResult> { let (response_tx, rx) = oneshot::channel(); - let _ = self.to_service.send(CacheAction::GetBlockFee { block_number, response_tx}); + let _ = self.to_service.send(CacheAction::GetBlockFee { block_number, response_tx }); rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? } } @@ -308,14 +304,15 @@ where if let Ok(Some(block)) = res { self.full_block_cache.insert(block_hash, block.clone()); self.fee_history_cache.insert( - block.number, - BlockFees { + block.number, + BlockFees { base_fee_per_gas: block.header.base_fee_per_gas.unwrap_or_default(), gas_used_ratio: block.header.gas_used as f64 / block.header.gas_limit as f64, gas_used: block.header.gas_used, gas_limit: block.header.gas_limit, - block_hash: block_hash, - }); + block_hash, + }, + ); } } @@ -456,7 +453,7 @@ where } } CacheAction::GetBlockFee { block_number, response_tx } => { - if let Some(fee) = this.fee_history_cache.get(&block_number).cloned() { + if let Some(fee) = this.fee_history_cache.get(&block_number).cloned() { let _ = response_tx.send(Ok(Some(fee))); continue } @@ -513,7 +510,7 @@ enum CacheAction { ReceiptsResult { block_hash: B256, res: RethResult>> }, EnvResult { block_hash: B256, res: Box> }, CacheNewCanonicalChain { blocks: Vec, receipts: Vec }, - GetBlockFee{ block_number:u64, response_tx: BlockFeeResponseSender }, + GetBlockFee { block_number: u64, response_tx: BlockFeeResponseSender }, } struct BlockReceipts { From 2f4082085ad6e423754325e05b78d19ae6cea60c Mon Sep 17 00:00:00 2001 From: allnil Date: Wed, 25 Oct 2023 18:22:17 +0100 Subject: [PATCH 05/32] add comment --- crates/rpc/rpc/src/eth/api/fees.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index b751ba440919..791c2e6cb9e6 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -1,3 +1,5 @@ +//! Contains RPC handler implementations for fee history. + use crate::{ eth::error::{EthApiError, EthResult}, EthApi, From acbbd76fd827fa2b93229f7be045da355a519c61 Mon Sep 17 00:00:00 2001 From: allnil Date: Wed, 25 Oct 2023 18:24:20 +0100 Subject: [PATCH 06/32] leave comment regarding collected values --- crates/rpc/rpc/src/eth/api/fees.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 791c2e6cb9e6..b78035062590 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -97,6 +97,7 @@ where let start_block = end_block_plus - block_count; + // Collect base fees, gas usage ratios and (optionally) reward percentile data let mut base_fee_per_gas: Vec = Vec::new(); let mut gas_used_ratio: Vec = Vec::new(); let mut rewards: Vec> = Vec::new(); From 43a4d4acdde0820d232f05d3bdcde458a1fa621a Mon Sep 17 00:00:00 2001 From: allnil Date: Wed, 25 Oct 2023 18:25:57 +0100 Subject: [PATCH 07/32] delete unused structs --- crates/rpc/rpc/src/eth/api/fees.rs | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index b78035062590..b0a88748f0e7 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -217,29 +217,3 @@ where Ok(rewards_in_block) } } - -/// Wrapper struct for LruMap -#[derive(Deref, DerefMut)] -struct BaseFeePerGasLruCache(LruMap), ByLength>); - -impl Debug for BaseFeePerGasLruCache { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("BaseFeePerGasLruCache") - .field("cache_length", &self.len()) - .field("cache_memory_usage", &self.memory_usage()) - .finish() - } -} - -/// Wrapper struct for LruMap -#[derive(Deref, DerefMut)] -struct GasUsedRatioLruCache(LruMap), ByLength>); - -impl Debug for GasUsedRatioLruCache { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("GasUsedRatioLruCache") - .field("cache_length", &self.len()) - .field("cache_memory_usage", &self.memory_usage()) - .finish() - } -} From 70dfb484307286e466e6969b505b5068ee1fda89 Mon Sep 17 00:00:00 2001 From: allnil Date: Wed, 25 Oct 2023 18:30:36 +0100 Subject: [PATCH 08/32] remove unused imports --- crates/rpc/rpc/src/eth/api/fees.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index b0a88748f0e7..b1f8ebfc3269 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -6,14 +6,11 @@ use crate::{ }; use crate::eth::cache::BlockFees; -use derive_more::{Deref, DerefMut}; use reth_network_api::NetworkInfo; -use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, B256, U256}; +use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, U256}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::{FeeHistory, TxGasAndReward}; use reth_transaction_pool::TransactionPool; -use schnellru::{ByLength, LruMap}; -use std::fmt::{self, Debug, Formatter}; use tracing::debug; impl EthApi From e690f426b7978c68abb037d36b9711c0f7b9c113 Mon Sep 17 00:00:00 2001 From: allnil Date: Thu, 26 Oct 2023 22:41:11 +0100 Subject: [PATCH 09/32] remove fee history cache from eth_state, prepare separate type and cache to store fees for blocks, refactor fee_history function --- crates/rpc/rpc/src/eth/api/fees.rs | 48 ++++++++++++++++++------- crates/rpc/rpc/src/eth/api/mod.rs | 56 ++++++++++++++++++++++++++++- crates/rpc/rpc/src/eth/cache/mod.rs | 50 ++------------------------ 3 files changed, 93 insertions(+), 61 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index b1f8ebfc3269..e66e069f2fff 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -5,9 +5,9 @@ use crate::{ EthApi, }; -use crate::eth::cache::BlockFees; +use crate::eth::api::{BlockFees}; use reth_network_api::NetworkInfo; -use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, U256}; +use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, U256, B256}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::{FeeHistory, TxGasAndReward}; use reth_transaction_pool::TransactionPool; @@ -48,6 +48,8 @@ where return Ok(FeeHistory::default()) } + let mut fee_history_cache = self.inner.fee_history_cache.lock().await; + // See https://github.com/ethereum/go-ethereum/blob/2754b197c935ee63101cbbca2752338246384fec/eth/gasprice/feehistory.go#L218C8-L225 let max_fee_history = if reward_percentiles.is_none() { self.gas_oracle().config().max_header_history @@ -67,7 +69,7 @@ where let Some(end_block) = self.provider().block_number_for_id(newest_block.into())? else { return Err(EthApiError::UnknownBlockNumber) }; - + // need to add 1 to the end block to get the correct (inclusive) range let end_block_plus = end_block + 1; // Ensure that we would not be querying outside of genesis @@ -75,6 +77,7 @@ where block_count = end_block_plus; } + // If reward percentiles were specified, we + schnellru::Limiterneed to validate that they are monotonically // increasing and 0 <= p <= 100 @@ -106,23 +109,40 @@ where } let mut last_block = LastBlock { gas_used: 0, gas_limit: 0, base_fee_per_gas: 0 }; - for n in start_block..end_block_plus { - let block_fees = - self.cache().get_fee_history(n).await?.ok_or(EthApiError::InternalEthError)?; + let end_block_header = self.provider().sealed_header(end_block)?.unwrap(); + let mut current_hash = end_block_header.hash; + let mut parrent_hash = end_block_header.parent_hash; + for n in (start_block..end_block).rev() { + let block_fees: BlockFees; + + if let Some(cached_block_fees) = fee_history_cache.get(¤t_hash) { + block_fees = cached_block_fees.to_owned(); + } else { + let current_header = self + .provider().sealed_header(n)?.unwrap(); - if n == end_block_plus { + parrent_hash = current_header.parent_hash; + block_fees = BlockFees::from_header(current_header.clone()); + fee_history_cache + .insert(current_hash, block_fees.clone()); + }; + + + if n == end_block { last_block.gas_used = block_fees.gas_used; last_block.gas_limit = block_fees.gas_limit; last_block.base_fee_per_gas = block_fees.base_fee_per_gas; } - base_fee_per_gas.push(U256::try_from(block_fees.base_fee_per_gas).unwrap()); - gas_used_ratio.push(block_fees.gas_used_ratio); + base_fee_per_gas.push(U256::try_from(block_fees.base_fee_per_gas.clone()).unwrap()); + gas_used_ratio.push(block_fees.gas_used_ratio.clone()); // Percentiles were specified, so we need to collect reward percentile ino if let Some(percentiles) = &reward_percentiles { - rewards.push(self.calculate_reward_percentiles(percentiles, &block_fees).await?); + rewards.push(self.calculate_reward_percentiles(percentiles, current_hash.clone(), &block_fees).await?); } + + current_hash = parrent_hash; } // Collect base fees, gas usage ratios and (optionally) reward percentile data @@ -138,7 +158,10 @@ where last_block.base_fee_per_gas, chain_spec.base_fee_params, ))); - + + base_fee_per_gas.reverse(); + gas_used_ratio.reverse(); + Ok(FeeHistory { base_fee_per_gas, gas_used_ratio, @@ -155,11 +178,12 @@ where async fn calculate_reward_percentiles( &self, percentiles: &[f64], + header_hash: B256, block_fees: &BlockFees, ) -> Result, EthApiError> { let (transactions, receipts) = self .cache() - .get_transactions_and_receipts(block_fees.block_hash) + .get_transactions_and_receipts(header_hash) .await? .ok_or(EthApiError::InvalidBlockRange)?; diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 9b6d000009b0..5e1d6b7ce49f 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -10,11 +10,12 @@ use crate::eth::{ gas_oracle::GasPriceOracle, signer::EthSigner, }; +use std::fmt::{self, Debug, Formatter}; use async_trait::async_trait; use reth_interfaces::RethResult; use reth_network_api::NetworkInfo; use reth_primitives::{ - Address, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, B256, U256, U64, + Address, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, B256, U256, U64, SealedHeader, }; use reth_provider::{ BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderBox, StateProviderFactory, @@ -29,6 +30,8 @@ use std::{ time::{Duration, Instant}, }; use tokio::sync::{oneshot, Mutex}; +use derive_more::{Deref, DerefMut}; +use schnellru::{ByLength, LruMap}; mod block; mod call; @@ -42,6 +45,8 @@ mod transactions; use crate::BlockingTaskPool; pub use transactions::{EthTransactions, TransactionSource}; +use super::revm_utils::FillableTransaction; + /// `Eth` API trait. /// /// Defines core functionality of the `eth` API implementation. @@ -137,6 +142,9 @@ where task_spawner, pending_block: Default::default(), blocking_task_pool, + fee_history_cache: Mutex::new(FeeHistoryLruCache(LruMap::new(ByLength::new( + 1024, + )))) }; Self { inner: Arc::new(inner) } } @@ -190,6 +198,11 @@ where pub fn pool(&self) -> &Pool { &self.inner.pool } + + /// Returns fee history cache + pub fn fee_history_cache(&self) -> &Mutex { + &self.inner.fee_history_cache + } } // === State access helpers === @@ -438,4 +451,45 @@ struct EthApiInner { pending_block: Mutex>, /// A pool dedicated to blocking tasks. blocking_task_pool: BlockingTaskPool, + /// Cache for block fees history + fee_history_cache: Mutex, +} + + +/// The type that contains fees data for associated block in cache +#[derive(Clone, Debug)] +pub struct BlockFees { + /// Block data on bas_fee_per_gas + pub base_fee_per_gas: u64, + /// Precalculated ratio + pub gas_used_ratio: f64, + /// Block data on gas_used + pub gas_used: u64, + /// Block data on gas_limit + pub gas_limit: u64, } + +impl BlockFees { + fn from_header(header: SealedHeader) -> BlockFees { + BlockFees { + base_fee_per_gas: header.base_fee_per_gas.unwrap_or_default(), + gas_used_ratio: header.gas_used as f64 / header.gas_limit as f64, + gas_used: header.gas_used, + gas_limit: header.gas_limit, + } + } +} + +/// Wrapper struct for LruMap +#[derive(Deref, DerefMut)] +pub struct FeeHistoryLruCache(LruMap); + +impl Debug for FeeHistoryLruCache { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("FeeHistoryLruCache") + .field("cache_length", &self.len()) + .field("cache_memory_usage", &self.memory_usage()) + .finish() + } +} + diff --git a/crates/rpc/rpc/src/eth/cache/mod.rs b/crates/rpc/rpc/src/eth/cache/mod.rs index bdc553757d5b..19127e25ba0b 100644 --- a/crates/rpc/rpc/src/eth/cache/mod.rs +++ b/crates/rpc/rpc/src/eth/cache/mod.rs @@ -42,23 +42,6 @@ type ReceiptsResponseSender = oneshot::Sender>>>; /// The type that can send the response to a requested env type EnvResponseSender = oneshot::Sender>; -/// The type that can send the response to a requested list of block headers -type BlockFeeResponseSender = oneshot::Sender>>; - -/// The type that contains fees data for associated block in cache -#[derive(Clone, Debug)] -pub struct BlockFees { - /// Hash of the block - pub block_hash: B256, - /// Block data on bas_fee_per_gas - pub base_fee_per_gas: u64, - /// Precalculated ratio - pub gas_used_ratio: f64, - /// Block data on gas_used - pub gas_used: u64, - /// Block data on gas_limit - pub gas_limit: u64, -} type BlockLruCache = MultiConsumerLruCache< B256, @@ -71,8 +54,6 @@ type ReceiptsLruCache = MultiConsumerLruCache, L, Receipts type EnvLruCache = MultiConsumerLruCache; -type FeeHistoryLruCache = - MultiConsumerLruCache; /// Provides async access to cached eth data /// @@ -99,7 +80,6 @@ impl EthStateCache { full_block_cache: BlockLruCache::new(max_blocks, "blocks"), receipts_cache: ReceiptsLruCache::new(max_receipts, "receipts"), evm_env_cache: EnvLruCache::new(max_envs, "evm_env"), - fee_history_cache: FeeHistoryLruCache::new(max_blocks, "fee_history"), action_tx: to_service.clone(), action_rx: UnboundedReceiverStream::new(rx), action_task_spawner, @@ -222,13 +202,7 @@ impl EthStateCache { let _ = self.to_service.send(CacheAction::GetEnv { block_hash, response_tx }); rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? } - - /// Requests available fee history based on cached block headers - pub(crate) async fn get_fee_history(&self, block_number: u64) -> RethResult> { - let (response_tx, rx) = oneshot::channel(); - let _ = self.to_service.send(CacheAction::GetBlockFee { block_number, response_tx }); - rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? - } + } /// A task than manages caches for data required by the `eth` rpc implementation. @@ -254,7 +228,7 @@ pub(crate) struct EthStateCacheService< LimitReceipts = ByLength, LimitEnvs = ByLength, > where - LimitBlocks: Limiter + Limiter, + LimitBlocks: Limiter, LimitReceipts: Limiter>, LimitEnvs: Limiter, { @@ -266,8 +240,6 @@ pub(crate) struct EthStateCacheService< receipts_cache: ReceiptsLruCache, /// The LRU cache for revm environments evm_env_cache: EnvLruCache, - /// The LRU cache for blocks fees history - fee_history_cache: FeeHistoryLruCache, /// Sender half of the action channel. action_tx: UnboundedSender, /// Receiver half of the action channel. @@ -303,16 +275,6 @@ where // cache good block if let Ok(Some(block)) = res { self.full_block_cache.insert(block_hash, block.clone()); - self.fee_history_cache.insert( - block.number, - BlockFees { - base_fee_per_gas: block.header.base_fee_per_gas.unwrap_or_default(), - gas_used_ratio: block.header.gas_used as f64 / block.header.gas_limit as f64, - gas_used: block.header.gas_used, - gas_limit: block.header.gas_limit, - block_hash, - }, - ); } } @@ -334,7 +296,6 @@ where self.full_block_cache.update_cached_metrics(); self.receipts_cache.update_cached_metrics(); self.evm_env_cache.update_cached_metrics(); - self.fee_history_cache.update_cached_metrics(); } } @@ -452,12 +413,6 @@ where })); } } - CacheAction::GetBlockFee { block_number, response_tx } => { - if let Some(fee) = this.fee_history_cache.get(&block_number).cloned() { - let _ = response_tx.send(Ok(Some(fee))); - continue - } - } CacheAction::BlockResult { block_hash, res } => { this.on_new_block(block_hash, res); } @@ -510,7 +465,6 @@ enum CacheAction { ReceiptsResult { block_hash: B256, res: RethResult>> }, EnvResult { block_hash: B256, res: Box> }, CacheNewCanonicalChain { blocks: Vec, receipts: Vec }, - GetBlockFee { block_number: u64, response_tx: BlockFeeResponseSender }, } struct BlockReceipts { From a9b0740855d565d95cc51b9ea03390bd74bc340c Mon Sep 17 00:00:00 2001 From: allnil Date: Fri, 27 Oct 2023 17:25:48 +0100 Subject: [PATCH 10/32] fix tests for single block --- crates/rpc/rpc/src/eth/api/fees.rs | 73 +++++++++++++++++------------ crates/rpc/rpc/src/eth/api/mod.rs | 19 ++++---- crates/rpc/rpc/src/eth/cache/mod.rs | 3 -- 3 files changed, 51 insertions(+), 44 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index e66e069f2fff..18799cd30810 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -5,9 +5,9 @@ use crate::{ EthApi, }; -use crate::eth::api::{BlockFees}; +use crate::eth::api::BlockFees; use reth_network_api::NetworkInfo; -use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, U256, B256}; +use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, B256, U256}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::{FeeHistory, TxGasAndReward}; use reth_transaction_pool::TransactionPool; @@ -69,7 +69,7 @@ where let Some(end_block) = self.provider().block_number_for_id(newest_block.into())? else { return Err(EthApiError::UnknownBlockNumber) }; - + // need to add 1 to the end block to get the correct (inclusive) range let end_block_plus = end_block + 1; // Ensure that we would not be querying outside of genesis @@ -77,7 +77,6 @@ where block_count = end_block_plus; } - // If reward percentiles were specified, we + schnellru::Limiterneed to validate that they are monotonically // increasing and 0 <= p <= 100 @@ -107,42 +106,54 @@ where gas_limit: u64, base_fee_per_gas: u64, } - let mut last_block = LastBlock { gas_used: 0, gas_limit: 0, base_fee_per_gas: 0 }; - let end_block_header = self.provider().sealed_header(end_block)?.unwrap(); - let mut current_hash = end_block_header.hash; - let mut parrent_hash = end_block_header.parent_hash; - for n in (start_block..end_block).rev() { + let Some(end_block_header) = self.provider().sealed_header(end_block)? else { + return Err(EthApiError::InvalidBlockRange) + }; + + let mut last_block = LastBlock{gas_used: 0, gas_limit: 0, base_fee_per_gas: 0}; + + let mut current_hash = end_block_header.hash; + let mut parent_hash: B256; + for n in (start_block..end_block_plus).rev() { let block_fees: BlockFees; if let Some(cached_block_fees) = fee_history_cache.get(¤t_hash) { - block_fees = cached_block_fees.to_owned(); - } else { - let current_header = self - .provider().sealed_header(n)?.unwrap(); - - parrent_hash = current_header.parent_hash; - block_fees = BlockFees::from_header(current_header.clone()); - fee_history_cache - .insert(current_hash, block_fees.clone()); + block_fees = cached_block_fees.to_owned(); + parent_hash = block_fees.parent_hash; + } else { + let Some(current_header) = self.provider().sealed_header(n)? else { + return Err(EthApiError::InvalidBlockRange); }; - - if n == end_block { - last_block.gas_used = block_fees.gas_used; - last_block.gas_limit = block_fees.gas_limit; - last_block.base_fee_per_gas = block_fees.base_fee_per_gas; - } + parent_hash = current_header.parent_hash; + block_fees = BlockFees::from_header(current_header.header.clone()); + fee_history_cache.insert(current_hash, block_fees.clone()); + }; base_fee_per_gas.push(U256::try_from(block_fees.base_fee_per_gas.clone()).unwrap()); gas_used_ratio.push(block_fees.gas_used_ratio.clone()); - + + if n == end_block { + last_block = LastBlock { + gas_used: block_fees.gas_used, + gas_limit: block_fees.gas_limit, + base_fee_per_gas: block_fees.base_fee_per_gas, + }; + } // Percentiles were specified, so we need to collect reward percentile ino if let Some(percentiles) = &reward_percentiles { - rewards.push(self.calculate_reward_percentiles(percentiles, current_hash.clone(), &block_fees).await?); + rewards.push( + self.calculate_reward_percentiles( + percentiles, + current_hash.clone(), + &block_fees, + ) + .await?, + ); } - current_hash = parrent_hash; + current_hash = parent_hash; } // Collect base fees, gas usage ratios and (optionally) reward percentile data @@ -152,16 +163,16 @@ where // // The unwrap is safe since we checked earlier that we got at least 1 header. let chain_spec = self.provider().chain_spec(); + + base_fee_per_gas.reverse(); + gas_used_ratio.reverse(); base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( last_block.gas_used, last_block.gas_limit, last_block.base_fee_per_gas, chain_spec.base_fee_params, ))); - - base_fee_per_gas.reverse(); - gas_used_ratio.reverse(); - + Ok(FeeHistory { base_fee_per_gas, gas_used_ratio, diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 5e1d6b7ce49f..d39a9c43044d 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -10,12 +10,12 @@ use crate::eth::{ gas_oracle::GasPriceOracle, signer::EthSigner, }; -use std::fmt::{self, Debug, Formatter}; use async_trait::async_trait; +use derive_more::{Deref, DerefMut}; use reth_interfaces::RethResult; use reth_network_api::NetworkInfo; use reth_primitives::{ - Address, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, B256, U256, U64, SealedHeader, + Address, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, Header, SealedHeader, B256, U256, U64, }; use reth_provider::{ BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderBox, StateProviderFactory, @@ -24,14 +24,14 @@ use reth_rpc_types::{SyncInfo, SyncStatus}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use reth_transaction_pool::TransactionPool; use revm_primitives::{BlockEnv, CfgEnv}; +use schnellru::{ByLength, LruMap}; use std::{ + fmt::{self, Debug, Formatter}, future::Future, sync::Arc, time::{Duration, Instant}, }; use tokio::sync::{oneshot, Mutex}; -use derive_more::{Deref, DerefMut}; -use schnellru::{ByLength, LruMap}; mod block; mod call; @@ -142,9 +142,7 @@ where task_spawner, pending_block: Default::default(), blocking_task_pool, - fee_history_cache: Mutex::new(FeeHistoryLruCache(LruMap::new(ByLength::new( - 1024, - )))) + fee_history_cache: Mutex::new(FeeHistoryLruCache(LruMap::new(ByLength::new(1024)))), }; Self { inner: Arc::new(inner) } } @@ -455,7 +453,6 @@ struct EthApiInner { fee_history_cache: Mutex, } - /// The type that contains fees data for associated block in cache #[derive(Clone, Debug)] pub struct BlockFees { @@ -467,15 +464,18 @@ pub struct BlockFees { pub gas_used: u64, /// Block data on gas_limit pub gas_limit: u64, + /// parent block hash + pub parent_hash: B256, } impl BlockFees { - fn from_header(header: SealedHeader) -> BlockFees { + fn from_header(header: Header) -> BlockFees { BlockFees { base_fee_per_gas: header.base_fee_per_gas.unwrap_or_default(), gas_used_ratio: header.gas_used as f64 / header.gas_limit as f64, gas_used: header.gas_used, gas_limit: header.gas_limit, + parent_hash: header.parent_hash, } } } @@ -492,4 +492,3 @@ impl Debug for FeeHistoryLruCache { .finish() } } - diff --git a/crates/rpc/rpc/src/eth/cache/mod.rs b/crates/rpc/rpc/src/eth/cache/mod.rs index 19127e25ba0b..68851fbe131f 100644 --- a/crates/rpc/rpc/src/eth/cache/mod.rs +++ b/crates/rpc/rpc/src/eth/cache/mod.rs @@ -42,7 +42,6 @@ type ReceiptsResponseSender = oneshot::Sender>>>; /// The type that can send the response to a requested env type EnvResponseSender = oneshot::Sender>; - type BlockLruCache = MultiConsumerLruCache< B256, Block, @@ -54,7 +53,6 @@ type ReceiptsLruCache = MultiConsumerLruCache, L, Receipts type EnvLruCache = MultiConsumerLruCache; - /// Provides async access to cached eth data /// /// This is the frontend for the async caching service which manages cached data on a different @@ -202,7 +200,6 @@ impl EthStateCache { let _ = self.to_service.send(CacheAction::GetEnv { block_hash, response_tx }); rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? } - } /// A task than manages caches for data required by the `eth` rpc implementation. From dae29647a3602d5c3f78d393198046d82a705334 Mon Sep 17 00:00:00 2001 From: allnil Date: Fri, 27 Oct 2023 17:40:51 +0100 Subject: [PATCH 11/32] add parent_hash in mock data for tests --- crates/rpc/rpc/src/eth/api/server.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index 9801878969d1..e1896c97ae87 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -441,6 +441,7 @@ mod tests { let mut gas_used_ratios = Vec::new(); let mut base_fees_per_gas = Vec::new(); let mut last_header = None; + let mut parent_hash = B256::default(); for i in (0..block_count).rev() { let hash = rng.gen(); @@ -454,9 +455,11 @@ mod tests { gas_limit, gas_used, base_fee_per_gas, + parent_hash, ..Default::default() }; last_header = Some(header.clone()); + parent_hash = hash; let mut transactions = vec![]; for _ in 0..100 { From 52d41568ef7b582c2d34fe6e09a79d2702aa4b02 Mon Sep 17 00:00:00 2001 From: allnil Date: Mon, 30 Oct 2023 22:19:25 +0000 Subject: [PATCH 12/32] BTreeMap wrapped with RwLock, prepare task which listens to new canonical blocks channel --- crates/rpc/rpc-builder/src/eth.rs | 5 +- crates/rpc/rpc-builder/src/lib.rs | 13 +++- crates/rpc/rpc/src/eth/api/fees.rs | 100 ++++++++------------------ crates/rpc/rpc/src/eth/api/mod.rs | 105 +++++++++++++++++++++------- crates/rpc/rpc/src/eth/cache/mod.rs | 2 +- 5 files changed, 129 insertions(+), 96 deletions(-) diff --git a/crates/rpc/rpc-builder/src/eth.rs b/crates/rpc/rpc-builder/src/eth.rs index e3b9d4dcc735..dd146e1eda81 100644 --- a/crates/rpc/rpc-builder/src/eth.rs +++ b/crates/rpc/rpc-builder/src/eth.rs @@ -3,7 +3,7 @@ use reth_rpc::{ eth::{ cache::{EthStateCache, EthStateCacheConfig}, gas_oracle::GasPriceOracleConfig, - RPC_DEFAULT_GAS_CAP, + FeeHistoryCacheConfig, RPC_DEFAULT_GAS_CAP, }, BlockingTaskPool, EthApi, EthFilter, EthPubSub, }; @@ -42,6 +42,8 @@ pub struct EthConfig { /// /// Sets TTL for stale filters pub stale_filter_ttl: std::time::Duration, + /// Settings for the fee history cache + pub fee_history_cache: FeeHistoryCacheConfig, } /// Default value for stale filter ttl @@ -56,6 +58,7 @@ impl Default for EthConfig { max_logs_per_response: DEFAULT_MAX_LOGS_PER_RESPONSE, rpc_gas_cap: RPC_DEFAULT_GAS_CAP.into(), stale_filter_ttl: DEFAULT_STALE_FILTER_TTL, + fee_history_cache: FeeHistoryCacheConfig::default(), } } } diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 7105a4de030a..9085237fb22e 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -115,7 +115,9 @@ use reth_provider::{ use reth_rpc::{ eth::{ cache::{cache_new_blocks_task, EthStateCache}, + fee_history_cache_new_blocks_task, gas_oracle::GasPriceOracle, + FeeHistoryCache, }, AdminApi, BlockingTaskGuard, BlockingTaskPool, DebugApi, EngineEthApi, EthApi, EthFilter, EthPubSub, EthSubscriptionIdProvider, NetApi, OtterscanApi, RPCApi, RethApi, TraceApi, @@ -1019,13 +1021,22 @@ where ); let new_canonical_blocks = self.events.canonical_state_stream(); let c = cache.clone(); + + let fee_history_cache = FeeHistoryCache::new(self.config.eth.fee_history_cache.clone()); + let fhc = fee_history_cache.clone(); + self.executor.spawn_critical( "cache canonical blocks task", Box::pin(async move { cache_new_blocks_task(c, new_canonical_blocks).await; }), ); - + self.executor.spawn_critical( + "cache canonical blocks for fee history task", + Box::pin(async move { + fee_history_cache_new_blocks_task(fhc, new_canonical_blocks).await; + }), + ); let executor = Box::new(self.executor.clone()); let blocking_task_pool = BlockingTaskPool::build().expect("failed to build tracing pool"); diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 18799cd30810..14cce7f898e4 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -5,9 +5,10 @@ use crate::{ EthApi, }; -use crate::eth::api::BlockFees; use reth_network_api::NetworkInfo; -use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, B256, U256}; +use reth_primitives::{ + basefee::calculate_next_block_base_fee, BlockNumberOrTag, SealedHeader, U256, +}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::{FeeHistory, TxGasAndReward}; use reth_transaction_pool::TransactionPool; @@ -48,8 +49,6 @@ where return Ok(FeeHistory::default()) } - let mut fee_history_cache = self.inner.fee_history_cache.lock().await; - // See https://github.com/ethereum/go-ethereum/blob/2754b197c935ee63101cbbca2752338246384fec/eth/gasprice/feehistory.go#L218C8-L225 let max_fee_history = if reward_percentiles.is_none() { self.gas_oracle().config().max_header_history @@ -96,80 +95,46 @@ where let start_block = end_block_plus - block_count; + let headers: Vec; + if start_block < (end_block - max_fee_history) { + headers = self.provider().sealed_headers_range(start_block..=end_block)?; + } else { + let read_guard = self.fee_history_cache().entries.read().await; + headers = read_guard + .range(start_block..=end_block) + .map(|(_, header)| header.clone()) + .collect(); + } + if headers.len() != block_count as usize { + return Err(EthApiError::InvalidBlockRange) + } // Collect base fees, gas usage ratios and (optionally) reward percentile data let mut base_fee_per_gas: Vec = Vec::new(); let mut gas_used_ratio: Vec = Vec::new(); let mut rewards: Vec> = Vec::new(); - struct LastBlock { - gas_used: u64, - gas_limit: u64, - base_fee_per_gas: u64, - } - - let Some(end_block_header) = self.provider().sealed_header(end_block)? else { - return Err(EthApiError::InvalidBlockRange) - }; - - let mut last_block = LastBlock{gas_used: 0, gas_limit: 0, base_fee_per_gas: 0}; - - let mut current_hash = end_block_header.hash; - let mut parent_hash: B256; - for n in (start_block..end_block_plus).rev() { - let block_fees: BlockFees; - - if let Some(cached_block_fees) = fee_history_cache.get(¤t_hash) { - block_fees = cached_block_fees.to_owned(); - parent_hash = block_fees.parent_hash; - } else { - let Some(current_header) = self.provider().sealed_header(n)? else { - return Err(EthApiError::InvalidBlockRange); - }; - - parent_hash = current_header.parent_hash; - block_fees = BlockFees::from_header(current_header.header.clone()); - fee_history_cache.insert(current_hash, block_fees.clone()); - }; - - base_fee_per_gas.push(U256::try_from(block_fees.base_fee_per_gas.clone()).unwrap()); - gas_used_ratio.push(block_fees.gas_used_ratio.clone()); - - if n == end_block { - last_block = LastBlock { - gas_used: block_fees.gas_used, - gas_limit: block_fees.gas_limit, - base_fee_per_gas: block_fees.base_fee_per_gas, - }; - } - // Percentiles were specified, so we need to collect reward percentile ino + for header in &headers { + base_fee_per_gas + .push(U256::try_from(header.base_fee_per_gas.unwrap_or_default()).unwrap()); + gas_used_ratio.push(header.gas_used as f64 / header.gas_limit as f64); if let Some(percentiles) = &reward_percentiles { - rewards.push( - self.calculate_reward_percentiles( - percentiles, - current_hash.clone(), - &block_fees, - ) - .await?, - ); + rewards.push(self.calculate_reward_percentiles(percentiles, header).await?); } - - current_hash = parent_hash; } - // Collect base fees, gas usage ratios and (optionally) reward percentile data - // The spec states that `base_fee_per_gas` "[..] includes the next block after the newest of // the returned range, because this value can be derived from the newest block" // // The unwrap is safe since we checked earlier that we got at least 1 header. + + let last_header = headers.last().unwrap(); + let chain_spec = self.provider().chain_spec(); - base_fee_per_gas.reverse(); - gas_used_ratio.reverse(); base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( - last_block.gas_used, - last_block.gas_limit, - last_block.base_fee_per_gas, + last_header.gas_used, + last_header.gas_limit, + last_header.base_fee_per_gas.unwrap_or_default(), chain_spec.base_fee_params, ))); @@ -189,12 +154,11 @@ where async fn calculate_reward_percentiles( &self, percentiles: &[f64], - header_hash: B256, - block_fees: &BlockFees, + header: &SealedHeader, ) -> Result, EthApiError> { let (transactions, receipts) = self .cache() - .get_transactions_and_receipts(header_hash) + .get_transactions_and_receipts(header.hash) .await? .ok_or(EthApiError::InvalidBlockRange)?; @@ -213,9 +177,7 @@ where Some(TxGasAndReward { gas_used, - reward: tx - .effective_gas_tip(Some(block_fees.base_fee_per_gas)) - .unwrap_or_default(), + reward: tx.effective_gas_tip(header.base_fee_per_gas).unwrap_or_default(), }) }) .collect::>(); @@ -238,7 +200,7 @@ where continue } - let threshold = (block_fees.gas_used as f64 * percentile / 100.) as u64; + let threshold = (header.gas_used as f64 * percentile / 100.) as u64; while cumulative_gas_used < threshold && tx_index < transactions.len() - 1 { tx_index += 1; cumulative_gas_used += transactions[tx_index].gas_used; diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index d39a9c43044d..3cf1ef7e9d6f 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -1,5 +1,3 @@ -//! Provides everything related to `eth_` namespace -//! //! The entire implementation of the namespace is quite large, hence it is divided across several //! files. @@ -11,27 +9,35 @@ use crate::eth::{ signer::EthSigner, }; use async_trait::async_trait; -use derive_more::{Deref, DerefMut}; use reth_interfaces::RethResult; use reth_network_api::NetworkInfo; use reth_primitives::{ - Address, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, Header, SealedHeader, B256, U256, U64, + Address, BlockId, BlockNumberOrTag, ChainInfo, Header, SealedBlock, SealedHeader, B256, U256, + U64, }; use reth_provider::{ - BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderBox, StateProviderFactory, + BlockReaderIdExt, CanonStateNotification, ChainSpecProvider, EvmEnvProvider, StateProviderBox, + StateProviderFactory, }; use reth_rpc_types::{SyncInfo, SyncStatus}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use reth_transaction_pool::TransactionPool; use revm_primitives::{BlockEnv, CfgEnv}; -use schnellru::{ByLength, LruMap}; +use serde::{Deserialize, Serialize}; use std::{ - fmt::{self, Debug, Formatter}, + collections::BTreeMap, + fmt::Debug, future::Future, sync::Arc, time::{Duration, Instant}, }; -use tokio::sync::{oneshot, Mutex}; + +use futures::{future::Either, Stream, StreamExt}; +use tokio::sync::{ + mpsc::{unbounded_channel, UnboundedSender}, + oneshot, Mutex, +}; +use tokio_stream::wrappers::UnboundedReceiverStream; mod block; mod call; @@ -97,6 +103,7 @@ where gas_oracle: GasPriceOracle, gas_cap: impl Into, blocking_task_pool: BlockingTaskPool, + fee_history_cache: FeeHistoryCache, ) -> Self { Self::with_spawner( provider, @@ -107,6 +114,7 @@ where gas_cap.into().into(), Box::::default(), blocking_task_pool, + fee_history_cache, ) } @@ -121,6 +129,7 @@ where gas_cap: u64, task_spawner: Box, blocking_task_pool: BlockingTaskPool, + fee_history_cache: FeeHistoryCache, ) -> Self { // get the block number of the latest block let latest_block = provider @@ -142,9 +151,12 @@ where task_spawner, pending_block: Default::default(), blocking_task_pool, - fee_history_cache: Mutex::new(FeeHistoryLruCache(LruMap::new(ByLength::new(1024)))), + fee_history_cache, }; - Self { inner: Arc::new(inner) } + + let eth_api = Self { inner: Arc::new(inner) }; + + eth_api } /// Executes the future on a new blocking task. @@ -198,7 +210,7 @@ where } /// Returns fee history cache - pub fn fee_history_cache(&self) -> &Mutex { + pub fn fee_history_cache(&self) -> &FeeHistoryCache { &self.inner.fee_history_cache } } @@ -450,7 +462,7 @@ struct EthApiInner { /// A pool dedicated to blocking tasks. blocking_task_pool: BlockingTaskPool, /// Cache for block fees history - fee_history_cache: Mutex, + fee_history_cache: FeeHistoryCache, } /// The type that contains fees data for associated block in cache @@ -464,8 +476,6 @@ pub struct BlockFees { pub gas_used: u64, /// Block data on gas_limit pub gas_limit: u64, - /// parent block hash - pub parent_hash: B256, } impl BlockFees { @@ -475,20 +485,67 @@ impl BlockFees { gas_used_ratio: header.gas_used as f64 / header.gas_limit as f64, gas_used: header.gas_used, gas_limit: header.gas_limit, - parent_hash: header.parent_hash, } } } -/// Wrapper struct for LruMap -#[derive(Deref, DerefMut)] -pub struct FeeHistoryLruCache(LruMap); +/// Settings for the [EthStateCache](crate::eth::cache::EthStateCache). +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FeeHistoryCacheConfig { + /// Max number of blocks in cache. + /// + /// Default is 1024. + pub max_blocks: u32, +} + +impl Default for FeeHistoryCacheConfig { + fn default() -> Self { + FeeHistoryCacheConfig { max_blocks: 1024 } + } +} + +/// Wrapper struct for BTreeMap +pub(crate) struct FeeHistoryCache { + config: FeeHistoryCacheConfig, + entries: tokio::sync::RwLock>, +} + +impl FeeHistoryCache { + pub fn new(config: FeeHistoryCacheConfig) -> Self { + let mut entries = tokio::sync::RwLock::new(BTreeMap::new()); + let cache = FeeHistoryCache { config, entries }; + + cache + } + + pub async fn on_new_block(&self, block: &SealedBlock) { + let mut entries = self.entries.write().await; + entries.insert(block.number, block.header.clone()); + while entries.len() > self.config.max_blocks as usize { + entries.pop_first(); + } + } +} + +/// Awaits for new chain events and directly inserts them into the cache so they're available +/// immediately before they need to be fetched from disk. +pub(crate) async fn fee_history_cache_new_blocks_task( + fee_history_cache: FeeHistoryCache, + mut events: St, +) where + St: Stream + Unpin + 'static, +{ + while let Some(event) = events.next().await { + if let Some(committed) = event.committed() { + // we're only interested in new committed blocks + let (blocks, _) = committed.inner(); + + let blocks = blocks.iter().map(|(_, block)| block.block.clone()).collect::>(); -impl Debug for FeeHistoryLruCache { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("FeeHistoryLruCache") - .field("cache_length", &self.len()) - .field("cache_memory_usage", &self.memory_usage()) - .finish() + for block in blocks { + fee_history_cache.on_new_block(&block).await; + } + } } } diff --git a/crates/rpc/rpc/src/eth/cache/mod.rs b/crates/rpc/rpc/src/eth/cache/mod.rs index 68851fbe131f..d70442a63dfa 100644 --- a/crates/rpc/rpc/src/eth/cache/mod.rs +++ b/crates/rpc/rpc/src/eth/cache/mod.rs @@ -1,6 +1,5 @@ //! Async caching support for eth RPC -use alloy_primitives::BlockNumber; use futures::{future::Either, Stream, StreamExt}; use reth_interfaces::{provider::ProviderError, RethResult}; use reth_primitives::{Block, Receipt, SealedBlock, TransactionSigned, B256}; @@ -484,6 +483,7 @@ where // also cache all receipts of the blocks let mut receipts = Vec::with_capacity(blocks.len()); + for block in &blocks { let block_receipts = BlockReceipts { block_hash: block.hash, From 93ed17967b605342fe3ac17fd49aebf9ae02815f Mon Sep 17 00:00:00 2001 From: allnil Date: Tue, 31 Oct 2023 22:38:14 +0000 Subject: [PATCH 13/32] add atomic bounds, refactor cache and provider --- crates/rpc/rpc/src/eth/api/fees.rs | 12 +-- crates/rpc/rpc/src/eth/api/mod.rs | 87 +++++++++++++++++----- crates/rpc/rpc/src/eth/api/server.rs | 5 +- crates/rpc/rpc/src/eth/api/state.rs | 6 +- crates/rpc/rpc/src/eth/api/transactions.rs | 8 +- crates/rpc/rpc/src/eth/mod.rs | 5 +- 6 files changed, 86 insertions(+), 37 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 14cce7f898e4..d24a12933878 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -95,16 +95,8 @@ where let start_block = end_block_plus - block_count; - let headers: Vec; - if start_block < (end_block - max_fee_history) { - headers = self.provider().sealed_headers_range(start_block..=end_block)?; - } else { - let read_guard = self.fee_history_cache().entries.read().await; - headers = read_guard - .range(start_block..=end_block) - .map(|(_, header)| header.clone()) - .collect(); - } + let headers = self.fee_history_cache().get_history(start_block, end_block).await?; + if headers.len() != block_count as usize { return Err(EthApiError::InvalidBlockRange) } diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 3cf1ef7e9d6f..782e1b7402f6 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -9,6 +9,7 @@ use crate::eth::{ signer::EthSigner, }; use async_trait::async_trait; +use metrics::atomics::AtomicU64; use reth_interfaces::RethResult; use reth_network_api::NetworkInfo; use reth_primitives::{ @@ -17,7 +18,7 @@ use reth_primitives::{ }; use reth_provider::{ BlockReaderIdExt, CanonStateNotification, ChainSpecProvider, EvmEnvProvider, StateProviderBox, - StateProviderFactory, + StateProviderFactory, BlockNumReader, }; use reth_rpc_types::{SyncInfo, SyncStatus}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; @@ -28,16 +29,15 @@ use std::{ collections::BTreeMap, fmt::Debug, future::Future, - sync::Arc, + sync::Arc, + sync::atomic::Ordering::SeqCst, time::{Duration, Instant}, }; -use futures::{future::Either, Stream, StreamExt}; +use futures::{Stream, StreamExt}; use tokio::sync::{ - mpsc::{unbounded_channel, UnboundedSender}, oneshot, Mutex, }; -use tokio_stream::wrappers::UnboundedReceiverStream; mod block; mod call; @@ -103,7 +103,7 @@ where gas_oracle: GasPriceOracle, gas_cap: impl Into, blocking_task_pool: BlockingTaskPool, - fee_history_cache: FeeHistoryCache, + fee_history_cache: FeeHistoryCache, ) -> Self { Self::with_spawner( provider, @@ -129,7 +129,7 @@ where gas_cap: u64, task_spawner: Box, blocking_task_pool: BlockingTaskPool, - fee_history_cache: FeeHistoryCache, + fee_history_cache: FeeHistoryCache, ) -> Self { // get the block number of the latest block let latest_block = provider @@ -210,7 +210,7 @@ where } /// Returns fee history cache - pub fn fee_history_cache(&self) -> &FeeHistoryCache { + pub fn fee_history_cache(&self) -> &FeeHistoryCache { &self.inner.fee_history_cache } } @@ -462,7 +462,7 @@ struct EthApiInner { /// A pool dedicated to blocking tasks. blocking_task_pool: BlockingTaskPool, /// Cache for block fees history - fee_history_cache: FeeHistoryCache, + fee_history_cache: FeeHistoryCache, } /// The type that contains fees data for associated block in cache @@ -496,7 +496,7 @@ pub struct FeeHistoryCacheConfig { /// Max number of blocks in cache. /// /// Default is 1024. - pub max_blocks: u32, + pub max_blocks: u64, } impl Default for FeeHistoryCacheConfig { @@ -506,16 +506,48 @@ impl Default for FeeHistoryCacheConfig { } /// Wrapper struct for BTreeMap -pub(crate) struct FeeHistoryCache { +#[derive(Debug, Clone)] +pub struct FeeHistoryCache { + lower_bound: Arc, + upper_bound: Arc, config: FeeHistoryCacheConfig, - entries: tokio::sync::RwLock>, + provider: Provider, + entries: Arc>>, } -impl FeeHistoryCache { - pub fn new(config: FeeHistoryCacheConfig) -> Self { - let mut entries = tokio::sync::RwLock::new(BTreeMap::new()); - let cache = FeeHistoryCache { config, entries }; - +impl FeeHistoryCache +where + Provider: + BlockReaderIdExt + ChainSpecProvider + StateProviderFactory + EvmEnvProvider + BlockNumReader + 'static, +{ + pub fn new( + config: FeeHistoryCacheConfig, + provider: Provider, + ) -> Self { + let mut init_tree_map = BTreeMap::new(); + + let last_block_number = provider.last_block_number().unwrap_or(0); + let start_block = last_block_number - config.max_blocks; + let headers = provider.sealed_headers_range(start_block..=last_block_number).unwrap_or_default(); + + for header in headers { + init_tree_map.insert(header.number, header); + } + let upper_bound = *init_tree_map.last_key_value().expect("Chain has at least 1 block").0; + let lower_bound = *init_tree_map.first_key_value().expect("Chain has at least one block").0; + + let entries = Arc::new(tokio::sync::RwLock::new(init_tree_map)); + + let upper_bound = Arc::new(AtomicU64::new(upper_bound)); + let lower_bound = Arc::new(AtomicU64::new(lower_bound)); + + let cache = FeeHistoryCache { + config, + entries, + provider, + upper_bound, + lower_bound, + }; cache } @@ -526,15 +558,32 @@ impl FeeHistoryCache { entries.pop_first(); } } + + pub async fn get_history(&self, start_block: u64, end_block: u64) -> RethResult> { + let headers: Vec; + + let lower_bound = self.lower_bound.load(SeqCst); + let upper_bound = self.upper_bound.load(SeqCst); + if start_block >= lower_bound && end_block <= upper_bound { + let entries = self.entries.read().await; + headers = entries.range(start_block..=end_block+1).map(|(_, header)| header.clone()).collect(); + } else { + headers = self.provider.sealed_headers_range(start_block..=end_block)?; + } + + Ok(headers) + } } /// Awaits for new chain events and directly inserts them into the cache so they're available /// immediately before they need to be fetched from disk. -pub(crate) async fn fee_history_cache_new_blocks_task( - fee_history_cache: FeeHistoryCache, +pub async fn fee_history_cache_new_blocks_task( + fee_history_cache: FeeHistoryCache, mut events: St, ) where St: Stream + Unpin + 'static, + Provider: + BlockReaderIdExt + ChainSpecProvider + StateProviderFactory + EvmEnvProvider + BlockNumReader + 'static, { while let Some(event) = events.next().await { if let Some(committed) = event.committed() { diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index e1896c97ae87..264ad7e4096d 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -385,7 +385,7 @@ where #[cfg(test)] mod tests { use crate::{ - eth::{cache::EthStateCache, gas_oracle::GasPriceOracle}, + eth::{cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, FeeHistoryCacheConfig}, BlockingTaskPool, EthApi, }; use jsonrpsee::types::error::INVALID_PARAMS_CODE; @@ -422,9 +422,10 @@ mod tests { testing_pool(), NoopNetwork::default(), cache.clone(), - GasPriceOracle::new(provider, Default::default(), cache), + GasPriceOracle::new(provider.clone(), Default::default(), cache), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), + FeeHistoryCache::new(FeeHistoryCacheConfig::default(), provider.clone()), ) } diff --git a/crates/rpc/rpc/src/eth/api/state.rs b/crates/rpc/rpc/src/eth/api/state.rs index 4cf4a69a6aa8..503e5529aba0 100644 --- a/crates/rpc/rpc/src/eth/api/state.rs +++ b/crates/rpc/rpc/src/eth/api/state.rs @@ -125,7 +125,7 @@ where mod tests { use super::*; use crate::{ - eth::{cache::EthStateCache, gas_oracle::GasPriceOracle}, + eth::{cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, FeeHistoryCacheConfig}, BlockingTaskPool, }; use reth_primitives::{constants::ETHEREUM_BLOCK_GAS_LIMIT, StorageKey, StorageValue}; @@ -147,6 +147,7 @@ mod tests { GasPriceOracle::new(NoopProvider::default(), Default::default(), cache), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), + FeeHistoryCache::new(FeeHistoryCacheConfig::default(), NoopProvider::default()), ); let address = Address::random(); let storage = eth_api.storage_at(address, U256::ZERO.into(), None).unwrap(); @@ -166,9 +167,10 @@ mod tests { pool, (), cache.clone(), - GasPriceOracle::new(mock_provider, Default::default(), cache), + GasPriceOracle::new(mock_provider.clone(), Default::default(), cache), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), + FeeHistoryCache::new(FeeHistoryCacheConfig::default(), mock_provider.clone()), ); let storage_key: U256 = storage_key.into(); diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index f3739451d9ff..8287f2ee7150 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -7,9 +7,9 @@ use crate::{ inspect, inspect_and_return_db, prepare_call_env, replay_transactions_until, transact, EvmOverrides, }, - utils::recover_raw_transaction, + utils::recover_raw_transaction, }, - EthApi, EthApiSpec, + EthApi, EthApiSpec, }; use async_trait::async_trait; use reth_network_api::NetworkInfo; @@ -1090,7 +1090,7 @@ pub(crate) fn build_transaction_receipt_with_block_receipts( mod tests { use super::*; use crate::{ - eth::{cache::EthStateCache, gas_oracle::GasPriceOracle}, + eth::{cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, FeeHistoryCacheConfig}, BlockingTaskPool, EthApi, }; use reth_network_api::noop::NoopNetwork; @@ -1106,6 +1106,7 @@ mod tests { let pool = testing_pool(); let cache = EthStateCache::spawn(noop_provider, Default::default()); + let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default(), noop_provider); let eth_api = EthApi::new( noop_provider, pool.clone(), @@ -1114,6 +1115,7 @@ mod tests { GasPriceOracle::new(noop_provider, Default::default(), cache), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), + fee_history_cache, ); // https://etherscan.io/tx/0xa694b71e6c128a2ed8e2e0f6770bddbe52e3bb8f10e8472f9a79ab81497a8b5d diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index 77baa36e32dd..82eb104d7c1c 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -13,7 +13,10 @@ pub mod revm_utils; mod signer; pub(crate) mod utils; -pub use api::{EthApi, EthApiSpec, EthTransactions, TransactionSource, RPC_DEFAULT_GAS_CAP}; +pub use api::{ + fee_history_cache_new_blocks_task, EthApi, EthApiSpec, EthTransactions, FeeHistoryCache, + FeeHistoryCacheConfig, TransactionSource, RPC_DEFAULT_GAS_CAP, +}; pub use bundle::EthBundle; pub use filter::EthFilter; pub use id_provider::EthSubscriptionIdProvider; From 1ae2aae04c423ba11f5375a7c82359790b2d5a70 Mon Sep 17 00:00:00 2001 From: allnil Date: Wed, 1 Nov 2023 15:46:18 +0000 Subject: [PATCH 14/32] handle empty block intervals and fee history --- crates/rpc/rpc-builder/src/auth.rs | 7 +- crates/rpc/rpc-builder/src/lib.rs | 12 +- crates/rpc/rpc/src/eth/api/fees.rs | 2 +- crates/rpc/rpc/src/eth/api/mod.rs | 132 ++++++++++----------- crates/rpc/rpc/src/eth/api/server.rs | 5 +- crates/rpc/rpc/src/eth/api/state.rs | 5 +- crates/rpc/rpc/src/eth/api/transactions.rs | 12 +- 7 files changed, 96 insertions(+), 79 deletions(-) diff --git a/crates/rpc/rpc-builder/src/auth.rs b/crates/rpc/rpc-builder/src/auth.rs index f3a1ba660820..136ea05406a8 100644 --- a/crates/rpc/rpc-builder/src/auth.rs +++ b/crates/rpc/rpc-builder/src/auth.rs @@ -16,7 +16,9 @@ use reth_provider::{ StateProviderFactory, }; use reth_rpc::{ - eth::{cache::EthStateCache, gas_oracle::GasPriceOracle}, + eth::{ + cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, FeeHistoryCacheConfig, + }, AuthLayer, BlockingTaskPool, Claims, EngineEthApi, EthApi, EthFilter, EthSubscriptionIdProvider, JwtAuthValidator, JwtSecret, }; @@ -58,6 +60,8 @@ where let eth_cache = EthStateCache::spawn_with(provider.clone(), Default::default(), executor.clone()); let gas_oracle = GasPriceOracle::new(provider.clone(), Default::default(), eth_cache.clone()); + let fee_history_cache = + FeeHistoryCache::new(FeeHistoryCacheConfig::default(), provider.clone()); let eth_api = EthApi::with_spawner( provider.clone(), pool.clone(), @@ -67,6 +71,7 @@ where EthConfig::default().rpc_gas_cap, Box::new(executor.clone()), BlockingTaskPool::build().expect("failed to build tracing pool"), + fee_history_cache, ); let eth_filter = EthFilter::new( provider, diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 9085237fb22e..7328cb0495b8 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1022,21 +1022,26 @@ where let new_canonical_blocks = self.events.canonical_state_stream(); let c = cache.clone(); - let fee_history_cache = FeeHistoryCache::new(self.config.eth.fee_history_cache.clone()); - let fhc = fee_history_cache.clone(); - + let fee_history_cache = FeeHistoryCache::new( + self.config.eth.fee_history_cache.clone(), + self.provider.clone(), + ); self.executor.spawn_critical( "cache canonical blocks task", Box::pin(async move { cache_new_blocks_task(c, new_canonical_blocks).await; }), ); + + let new_canonical_blocks = self.events.canonical_state_stream(); + let fhc = fee_history_cache.clone(); self.executor.spawn_critical( "cache canonical blocks for fee history task", Box::pin(async move { fee_history_cache_new_blocks_task(fhc, new_canonical_blocks).await; }), ); + let executor = Box::new(self.executor.clone()); let blocking_task_pool = BlockingTaskPool::build().expect("failed to build tracing pool"); @@ -1049,6 +1054,7 @@ where self.config.eth.rpc_gas_cap, executor.clone(), blocking_task_pool.clone(), + fee_history_cache, ); let filter = EthFilter::new( self.provider.clone(), diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index d24a12933878..213fc1ac873c 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -96,7 +96,7 @@ where let start_block = end_block_plus - block_count; let headers = self.fee_history_cache().get_history(start_block, end_block).await?; - + if headers.len() != block_count as usize { return Err(EthApiError::InvalidBlockRange) } diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 782e1b7402f6..5ca07ecd5935 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -13,12 +13,11 @@ use metrics::atomics::AtomicU64; use reth_interfaces::RethResult; use reth_network_api::NetworkInfo; use reth_primitives::{ - Address, BlockId, BlockNumberOrTag, ChainInfo, Header, SealedBlock, SealedHeader, B256, U256, - U64, + Address, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, SealedHeader, B256, U256, U64, }; use reth_provider::{ - BlockReaderIdExt, CanonStateNotification, ChainSpecProvider, EvmEnvProvider, StateProviderBox, - StateProviderFactory, BlockNumReader, + BlockNumReader, BlockReaderIdExt, CanonStateNotification, ChainSpecProvider, EvmEnvProvider, + StateProviderBox, StateProviderFactory, }; use reth_rpc_types::{SyncInfo, SyncStatus}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; @@ -29,15 +28,12 @@ use std::{ collections::BTreeMap, fmt::Debug, future::Future, - sync::Arc, - sync::atomic::Ordering::SeqCst, + sync::{atomic::Ordering::SeqCst, Arc}, time::{Duration, Instant}, }; use futures::{Stream, StreamExt}; -use tokio::sync::{ - oneshot, Mutex, -}; +use tokio::sync::{oneshot, Mutex}; mod block; mod call; @@ -51,8 +47,6 @@ mod transactions; use crate::BlockingTaskPool; pub use transactions::{EthTransactions, TransactionSource}; -use super::revm_utils::FillableTransaction; - /// `Eth` API trait. /// /// Defines core functionality of the `eth` API implementation. @@ -154,9 +148,7 @@ where fee_history_cache, }; - let eth_api = Self { inner: Arc::new(inner) }; - - eth_api + Self { inner: Arc::new(inner) } } /// Executes the future on a new blocking task. @@ -465,30 +457,6 @@ struct EthApiInner { fee_history_cache: FeeHistoryCache, } -/// The type that contains fees data for associated block in cache -#[derive(Clone, Debug)] -pub struct BlockFees { - /// Block data on bas_fee_per_gas - pub base_fee_per_gas: u64, - /// Precalculated ratio - pub gas_used_ratio: f64, - /// Block data on gas_used - pub gas_used: u64, - /// Block data on gas_limit - pub gas_limit: u64, -} - -impl BlockFees { - fn from_header(header: Header) -> BlockFees { - BlockFees { - base_fee_per_gas: header.base_fee_per_gas.unwrap_or_default(), - gas_used_ratio: header.gas_used as f64 / header.gas_limit as f64, - gas_used: header.gas_used, - gas_limit: header.gas_limit, - } - } -} - /// Settings for the [EthStateCache](crate::eth::cache::EthStateCache). #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -517,60 +485,84 @@ pub struct FeeHistoryCache { impl FeeHistoryCache where - Provider: - BlockReaderIdExt + ChainSpecProvider + StateProviderFactory + EvmEnvProvider + BlockNumReader + 'static, + Provider: BlockReaderIdExt + + ChainSpecProvider + + StateProviderFactory + + EvmEnvProvider + + BlockNumReader + + 'static, { - pub fn new( - config: FeeHistoryCacheConfig, - provider: Provider, - ) -> Self { + /// Creates new FeeHistoryCache instance, initialize it with the mose recent data, set bounds + pub fn new(config: FeeHistoryCacheConfig, provider: Provider) -> Self { let mut init_tree_map = BTreeMap::new(); - let last_block_number = provider.last_block_number().unwrap_or(0); - let start_block = last_block_number - config.max_blocks; - let headers = provider.sealed_headers_range(start_block..=last_block_number).unwrap_or_default(); - + let last_block_number = provider.last_block_number().unwrap_or(0); + + let start_block = if last_block_number > config.max_blocks { + last_block_number - config.max_blocks + } else { + 0 + }; + + let headers = + provider.sealed_headers_range(start_block..=last_block_number).unwrap_or_default(); + + let (mut lower_bound, mut upper_bound) = (0, 0); for header in headers { - init_tree_map.insert(header.number, header); + init_tree_map.insert(header.number, header.clone()); + upper_bound = header.number; + if lower_bound == 0 { + lower_bound = header.number; + } } - let upper_bound = *init_tree_map.last_key_value().expect("Chain has at least 1 block").0; - let lower_bound = *init_tree_map.first_key_value().expect("Chain has at least one block").0; - + let entries = Arc::new(tokio::sync::RwLock::new(init_tree_map)); - + let upper_bound = Arc::new(AtomicU64::new(upper_bound)); let lower_bound = Arc::new(AtomicU64::new(lower_bound)); - let cache = FeeHistoryCache { - config, - entries, - provider, - upper_bound, - lower_bound, - }; - cache + FeeHistoryCache { config, entries, provider, upper_bound, lower_bound } } + /// Processing of the arriving blocks pub async fn on_new_block(&self, block: &SealedBlock) { let mut entries = self.entries.write().await; entries.insert(block.number, block.header.clone()); while entries.len() > self.config.max_blocks as usize { entries.pop_first(); } + if entries.len() == 0 { + self.upper_bound.store(0, SeqCst); + self.lower_bound.store(0, SeqCst); + return + } + let upper_bound = *entries.last_entry().expect("Contains be at least one entry").key(); + let lower_bound = *entries.first_entry().expect("Contains at least one entry").key(); + self.upper_bound.store(upper_bound, SeqCst); + self.lower_bound.store(lower_bound, SeqCst); } - pub async fn get_history(&self, start_block: u64, end_block: u64) -> RethResult> { + /// Collect fee history for given range. It will try to use a cache to take the most recent + /// headers or if the range is out of caching config it will fallback to the database provider + pub async fn get_history( + &self, + start_block: u64, + end_block: u64, + ) -> RethResult> { let headers: Vec; - + let lower_bound = self.lower_bound.load(SeqCst); - let upper_bound = self.upper_bound.load(SeqCst); + let upper_bound = self.upper_bound.load(SeqCst); if start_block >= lower_bound && end_block <= upper_bound { let entries = self.entries.read().await; - headers = entries.range(start_block..=end_block+1).map(|(_, header)| header.clone()).collect(); + headers = entries + .range(start_block..=end_block + 1) + .map(|(_, header)| header.clone()) + .collect(); } else { headers = self.provider.sealed_headers_range(start_block..=end_block)?; } - + Ok(headers) } } @@ -582,8 +574,12 @@ pub async fn fee_history_cache_new_blocks_task( mut events: St, ) where St: Stream + Unpin + 'static, - Provider: - BlockReaderIdExt + ChainSpecProvider + StateProviderFactory + EvmEnvProvider + BlockNumReader + 'static, + Provider: BlockReaderIdExt + + ChainSpecProvider + + StateProviderFactory + + EvmEnvProvider + + BlockNumReader + + 'static, { while let Some(event) = events.next().await { if let Some(committed) = event.committed() { diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index 264ad7e4096d..cb45598b3624 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -385,7 +385,10 @@ where #[cfg(test)] mod tests { use crate::{ - eth::{cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, FeeHistoryCacheConfig}, + eth::{ + cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, + FeeHistoryCacheConfig, + }, BlockingTaskPool, EthApi, }; use jsonrpsee::types::error::INVALID_PARAMS_CODE; diff --git a/crates/rpc/rpc/src/eth/api/state.rs b/crates/rpc/rpc/src/eth/api/state.rs index 503e5529aba0..dde1e574bc15 100644 --- a/crates/rpc/rpc/src/eth/api/state.rs +++ b/crates/rpc/rpc/src/eth/api/state.rs @@ -125,7 +125,10 @@ where mod tests { use super::*; use crate::{ - eth::{cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, FeeHistoryCacheConfig}, + eth::{ + cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, + FeeHistoryCacheConfig, + }, BlockingTaskPool, }; use reth_primitives::{constants::ETHEREUM_BLOCK_GAS_LIMIT, StorageKey, StorageValue}; diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index 8287f2ee7150..3f241ccd0bfe 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -7,9 +7,9 @@ use crate::{ inspect, inspect_and_return_db, prepare_call_env, replay_transactions_until, transact, EvmOverrides, }, - utils::recover_raw_transaction, + utils::recover_raw_transaction, }, - EthApi, EthApiSpec, + EthApi, EthApiSpec, }; use async_trait::async_trait; use reth_network_api::NetworkInfo; @@ -1090,7 +1090,10 @@ pub(crate) fn build_transaction_receipt_with_block_receipts( mod tests { use super::*; use crate::{ - eth::{cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, FeeHistoryCacheConfig}, + eth::{ + cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, + FeeHistoryCacheConfig, + }, BlockingTaskPool, EthApi, }; use reth_network_api::noop::NoopNetwork; @@ -1106,7 +1109,8 @@ mod tests { let pool = testing_pool(); let cache = EthStateCache::spawn(noop_provider, Default::default()); - let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default(), noop_provider); + let fee_history_cache = + FeeHistoryCache::new(FeeHistoryCacheConfig::default(), noop_provider); let eth_api = EthApi::new( noop_provider, pool.clone(), From b3c2d8db5945b698611604e2cbabf4d36237c427 Mon Sep 17 00:00:00 2001 From: allnil Date: Fri, 3 Nov 2023 21:41:11 +0000 Subject: [PATCH 15/32] prepare ground for futher refactoring, remove provider from FeeHistoryCache type --- crates/rpc/rpc-engine-api/src/engine_api.rs | 6 +-- crates/rpc/rpc/src/eth/api/fees.rs | 23 ++++++++-- crates/rpc/rpc/src/eth/api/mod.rs | 49 +++------------------ 3 files changed, 28 insertions(+), 50 deletions(-) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 5218e1b963cc..7fd29331d0ff 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -262,12 +262,12 @@ where self.inner.task_spawner.spawn_blocking(Box::pin(async move { if count > MAX_PAYLOAD_BODIES_LIMIT { tx.send(Err(EngineApiError::PayloadRequestTooLarge { len: count })).ok(); - return + return; } if start == 0 || count == 0 { tx.send(Err(EngineApiError::InvalidBodiesRange { start, count })).ok(); - return + return; } let mut result = Vec::with_capacity(count as usize); @@ -291,7 +291,7 @@ where } Err(err) => { tx.send(Err(EngineApiError::Internal(Box::new(err)))).ok(); - return + return; } }; } diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 213fc1ac873c..0f6b7fd7d647 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -76,10 +76,9 @@ where block_count = end_block_plus; } - // If reward percentiles were specified, we + schnellru::Limiterneed to validate that they are monotonically + // If reward percentiles were specified, we + // need to validate that they are monotonically // increasing and 0 <= p <= 100 - // // Note: The types used ensure that the percentiles are never < 0 if let Some(percentiles) = &reward_percentiles { if percentiles.windows(2).any(|w| w[0] > w[1] || w[0] > 100.) { @@ -203,3 +202,21 @@ where Ok(rewards_in_block) } } + +/// Collect fee history for given range. It will try to use a cache to take the most recent +/// headers or if the range is out of caching config it will fallback to the database provider +pub async fn get_history(start_block: u64, end_block: u64) -> RethResult> { + let headers: Vec; + + let lower_bound = self.lower_bound.load(SeqCst); + let upper_bound = self.upper_bound.load(SeqCst); + if start_block >= lower_bound && end_block <= upper_bound { + let entries = self.entries.read().await; + headers = + entries.range(start_block..=end_block + 1).map(|(_, header)| header.clone()).collect(); + } else { + headers = self.provider.sealed_headers_range(start_block..=end_block)?; + } + + Ok(headers) +} diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 5ca07ecd5935..a803ba7e1103 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -475,25 +475,16 @@ impl Default for FeeHistoryCacheConfig { /// Wrapper struct for BTreeMap #[derive(Debug, Clone)] -pub struct FeeHistoryCache { +pub struct FeeHistoryCache { lower_bound: Arc, upper_bound: Arc, config: FeeHistoryCacheConfig, - provider: Provider, entries: Arc>>, } -impl FeeHistoryCache -where - Provider: BlockReaderIdExt - + ChainSpecProvider - + StateProviderFactory - + EvmEnvProvider - + BlockNumReader - + 'static, -{ +impl FeeHistoryCache { /// Creates new FeeHistoryCache instance, initialize it with the mose recent data, set bounds - pub fn new(config: FeeHistoryCacheConfig, provider: Provider) -> Self { + pub fn new(config: FeeHistoryCacheConfig) -> Self { let mut init_tree_map = BTreeMap::new(); let last_block_number = provider.last_block_number().unwrap_or(0); @@ -521,7 +512,7 @@ where let upper_bound = Arc::new(AtomicU64::new(upper_bound)); let lower_bound = Arc::new(AtomicU64::new(lower_bound)); - FeeHistoryCache { config, entries, provider, upper_bound, lower_bound } + FeeHistoryCache { config, entries, upper_bound, lower_bound } } /// Processing of the arriving blocks @@ -541,45 +532,15 @@ where self.upper_bound.store(upper_bound, SeqCst); self.lower_bound.store(lower_bound, SeqCst); } - - /// Collect fee history for given range. It will try to use a cache to take the most recent - /// headers or if the range is out of caching config it will fallback to the database provider - pub async fn get_history( - &self, - start_block: u64, - end_block: u64, - ) -> RethResult> { - let headers: Vec; - - let lower_bound = self.lower_bound.load(SeqCst); - let upper_bound = self.upper_bound.load(SeqCst); - if start_block >= lower_bound && end_block <= upper_bound { - let entries = self.entries.read().await; - headers = entries - .range(start_block..=end_block + 1) - .map(|(_, header)| header.clone()) - .collect(); - } else { - headers = self.provider.sealed_headers_range(start_block..=end_block)?; - } - - Ok(headers) - } } /// Awaits for new chain events and directly inserts them into the cache so they're available /// immediately before they need to be fetched from disk. pub async fn fee_history_cache_new_blocks_task( - fee_history_cache: FeeHistoryCache, + fee_history_cache: FeeHistoryCache, mut events: St, ) where St: Stream + Unpin + 'static, - Provider: BlockReaderIdExt - + ChainSpecProvider - + StateProviderFactory - + EvmEnvProvider - + BlockNumReader - + 'static, { while let Some(event) = events.next().await { if let Some(committed) = event.committed() { From e4ade22425b1906db83c3b943951772e4aecf956 Mon Sep 17 00:00:00 2001 From: allnil Date: Sat, 4 Nov 2023 16:16:20 +0000 Subject: [PATCH 16/32] refactor logic to use iterator over blocks in on_new_blocks function, change upper_bound and lower_bound to function --- crates/rpc/rpc-builder/src/auth.rs | 3 +- crates/rpc/rpc-builder/src/lib.rs | 9 +- crates/rpc/rpc/src/eth/api/fees.rs | 23 +---- crates/rpc/rpc/src/eth/api/mod.rs | 105 ++++++++++++++------- crates/rpc/rpc/src/eth/api/server.rs | 2 +- crates/rpc/rpc/src/eth/api/state.rs | 4 +- crates/rpc/rpc/src/eth/api/transactions.rs | 3 +- 7 files changed, 82 insertions(+), 67 deletions(-) diff --git a/crates/rpc/rpc-builder/src/auth.rs b/crates/rpc/rpc-builder/src/auth.rs index 136ea05406a8..bc526b32dca6 100644 --- a/crates/rpc/rpc-builder/src/auth.rs +++ b/crates/rpc/rpc-builder/src/auth.rs @@ -60,8 +60,7 @@ where let eth_cache = EthStateCache::spawn_with(provider.clone(), Default::default(), executor.clone()); let gas_oracle = GasPriceOracle::new(provider.clone(), Default::default(), eth_cache.clone()); - let fee_history_cache = - FeeHistoryCache::new(FeeHistoryCacheConfig::default(), provider.clone()); + let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default()); let eth_api = EthApi::with_spawner( provider.clone(), pool.clone(), diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 7328cb0495b8..b12fc0280406 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1022,10 +1022,7 @@ where let new_canonical_blocks = self.events.canonical_state_stream(); let c = cache.clone(); - let fee_history_cache = FeeHistoryCache::new( - self.config.eth.fee_history_cache.clone(), - self.provider.clone(), - ); + let fee_history_cache = FeeHistoryCache::new(self.config.eth.fee_history_cache.clone()); self.executor.spawn_critical( "cache canonical blocks task", Box::pin(async move { @@ -1035,10 +1032,12 @@ where let new_canonical_blocks = self.events.canonical_state_stream(); let fhc = fee_history_cache.clone(); + let provider_clone = self.provider.clone(); self.executor.spawn_critical( "cache canonical blocks for fee history task", Box::pin(async move { - fee_history_cache_new_blocks_task(fhc, new_canonical_blocks).await; + fee_history_cache_new_blocks_task(fhc, new_canonical_blocks, provider_clone) + .await; }), ); diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 0f6b7fd7d647..c2392c9b5c59 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -94,7 +94,10 @@ where let start_block = end_block_plus - block_count; - let headers = self.fee_history_cache().get_history(start_block, end_block).await?; + let mut headers = self.fee_history_cache().get_history(start_block, end_block).await?; + if headers.is_empty() { + headers = self.provider().sealed_headers_range(start_block..=end_block)?; + } if headers.len() != block_count as usize { return Err(EthApiError::InvalidBlockRange) @@ -202,21 +205,3 @@ where Ok(rewards_in_block) } } - -/// Collect fee history for given range. It will try to use a cache to take the most recent -/// headers or if the range is out of caching config it will fallback to the database provider -pub async fn get_history(start_block: u64, end_block: u64) -> RethResult> { - let headers: Vec; - - let lower_bound = self.lower_bound.load(SeqCst); - let upper_bound = self.upper_bound.load(SeqCst); - if start_block >= lower_bound && end_block <= upper_bound { - let entries = self.entries.read().await; - headers = - entries.range(start_block..=end_block + 1).map(|(_, header)| header.clone()).collect(); - } else { - headers = self.provider.sealed_headers_range(start_block..=end_block)?; - } - - Ok(headers) -} diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index a803ba7e1103..e42bfe0b1ccf 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -16,8 +16,8 @@ use reth_primitives::{ Address, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, SealedHeader, B256, U256, U64, }; use reth_provider::{ - BlockNumReader, BlockReaderIdExt, CanonStateNotification, ChainSpecProvider, EvmEnvProvider, - StateProviderBox, StateProviderFactory, + BlockReaderIdExt, CanonStateNotification, ChainSpecProvider, EvmEnvProvider, StateProviderBox, + StateProviderFactory, }; use reth_rpc_types::{SyncInfo, SyncStatus}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; @@ -97,7 +97,7 @@ where gas_oracle: GasPriceOracle, gas_cap: impl Into, blocking_task_pool: BlockingTaskPool, - fee_history_cache: FeeHistoryCache, + fee_history_cache: FeeHistoryCache, ) -> Self { Self::with_spawner( provider, @@ -123,7 +123,7 @@ where gas_cap: u64, task_spawner: Box, blocking_task_pool: BlockingTaskPool, - fee_history_cache: FeeHistoryCache, + fee_history_cache: FeeHistoryCache, ) -> Self { // get the block number of the latest block let latest_block = provider @@ -202,7 +202,7 @@ where } /// Returns fee history cache - pub fn fee_history_cache(&self) -> &FeeHistoryCache { + pub fn fee_history_cache(&self) -> &FeeHistoryCache { &self.inner.fee_history_cache } } @@ -454,7 +454,7 @@ struct EthApiInner { /// A pool dedicated to blocking tasks. blocking_task_pool: BlockingTaskPool, /// Cache for block fees history - fee_history_cache: FeeHistoryCache, + fee_history_cache: FeeHistoryCache, } /// Settings for the [EthStateCache](crate::eth::cache::EthStateCache). @@ -485,40 +485,25 @@ pub struct FeeHistoryCache { impl FeeHistoryCache { /// Creates new FeeHistoryCache instance, initialize it with the mose recent data, set bounds pub fn new(config: FeeHistoryCacheConfig) -> Self { - let mut init_tree_map = BTreeMap::new(); - - let last_block_number = provider.last_block_number().unwrap_or(0); - - let start_block = if last_block_number > config.max_blocks { - last_block_number - config.max_blocks - } else { - 0 - }; - - let headers = - provider.sealed_headers_range(start_block..=last_block_number).unwrap_or_default(); - - let (mut lower_bound, mut upper_bound) = (0, 0); - for header in headers { - init_tree_map.insert(header.number, header.clone()); - upper_bound = header.number; - if lower_bound == 0 { - lower_bound = header.number; - } - } + let init_tree_map = BTreeMap::new(); let entries = Arc::new(tokio::sync::RwLock::new(init_tree_map)); - let upper_bound = Arc::new(AtomicU64::new(upper_bound)); - let lower_bound = Arc::new(AtomicU64::new(lower_bound)); + let upper_bound = Arc::new(AtomicU64::new(0)); + let lower_bound = Arc::new(AtomicU64::new(0)); FeeHistoryCache { config, entries, upper_bound, lower_bound } } /// Processing of the arriving blocks - pub async fn on_new_block(&self, block: &SealedBlock) { + pub async fn on_new_block<'a, I>(&self, headers: I) + where + I: Iterator, + { let mut entries = self.entries.write().await; - entries.insert(block.number, block.header.clone()); + for header in headers { + entries.insert(header.number, header.clone()); + } while entries.len() > self.config.max_blocks as usize { entries.pop_first(); } @@ -527,11 +512,43 @@ impl FeeHistoryCache { self.lower_bound.store(0, SeqCst); return } - let upper_bound = *entries.last_entry().expect("Contains be at least one entry").key(); + let upper_bound = *entries.last_entry().expect("Contains at least one entry").key(); let lower_bound = *entries.first_entry().expect("Contains at least one entry").key(); self.upper_bound.store(upper_bound, SeqCst); self.lower_bound.store(lower_bound, SeqCst); } + + /// Get UpperBound value for FeeHistoryCache + pub fn upper_bound(&self) -> u64 { + self.upper_bound.load(SeqCst) + } + + /// Get LowerBound value for FeeHistoryCache + pub fn lower_bound(&self) -> u64 { + self.lower_bound.load(SeqCst) + } + + /// Collect fee history for given range. It will try to use a cache to take the most recent + /// headers or if the range is out of caching config it will fallback to the database provider + pub async fn get_history( + &self, + start_block: u64, + end_block: u64, + ) -> RethResult> { + let mut headers = Vec::new(); + + let lower_bound = self.lower_bound(); + let upper_bound = self.upper_bound(); + if start_block >= lower_bound && end_block <= upper_bound { + let entries = self.entries.read().await; + headers = entries + .range(start_block..=end_block + 1) + .map(|(_, header)| header.clone()) + .collect(); + } + + Ok(headers) + } } /// Awaits for new chain events and directly inserts them into the cache so they're available @@ -539,19 +556,35 @@ impl FeeHistoryCache { pub async fn fee_history_cache_new_blocks_task( fee_history_cache: FeeHistoryCache, mut events: St, + provider: Provider, ) where St: Stream + Unpin + 'static, + Provider: BlockReaderIdExt + ChainSpecProvider + 'static, { + // Init default state + if fee_history_cache.upper_bound() == 0 { + let last_block_number = provider.last_block_number().unwrap_or(0); + + let start_block = if last_block_number > fee_history_cache.config.max_blocks { + last_block_number - fee_history_cache.config.max_blocks + } else { + 0 + }; + + let headers = + provider.sealed_headers_range(start_block..=last_block_number).unwrap_or_default(); + + fee_history_cache.on_new_block(headers.iter()).await; + } + while let Some(event) = events.next().await { if let Some(committed) = event.committed() { // we're only interested in new committed blocks let (blocks, _) = committed.inner(); - let blocks = blocks.iter().map(|(_, block)| block.block.clone()).collect::>(); + let headers = blocks.iter().map(|(_, block)| block.header.clone()).collect::>(); - for block in blocks { - fee_history_cache.on_new_block(&block).await; - } + fee_history_cache.on_new_block(headers.iter()).await; } } } diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index cb45598b3624..89c4ecd36727 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -428,7 +428,7 @@ mod tests { GasPriceOracle::new(provider.clone(), Default::default(), cache), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), - FeeHistoryCache::new(FeeHistoryCacheConfig::default(), provider.clone()), + FeeHistoryCache::new(FeeHistoryCacheConfig::default()), ) } diff --git a/crates/rpc/rpc/src/eth/api/state.rs b/crates/rpc/rpc/src/eth/api/state.rs index dde1e574bc15..c4a993c2bd06 100644 --- a/crates/rpc/rpc/src/eth/api/state.rs +++ b/crates/rpc/rpc/src/eth/api/state.rs @@ -150,7 +150,7 @@ mod tests { GasPriceOracle::new(NoopProvider::default(), Default::default(), cache), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), - FeeHistoryCache::new(FeeHistoryCacheConfig::default(), NoopProvider::default()), + FeeHistoryCache::new(FeeHistoryCacheConfig::default()), ); let address = Address::random(); let storage = eth_api.storage_at(address, U256::ZERO.into(), None).unwrap(); @@ -173,7 +173,7 @@ mod tests { GasPriceOracle::new(mock_provider.clone(), Default::default(), cache), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), - FeeHistoryCache::new(FeeHistoryCacheConfig::default(), mock_provider.clone()), + FeeHistoryCache::new(FeeHistoryCacheConfig::default()), ); let storage_key: U256 = storage_key.into(); diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index 3f241ccd0bfe..975c8f8db667 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -1109,8 +1109,7 @@ mod tests { let pool = testing_pool(); let cache = EthStateCache::spawn(noop_provider, Default::default()); - let fee_history_cache = - FeeHistoryCache::new(FeeHistoryCacheConfig::default(), noop_provider); + let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default()); let eth_api = EthApi::new( noop_provider, pool.clone(), From 912fd0f2c45521a3eec87eae17bb1af0f8f40093 Mon Sep 17 00:00:00 2001 From: allnil Date: Sat, 4 Nov 2023 17:20:38 +0000 Subject: [PATCH 17/32] prepare type to store fee data --- crates/rpc/rpc-builder/src/auth.rs | 3 +- crates/rpc/rpc/src/eth/api/fees.rs | 46 +++++++++++++++++------------- crates/rpc/rpc/src/eth/api/mod.rs | 35 ++++++++++++++++++----- 3 files changed, 56 insertions(+), 28 deletions(-) diff --git a/crates/rpc/rpc-builder/src/auth.rs b/crates/rpc/rpc-builder/src/auth.rs index afb0916a9293..9a7c9edc9615 100644 --- a/crates/rpc/rpc-builder/src/auth.rs +++ b/crates/rpc/rpc-builder/src/auth.rs @@ -17,7 +17,8 @@ use reth_provider::{ }; use reth_rpc::{ eth::{ - cache::EthStateCache, gas_oracle::GasPriceOracle, FeeHistoryCache, FeeHistoryCacheConfig, + cache::EthStateCache, gas_oracle::GasPriceOracle, EthFilterConfig, FeeHistoryCache, + FeeHistoryCacheConfig, }, AuthLayer, BlockingTaskPool, Claims, EngineEthApi, EthApi, EthFilter, EthSubscriptionIdProvider, JwtAuthValidator, JwtSecret, diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 7ab5c873a478..3837df86e799 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -6,14 +6,14 @@ use crate::{ }; use reth_network_api::NetworkInfo; -use reth_primitives::{ - basefee::calculate_next_block_base_fee, BlockNumberOrTag, SealedHeader, U256, -}; +use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, U256}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::{FeeHistory, TxGasAndReward}; use reth_transaction_pool::TransactionPool; use tracing::debug; +use super::FeeHistoryEntry; + impl EthApi where Pool: TransactionPool + Clone + 'static, @@ -94,12 +94,17 @@ where let start_block = end_block_plus - block_count; - let mut headers = self.fee_history_cache().get_history(start_block, end_block).await?; - if headers.is_empty() { - headers = self.provider().sealed_headers_range(start_block..=end_block)?; + let mut fee_entries = self.fee_history_cache().get_history(start_block, end_block).await?; + if fee_entries.is_empty() { + fee_entries = self + .provider() + .sealed_headers_range(start_block..=end_block)? + .into_iter() + .map(|header| FeeHistoryEntry::from(&header)) + .collect(); } - if headers.len() != block_count as usize { + if fee_entries.len() != block_count as usize { return Err(EthApiError::InvalidBlockRange) } // Collect base fees, gas usage ratios and (optionally) reward percentile data @@ -107,12 +112,11 @@ where let mut gas_used_ratio: Vec = Vec::new(); let mut rewards: Vec> = Vec::new(); - for header in &headers { - base_fee_per_gas - .push(U256::try_from(header.base_fee_per_gas.unwrap_or_default()).unwrap()); - gas_used_ratio.push(header.gas_used as f64 / header.gas_limit as f64); + for entry in &fee_entries { + base_fee_per_gas.push(U256::try_from(entry.base_fee_per_gas).unwrap()); + gas_used_ratio.push(entry.gas_used_ratio); if let Some(percentiles) = &reward_percentiles { - rewards.push(self.calculate_reward_percentiles(percentiles, header).await?); + rewards.push(self.calculate_reward_percentiles(percentiles, entry).await?); } } @@ -121,14 +125,14 @@ where // // The unwrap is safe since we checked earlier that we got at least 1 header. - let last_header = headers.last().unwrap(); + let last_entry = fee_entries.last().unwrap(); let chain_spec = self.provider().chain_spec(); base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( - last_header.gas_used, - last_header.gas_limit, - last_header.base_fee_per_gas.unwrap_or_default(), + last_entry.gas_used, + last_entry.gas_limit, + last_entry.base_fee_per_gas, chain_spec.base_fee_params, ))); @@ -148,11 +152,11 @@ where async fn calculate_reward_percentiles( &self, percentiles: &[f64], - header: &SealedHeader, + fee_entry: &FeeHistoryEntry, ) -> Result, EthApiError> { let (transactions, receipts) = self .cache() - .get_transactions_and_receipts(header.hash) + .get_transactions_and_receipts(fee_entry.header_hash) .await? .ok_or(EthApiError::InvalidBlockRange)?; @@ -171,7 +175,9 @@ where Some(TxGasAndReward { gas_used, - reward: tx.effective_tip_per_gas(header.base_fee_per_gas).unwrap_or_default(), + reward: tx + .effective_tip_per_gas(Some(fee_entry.base_fee_per_gas)) + .unwrap_or_default(), }) }) .collect::>(); @@ -194,7 +200,7 @@ where continue } - let threshold = (header.gas_used as f64 * percentile / 100.) as u64; + let threshold = (fee_entry.gas_used as f64 * percentile / 100.) as u64; while cumulative_gas_used < threshold && tx_index < transactions.len() - 1 { tx_index += 1; cumulative_gas_used += transactions[tx_index].gas_used; diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index e42bfe0b1ccf..b7056e6de019 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -479,7 +479,7 @@ pub struct FeeHistoryCache { lower_bound: Arc, upper_bound: Arc, config: FeeHistoryCacheConfig, - entries: Arc>>, + entries: Arc>>, } impl FeeHistoryCache { @@ -502,7 +502,7 @@ impl FeeHistoryCache { { let mut entries = self.entries.write().await; for header in headers { - entries.insert(header.number, header.clone()); + entries.insert(header.number, FeeHistoryEntry::from(header)); } while entries.len() > self.config.max_blocks as usize { entries.pop_first(); @@ -534,20 +534,20 @@ impl FeeHistoryCache { &self, start_block: u64, end_block: u64, - ) -> RethResult> { - let mut headers = Vec::new(); + ) -> RethResult> { + let mut result = Vec::new(); let lower_bound = self.lower_bound(); let upper_bound = self.upper_bound(); if start_block >= lower_bound && end_block <= upper_bound { let entries = self.entries.read().await; - headers = entries + result = entries .range(start_block..=end_block + 1) - .map(|(_, header)| header.clone()) + .map(|(_, fee_entry)| fee_entry.clone()) .collect(); } - Ok(headers) + Ok(result) } } @@ -588,3 +588,24 @@ pub async fn fee_history_cache_new_blocks_task( } } } + +#[derive(Debug, Clone)] +pub struct FeeHistoryEntry { + base_fee_per_gas: u64, + gas_used_ratio: f64, + gas_used: u64, + gas_limit: u64, + header_hash: B256, +} + +impl From<&SealedHeader> for FeeHistoryEntry { + fn from(header: &SealedHeader) -> Self { + FeeHistoryEntry { + base_fee_per_gas: header.base_fee_per_gas.unwrap_or_default(), + gas_used_ratio: header.gas_used as f64 / header.gas_limit as f64, + gas_used: header.gas_used, + header_hash: header.hash, + gas_limit: header.gas_limit, + } + } +} From 2805a7a391c0f869de281eea896e2d51042a28a7 Mon Sep 17 00:00:00 2001 From: allnil Date: Sat, 4 Nov 2023 17:47:29 +0000 Subject: [PATCH 18/32] add annotation for clippy to ignore too many arguments on new fn --- crates/rpc/rpc/src/eth/api/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index b7056e6de019..b52c6c42e764 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -89,6 +89,7 @@ where Provider: BlockReaderIdExt + ChainSpecProvider, { /// Creates a new, shareable instance using the default tokio task spawner. + #[allow(clippy::too_many_arguments)] pub fn new( provider: Provider, pool: Pool, From 398e885893c5f1f91b71b2e3b76ecbf4b2bd5064 Mon Sep 17 00:00:00 2001 From: allnil Date: Wed, 15 Nov 2023 21:23:38 +0000 Subject: [PATCH 19/32] refactor, precalculate percentiles, rewards --- crates/net/eth-wire/src/capability.rs | 2 +- crates/rpc/rpc-builder/src/auth.rs | 3 +- crates/rpc/rpc-builder/src/lib.rs | 4 +- .../rpc-types/src/serde_helpers/json_u256.rs | 4 +- crates/rpc/rpc/src/eth/api/fees.rs | 91 +++--------- crates/rpc/rpc/src/eth/api/mod.rs | 132 +++++++++++++++--- crates/rpc/rpc/src/eth/api/server.rs | 2 +- crates/rpc/rpc/src/eth/api/state.rs | 4 +- crates/rpc/rpc/src/eth/api/transactions.rs | 3 +- .../src/providers/database/provider.rs | 4 +- 10 files changed, 148 insertions(+), 101 deletions(-) diff --git a/crates/net/eth-wire/src/capability.rs b/crates/net/eth-wire/src/capability.rs index f33993a52a42..53ffa67a1347 100644 --- a/crates/net/eth-wire/src/capability.rs +++ b/crates/net/eth-wire/src/capability.rs @@ -254,7 +254,7 @@ impl SharedCapability { /// Returns an error if the offset is equal or less than [`MAX_RESERVED_MESSAGE_ID`]. pub(crate) fn new(name: &str, version: u8, offset: u8) -> Result { if offset <= MAX_RESERVED_MESSAGE_ID { - return Err(SharedCapabilityError::ReservedMessageIdOffset(offset)); + return Err(SharedCapabilityError::ReservedMessageIdOffset(offset)) } match name { diff --git a/crates/rpc/rpc-builder/src/auth.rs b/crates/rpc/rpc-builder/src/auth.rs index 9a7c9edc9615..7d61b1ea645c 100644 --- a/crates/rpc/rpc-builder/src/auth.rs +++ b/crates/rpc/rpc-builder/src/auth.rs @@ -61,7 +61,8 @@ where let eth_cache = EthStateCache::spawn_with(provider.clone(), Default::default(), executor.clone()); let gas_oracle = GasPriceOracle::new(provider.clone(), Default::default(), eth_cache.clone()); - let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default()); + let fee_history_cache = + FeeHistoryCache::new(eth_cache.clone(), FeeHistoryCacheConfig::default()); let eth_api = EthApi::with_spawner( provider.clone(), pool.clone(), diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index ff0da2a5c6bb..40660ffd3760 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1073,7 +1073,6 @@ where let new_canonical_blocks = self.events.canonical_state_stream(); let c = cache.clone(); - let fee_history_cache = FeeHistoryCache::new(self.config.eth.fee_history_cache.clone()); self.executor.spawn_critical( "cache canonical blocks task", Box::pin(async move { @@ -1081,6 +1080,9 @@ where }), ); + let c = cache.clone(); + let fee_history_cache = + FeeHistoryCache::new(c, self.config.eth.fee_history_cache.clone()); let new_canonical_blocks = self.events.canonical_state_stream(); let fhc = fee_history_cache.clone(); let provider_clone = self.provider.clone(); diff --git a/crates/rpc/rpc-types/src/serde_helpers/json_u256.rs b/crates/rpc/rpc-types/src/serde_helpers/json_u256.rs index 3ed3859a2c84..a22280e8a884 100644 --- a/crates/rpc/rpc-types/src/serde_helpers/json_u256.rs +++ b/crates/rpc/rpc-types/src/serde_helpers/json_u256.rs @@ -162,11 +162,11 @@ where } else { // We could try to convert to a u128 here but there would probably be loss of // precision, so we just return an error. - return Err(Error::custom("Deserializing a large non-mainnet TTD is not supported")); + return Err(Error::custom("Deserializing a large non-mainnet TTD is not supported")) } } else { // must be i64 - negative numbers are not supported - return Err(Error::custom("Negative TTD values are invalid and will not be deserialized")); + return Err(Error::custom("Negative TTD values are invalid and will not be deserialized")) }; Ok(num) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 0cbd0ec84a75..00b9c9aff915 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -8,7 +8,7 @@ use crate::{ use reth_network_api::NetworkInfo; use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, U256}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; -use reth_rpc_types::{FeeHistory, TxGasAndReward}; +use reth_rpc_types::FeeHistory; use reth_transaction_pool::TransactionPool; use tracing::debug; @@ -108,10 +108,11 @@ where if fee_entries.is_empty() { fee_entries = self .provider() - .sealed_headers_range(start_block..=end_block)? - .into_iter() - .map(|header| FeeHistoryEntry::from(&header)) + .block_range(start_block..=end_block)? + .iter() + .map(|block| FeeHistoryEntry::new(&block, block.clone().seal_slow().hash)) .collect(); + // calculate rewards on the fly } if fee_entries.len() != block_count as usize { @@ -125,8 +126,14 @@ where for entry in &fee_entries { base_fee_per_gas.push(U256::try_from(entry.base_fee_per_gas).unwrap()); gas_used_ratio.push(entry.gas_used_ratio); + if let Some(percentiles) = &reward_percentiles { - rewards.push(self.calculate_reward_percentiles(percentiles, entry).await?); + let mut block_rewards = Vec::new(); + + for &percentile in percentiles.iter() { + block_rewards.push(self.fetch_reward_for_percentile(entry, percentile)); + } + rewards.push(block_rewards); } } @@ -150,74 +157,22 @@ where base_fee_per_gas, gas_used_ratio, oldest_block: U256::from(start_block), - reward: reward_percentiles.map(|_| rewards), + reward: Some(rewards), }) } - /// Calculates reward percentiles for transactions in a block header. - /// Given a list of percentiles and a sealed block header, this function computes - /// the corresponding rewards for the transactions at each percentile. - /// - /// The results are returned as a vector of U256 values. - async fn calculate_reward_percentiles( + fn fetch_reward_for_percentile( &self, - percentiles: &[f64], - fee_entry: &FeeHistoryEntry, - ) -> Result, EthApiError> { - let (transactions, receipts) = self - .cache() - .get_transactions_and_receipts(fee_entry.header_hash) - .await? - .ok_or(EthApiError::InvalidBlockRange)?; - - let mut transactions = transactions - .into_iter() - .zip(receipts) - .scan(0, |previous_gas, (tx, receipt)| { - // Convert the cumulative gas used in the receipts - // to the gas usage by the transaction - // - // While we will sum up the gas again later, it is worth - // noting that the order of the transactions will be different, - // so the sum will also be different for each receipt. - let gas_used = receipt.cumulative_gas_used - *previous_gas; - *previous_gas = receipt.cumulative_gas_used; - - Some(TxGasAndReward { - gas_used, - reward: tx - .effective_tip_per_gas(Some(fee_entry.base_fee_per_gas)) - .unwrap_or_default(), - }) - }) - .collect::>(); - - // Sort the transactions by their rewards in ascending order - transactions.sort_by_key(|tx| tx.reward); - - // Find the transaction that corresponds to the given percentile - // - // We use a `tx_index` here that is shared across all percentiles, since we know - // the percentiles are monotonically increasing. - let mut tx_index = 0; - let mut cumulative_gas_used = - transactions.first().map(|tx| tx.gas_used).unwrap_or_default(); - let mut rewards_in_block = Vec::new(); - for percentile in percentiles { - // Empty blocks should return in a zero row - if transactions.is_empty() { - rewards_in_block.push(U256::ZERO); - continue - } + entry: &FeeHistoryEntry, + requested_percentile: f64, + ) -> U256 { + let rounded_percentile = (requested_percentile / 0.25).round() * 0.25; + let rounded_percentile = rounded_percentile.clamp(0.0, 100.0); - let threshold = (fee_entry.gas_used as f64 * percentile / 100.) as u64; - while cumulative_gas_used < threshold && tx_index < transactions.len() - 1 { - tx_index += 1; - cumulative_gas_used += transactions[tx_index].gas_used; - } - rewards_in_block.push(U256::from(transactions[tx_index].reward)); - } + // Calculate the index in the precomputed rewards array + let index = (rounded_percentile / 0.25) as usize; - Ok(rewards_in_block) + // Fetch the reward from the FeeHistoryEntry + entry.rewards.get(index).cloned().unwrap_or(U256::ZERO) } } diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 8cb5147bed10..746f6b9cb160 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -13,13 +13,13 @@ use metrics::atomics::AtomicU64; use reth_interfaces::RethResult; use reth_network_api::NetworkInfo; use reth_primitives::{ - Address, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, SealedHeader, B256, U256, U64, + Address, Block, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, B256, U256, U64, }; use reth_provider::{ BlockReaderIdExt, CanonStateNotification, ChainSpecProvider, EvmEnvProvider, StateProviderBox, StateProviderFactory, }; -use reth_rpc_types::{SyncInfo, SyncStatus}; +use reth_rpc_types::{SyncInfo, SyncStatus, TxGasAndReward}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use reth_transaction_pool::TransactionPool; use revm_primitives::{BlockEnv, CfgEnv}; @@ -471,7 +471,7 @@ struct EthApiInner { http_client: reqwest::Client, } -/// Settings for the [EthStateCache](crate::eth::cache::EthStateCache). +/// Settings for the [FeeHistoryCache](crate::eth::FeeHistoryCache). #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct FeeHistoryCacheConfig { @@ -494,11 +494,12 @@ pub struct FeeHistoryCache { upper_bound: Arc, config: FeeHistoryCacheConfig, entries: Arc>>, + eth_cache: EthStateCache, } impl FeeHistoryCache { /// Creates new FeeHistoryCache instance, initialize it with the mose recent data, set bounds - pub fn new(config: FeeHistoryCacheConfig) -> Self { + pub fn new(eth_cache: EthStateCache, config: FeeHistoryCacheConfig) -> Self { let init_tree_map = BTreeMap::new(); let entries = Arc::new(tokio::sync::RwLock::new(init_tree_map)); @@ -506,18 +507,33 @@ impl FeeHistoryCache { let upper_bound = Arc::new(AtomicU64::new(0)); let lower_bound = Arc::new(AtomicU64::new(0)); - FeeHistoryCache { config, entries, upper_bound, lower_bound } + FeeHistoryCache { config, entries, upper_bound, lower_bound, eth_cache } } /// Processing of the arriving blocks - pub async fn on_new_block<'a, I>(&self, headers: I) + pub async fn on_new_block<'a, I>(&self, blocks: I) where - I: Iterator, + I: Iterator, { let mut entries = self.entries.write().await; - for header in headers { - entries.insert(header.number, FeeHistoryEntry::from(header)); + + for block in blocks { + let mut fee_history_entry = FeeHistoryEntry::new(block, block.clone().seal_slow().hash); + + let percentiles = predefined_percentiles(); + + fee_history_entry.rewards = calculate_reward_percentiles_for_block( + &percentiles, + &fee_history_entry, + self.eth_cache.clone(), + ) + .await + .unwrap_or_default(); + + // calculate rewards + entries.insert(block.number, fee_history_entry); } + while entries.len() > self.config.max_blocks as usize { entries.pop_first(); } @@ -585,10 +601,9 @@ pub async fn fee_history_cache_new_blocks_task( 0 }; - let headers = - provider.sealed_headers_range(start_block..=last_block_number).unwrap_or_default(); + let blocks = provider.block_range(start_block..=last_block_number).unwrap_or_default(); - fee_history_cache.on_new_block(headers.iter()).await; + fee_history_cache.on_new_block(blocks.iter()).await; } while let Some(event) = events.next().await { @@ -596,13 +611,86 @@ pub async fn fee_history_cache_new_blocks_task( // we're only interested in new committed blocks let (blocks, _) = committed.inner(); - let headers = blocks.iter().map(|(_, block)| block.header.clone()).collect::>(); + let blocks = blocks + .iter() + .map(|(_, block)| block.block.clone().unseal().clone()) + .collect::>(); - fee_history_cache.on_new_block(headers.iter()).await; + fee_history_cache.on_new_block(blocks.iter()).await; } } } +/// Generates predefined set of percentiles +fn predefined_percentiles() -> Vec { + (0..=400).map(|p| p as f64 * 0.25).collect() +} + +/// Calculates reward percentiles for transactions in a block header. +/// Given a list of percentiles and a sealed block header, this function computes +/// the corresponding rewards for the transactions at each percentile. +/// +/// The results are returned as a vector of U256 values. +async fn calculate_reward_percentiles_for_block( + percentiles: &[f64], + fee_entry: &FeeHistoryEntry, + cache: EthStateCache, +) -> Result, EthApiError> { + let (transactions, receipts) = cache + .get_transactions_and_receipts(fee_entry.header_hash) + .await? + .ok_or(EthApiError::InvalidBlockRange)?; + + let mut transactions = transactions + .into_iter() + .zip(receipts) + .scan(0, |previous_gas, (tx, receipt)| { + // Convert the cumulative gas used in the receipts + // to the gas usage by the transaction + // + // While we will sum up the gas again later, it is worth + // noting that the order of the transactions will be different, + // so the sum will also be different for each receipt. + let gas_used = receipt.cumulative_gas_used - *previous_gas; + *previous_gas = receipt.cumulative_gas_used; + + Some(TxGasAndReward { + gas_used, + reward: tx + .effective_tip_per_gas(Some(fee_entry.base_fee_per_gas)) + .unwrap_or_default(), + }) + }) + .collect::>(); + + // Sort the transactions by their rewards in ascending order + transactions.sort_by_key(|tx| tx.reward); + + // Find the transaction that corresponds to the given percentile + // + // We use a `tx_index` here that is shared across all percentiles, since we know + // the percentiles are monotonically increasing. + let mut tx_index = 0; + let mut cumulative_gas_used = transactions.first().map(|tx| tx.gas_used).unwrap_or_default(); + let mut rewards_in_block = Vec::new(); + for percentile in percentiles { + // Empty blocks should return in a zero row + if transactions.is_empty() { + rewards_in_block.push(U256::ZERO); + continue + } + + let threshold = (fee_entry.gas_used as f64 * percentile / 100.) as u64; + while cumulative_gas_used < threshold && tx_index < transactions.len() - 1 { + tx_index += 1; + cumulative_gas_used += transactions[tx_index].gas_used; + } + rewards_in_block.push(U256::from(transactions[tx_index].reward)); + } + + Ok(rewards_in_block) +} + #[derive(Debug, Clone)] pub struct FeeHistoryEntry { base_fee_per_gas: u64, @@ -610,16 +698,18 @@ pub struct FeeHistoryEntry { gas_used: u64, gas_limit: u64, header_hash: B256, + rewards: Vec, } -impl From<&SealedHeader> for FeeHistoryEntry { - fn from(header: &SealedHeader) -> Self { +impl FeeHistoryEntry { + fn new(block: &Block, hash: B256) -> Self { FeeHistoryEntry { - base_fee_per_gas: header.base_fee_per_gas.unwrap_or_default(), - gas_used_ratio: header.gas_used as f64 / header.gas_limit as f64, - gas_used: header.gas_used, - header_hash: header.hash, - gas_limit: header.gas_limit, + base_fee_per_gas: block.base_fee_per_gas.unwrap_or_default(), + gas_used_ratio: block.gas_used as f64 / block.gas_limit as f64, + gas_used: block.gas_used, + header_hash: hash, + gas_limit: block.gas_limit, + rewards: Vec::new(), } } } diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index cbaaeedd8935..c39f86273d66 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -434,7 +434,7 @@ mod tests { GasPriceOracle::new(provider.clone(), Default::default(), cache), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), - FeeHistoryCache::new(FeeHistoryCacheConfig::default()), + FeeHistoryCache::new(cache.clone(), FeeHistoryCacheConfig::default()), ) } diff --git a/crates/rpc/rpc/src/eth/api/state.rs b/crates/rpc/rpc/src/eth/api/state.rs index c4a993c2bd06..df30675cf006 100644 --- a/crates/rpc/rpc/src/eth/api/state.rs +++ b/crates/rpc/rpc/src/eth/api/state.rs @@ -150,7 +150,7 @@ mod tests { GasPriceOracle::new(NoopProvider::default(), Default::default(), cache), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), - FeeHistoryCache::new(FeeHistoryCacheConfig::default()), + FeeHistoryCache::new(cache.clone(), FeeHistoryCacheConfig::default()), ); let address = Address::random(); let storage = eth_api.storage_at(address, U256::ZERO.into(), None).unwrap(); @@ -173,7 +173,7 @@ mod tests { GasPriceOracle::new(mock_provider.clone(), Default::default(), cache), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), - FeeHistoryCache::new(FeeHistoryCacheConfig::default()), + FeeHistoryCache::new(cache.clone(), FeeHistoryCacheConfig::default()), ); let storage_key: U256 = storage_key.into(); diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index 23d4b483a575..f08ce26862ea 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -1286,7 +1286,8 @@ mod tests { let pool = testing_pool(); let cache = EthStateCache::spawn(noop_provider, Default::default()); - let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default()); + let fee_history_cache = + FeeHistoryCache::new(cache.clone(), FeeHistoryCacheConfig::default()); let eth_api = EthApi::new( noop_provider, pool.clone(), diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index c082e8d4ec48..2aba51b990ea 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1046,9 +1046,7 @@ impl BlockReader for DatabaseProvider { id: BlockHashOrNumber, transaction_kind: TransactionVariant, ) -> RethResult> { - let Some(block_number) = self.convert_hash_or_number(id)? else { - return Ok(None); - }; + let Some(block_number) = self.convert_hash_or_number(id)? else { return Ok(None) }; let Some(header) = self.header_by_number(block_number)? else { return Ok(None) }; From e213d135505506f857efab9e8f47efad28d7a7fc Mon Sep 17 00:00:00 2001 From: allnil Date: Thu, 16 Nov 2023 17:19:36 +0000 Subject: [PATCH 20/32] add mock for block_range, update tests --- crates/rpc/rpc-builder/src/auth.rs | 2 ++ crates/rpc/rpc/src/eth/api/fees.rs | 14 ++++++++------ crates/rpc/rpc/src/eth/api/server.rs | 8 ++++++-- crates/rpc/rpc/src/eth/api/state.rs | 4 ++-- crates/rpc/rpc/src/eth/api/transactions.rs | 2 +- crates/storage/provider/src/test_utils/mock.rs | 10 ++++++++-- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/crates/rpc/rpc-builder/src/auth.rs b/crates/rpc/rpc-builder/src/auth.rs index 7d61b1ea645c..7f1158e1cf5c 100644 --- a/crates/rpc/rpc-builder/src/auth.rs +++ b/crates/rpc/rpc-builder/src/auth.rs @@ -60,7 +60,9 @@ where // spawn a new cache task let eth_cache = EthStateCache::spawn_with(provider.clone(), Default::default(), executor.clone()); + let gas_oracle = GasPriceOracle::new(provider.clone(), Default::default(), eth_cache.clone()); + let fee_history_cache = FeeHistoryCache::new(eth_cache.clone(), FeeHistoryCacheConfig::default()); let eth_api = EthApi::with_spawner( diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 00b9c9aff915..68e33c2987fd 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -56,7 +56,7 @@ where reward_percentiles: Option>, ) -> EthResult { if block_count == 0 { - return Ok(FeeHistory::default()) + return Ok(FeeHistory::default()); } // See https://github.com/ethereum/go-ethereum/blob/2754b197c935ee63101cbbca2752338246384fec/eth/gasprice/feehistory.go#L218C8-L225 @@ -76,7 +76,7 @@ where } let Some(end_block) = self.provider().block_number_for_id(newest_block.into())? else { - return Err(EthApiError::UnknownBlockNumber) + return Err(EthApiError::UnknownBlockNumber); }; // need to add 1 to the end block to get the correct (inclusive) range @@ -92,7 +92,7 @@ where // Note: The types used ensure that the percentiles are never < 0 if let Some(percentiles) = &reward_percentiles { if percentiles.windows(2).any(|w| w[0] > w[1] || w[0] > 100.) { - return Err(EthApiError::InvalidRewardPercentiles) + return Err(EthApiError::InvalidRewardPercentiles); } } @@ -110,13 +110,15 @@ where .provider() .block_range(start_block..=end_block)? .iter() - .map(|block| FeeHistoryEntry::new(&block, block.clone().seal_slow().hash)) + .map(|block| FeeHistoryEntry::new(block, block.clone().seal_slow().hash)) .collect(); // calculate rewards on the fly + println!("took fee entries from the db on the fly") } + println!("fee entries len: {}", fee_entries.len()); if fee_entries.len() != block_count as usize { - return Err(EthApiError::InvalidBlockRange) + return Err(EthApiError::InvalidBlockRange); } // Collect base fees, gas usage ratios and (optionally) reward percentile data let mut base_fee_per_gas: Vec = Vec::new(); @@ -157,7 +159,7 @@ where base_fee_per_gas, gas_used_ratio, oldest_block: U256::from(start_block), - reward: Some(rewards), + reward: if rewards.is_empty() { None } else { Some(rewards) }, }) } diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index b249c15149e5..c63de87e710b 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -425,15 +425,19 @@ mod tests { provider: P, ) -> EthApi { let cache = EthStateCache::spawn(provider.clone(), Default::default()); + + let fee_history_cache = + FeeHistoryCache::new(cache.clone(), FeeHistoryCacheConfig::default()); + EthApi::new( provider.clone(), testing_pool(), NoopNetwork::default(), cache.clone(), - GasPriceOracle::new(provider.clone(), Default::default(), cache), + GasPriceOracle::new(provider.clone(), Default::default(), cache.clone()), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), - FeeHistoryCache::new(cache.clone(), FeeHistoryCacheConfig::default()), + fee_history_cache, ) } diff --git a/crates/rpc/rpc/src/eth/api/state.rs b/crates/rpc/rpc/src/eth/api/state.rs index df30675cf006..a63e08126d52 100644 --- a/crates/rpc/rpc/src/eth/api/state.rs +++ b/crates/rpc/rpc/src/eth/api/state.rs @@ -147,7 +147,7 @@ mod tests { pool.clone(), (), cache.clone(), - GasPriceOracle::new(NoopProvider::default(), Default::default(), cache), + GasPriceOracle::new(NoopProvider::default(), Default::default(), cache.clone()), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), FeeHistoryCache::new(cache.clone(), FeeHistoryCacheConfig::default()), @@ -170,7 +170,7 @@ mod tests { pool, (), cache.clone(), - GasPriceOracle::new(mock_provider.clone(), Default::default(), cache), + GasPriceOracle::new(mock_provider.clone(), Default::default(), cache.clone()), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), FeeHistoryCache::new(cache.clone(), FeeHistoryCacheConfig::default()), diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index 5f3ea170372c..429228e31bfb 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -1293,7 +1293,7 @@ mod tests { pool.clone(), noop_network_provider, cache.clone(), - GasPriceOracle::new(noop_provider, Default::default(), cache), + GasPriceOracle::new(noop_provider, Default::default(), cache.clone()), ETHEREUM_BLOCK_GAS_LIMIT, BlockingTaskPool::build().expect("failed to build tracing pool"), fee_history_cache, diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 5c794c77a82a..fa9acb2547e8 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -445,8 +445,14 @@ impl BlockReader for MockEthProvider { Ok(None) } - fn block_range(&self, _range: RangeInclusive) -> RethResult> { - Ok(vec![]) + fn block_range(&self, range: RangeInclusive) -> RethResult> { + let lock = self.blocks.lock(); + + let mut blocks: Vec<_> = + lock.values().filter(|block| range.contains(&block.number)).cloned().collect(); + blocks.sort_by_key(|block| block.number); + + Ok(blocks) } } From 08a25fcb07764bf5ce123a7ebba19cd0e24632e9 Mon Sep 17 00:00:00 2001 From: allnil Date: Thu, 16 Nov 2023 18:15:15 +0000 Subject: [PATCH 21/32] add simple test on rewards, calculate rewards on the fly --- crates/rpc/rpc/src/eth/api/fees.rs | 31 +++++++++++---------- crates/rpc/rpc/src/eth/api/mod.rs | 1 - crates/rpc/rpc/src/eth/api/server.rs | 41 ++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 68e33c2987fd..0e5a0c5359b5 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -13,7 +13,7 @@ use reth_transaction_pool::TransactionPool; use tracing::debug; -use super::FeeHistoryEntry; +use super::{calculate_reward_percentiles_for_block, predefined_percentiles, FeeHistoryEntry}; impl EthApi where @@ -56,7 +56,7 @@ where reward_percentiles: Option>, ) -> EthResult { if block_count == 0 { - return Ok(FeeHistory::default()); + return Ok(FeeHistory::default()) } // See https://github.com/ethereum/go-ethereum/blob/2754b197c935ee63101cbbca2752338246384fec/eth/gasprice/feehistory.go#L218C8-L225 @@ -76,7 +76,7 @@ where } let Some(end_block) = self.provider().block_number_for_id(newest_block.into())? else { - return Err(EthApiError::UnknownBlockNumber); + return Err(EthApiError::UnknownBlockNumber) }; // need to add 1 to the end block to get the correct (inclusive) range @@ -92,7 +92,7 @@ where // Note: The types used ensure that the percentiles are never < 0 if let Some(percentiles) = &reward_percentiles { if percentiles.windows(2).any(|w| w[0] > w[1] || w[0] > 100.) { - return Err(EthApiError::InvalidRewardPercentiles); + return Err(EthApiError::InvalidRewardPercentiles) } } @@ -106,19 +106,23 @@ where let mut fee_entries = self.fee_history_cache().get_history(start_block, end_block).await?; if fee_entries.is_empty() { - fee_entries = self - .provider() - .block_range(start_block..=end_block)? - .iter() - .map(|block| FeeHistoryEntry::new(block, block.clone().seal_slow().hash)) - .collect(); + for block in self.provider().block_range(start_block..=end_block)?.iter() { + let mut entry = FeeHistoryEntry::new(block, block.clone().seal_slow().hash); + let percentiles = predefined_percentiles(); + + entry.rewards = calculate_reward_percentiles_for_block( + &percentiles, + &entry, + self.cache().clone(), + ) + .await?; + fee_entries.push(entry); + } // calculate rewards on the fly - println!("took fee entries from the db on the fly") } - println!("fee entries len: {}", fee_entries.len()); if fee_entries.len() != block_count as usize { - return Err(EthApiError::InvalidBlockRange); + return Err(EthApiError::InvalidBlockRange) } // Collect base fees, gas usage ratios and (optionally) reward percentile data let mut base_fee_per_gas: Vec = Vec::new(); @@ -173,7 +177,6 @@ where // Calculate the index in the precomputed rewards array let index = (rounded_percentile / 0.25) as usize; - // Fetch the reward from the FeeHistoryEntry entry.rewards.get(index).cloned().unwrap_or(U256::ZERO) } diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index ee6d0196683d..6a20ad29d48e 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -642,7 +642,6 @@ async fn calculate_reward_percentiles_for_block( .get_transactions_and_receipts(fee_entry.header_hash) .await? .ok_or(EthApiError::InvalidBlockRange)?; - let mut transactions = transactions .into_iter() .zip(receipts) diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index c63de87e710b..5081063350fc 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -688,4 +688,45 @@ mod tests { "all: no percentiles were requested, so there should be no rewards result" ); } + + #[tokio::test] + /// Requesting all blocks + percentiles should be ok + async fn test_fee_history_all_blocks_with_percentiles() { + let block_count = 10; + let newest_block = 1337; + let oldest_block = None; + + let (eth_api, base_fees_per_gas, gas_used_ratios) = + prepare_eth_api(newest_block, oldest_block, block_count, MockEthProvider::default()); + + let reward_percentiles: Option> = Some(vec![50.0, 75.0, 99.0]); + + let fee_history = eth_api + .fee_history(block_count, (newest_block).into(), reward_percentiles) + .await + .unwrap(); + + assert_eq!( + &fee_history.base_fee_per_gas, &base_fees_per_gas, + "all: base fee per gas is incorrect" + ); + assert_eq!( + fee_history.base_fee_per_gas.len() as u64, + block_count + 1, + "all: should return base fee of the next block as well" + ); + assert_eq!( + &fee_history.gas_used_ratio, &gas_used_ratios, + "all: gas used ratio is incorrect" + ); + assert_eq!( + fee_history.oldest_block, + U256::from(newest_block - block_count + 1), + "all: oldest block is incorrect" + ); + assert!( + !fee_history.reward.is_none(), + "all: percentiles were requested, so there should be rewards result" + ); + } } From ff3aed8eba640927ef882f994b9b079850a467d9 Mon Sep 17 00:00:00 2001 From: allnil Date: Thu, 16 Nov 2023 18:19:38 +0000 Subject: [PATCH 22/32] remove test because of transactions, for now --- crates/rpc/rpc/src/eth/api/server.rs | 41 ---------------------------- 1 file changed, 41 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index 5081063350fc..c63de87e710b 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -688,45 +688,4 @@ mod tests { "all: no percentiles were requested, so there should be no rewards result" ); } - - #[tokio::test] - /// Requesting all blocks + percentiles should be ok - async fn test_fee_history_all_blocks_with_percentiles() { - let block_count = 10; - let newest_block = 1337; - let oldest_block = None; - - let (eth_api, base_fees_per_gas, gas_used_ratios) = - prepare_eth_api(newest_block, oldest_block, block_count, MockEthProvider::default()); - - let reward_percentiles: Option> = Some(vec![50.0, 75.0, 99.0]); - - let fee_history = eth_api - .fee_history(block_count, (newest_block).into(), reward_percentiles) - .await - .unwrap(); - - assert_eq!( - &fee_history.base_fee_per_gas, &base_fees_per_gas, - "all: base fee per gas is incorrect" - ); - assert_eq!( - fee_history.base_fee_per_gas.len() as u64, - block_count + 1, - "all: should return base fee of the next block as well" - ); - assert_eq!( - &fee_history.gas_used_ratio, &gas_used_ratios, - "all: gas used ratio is incorrect" - ); - assert_eq!( - fee_history.oldest_block, - U256::from(newest_block - block_count + 1), - "all: oldest block is incorrect" - ); - assert!( - !fee_history.reward.is_none(), - "all: percentiles were requested, so there should be rewards result" - ); - } } From 2f61a8cceb25c20980a06279574f7a776d9e7ab8 Mon Sep 17 00:00:00 2001 From: Nil Medvedev Date: Mon, 20 Nov 2023 19:54:47 +0000 Subject: [PATCH 23/32] Update crates/rpc/rpc/src/eth/api/mod.rs Co-authored-by: Matthias Seitz --- crates/rpc/rpc/src/eth/api/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 6a20ad29d48e..5c29856d12ff 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -513,7 +513,7 @@ impl FeeHistoryCache { } /// Processing of the arriving blocks - pub async fn on_new_block<'a, I>(&self, blocks: I) + pub async fn on_new_blocks<'a, I>(&self, blocks: I) where I: Iterator, { From b131559698886b93ddfff3dce6f3882c277c55e0 Mon Sep 17 00:00:00 2001 From: allnil Date: Mon, 20 Nov 2023 22:14:50 +0000 Subject: [PATCH 24/32] fix docs, add resolution in config, improve naming, move from blocks to sealed_blocks --- .../src/test_utils/bodies_client.rs | 2 +- crates/net/eth-wire/src/capability.rs | 12 +- crates/primitives/src/snapshot/segment.rs | 4 +- crates/rpc/rpc-builder/src/lib.rs | 31 +++-- crates/rpc/rpc-engine-api/src/engine_api.rs | 44 +++---- crates/rpc/rpc/src/eth/api/fees.rs | 54 ++++----- crates/rpc/rpc/src/eth/api/mod.rs | 111 ++++++++++-------- 7 files changed, 130 insertions(+), 128 deletions(-) diff --git a/crates/net/downloaders/src/test_utils/bodies_client.rs b/crates/net/downloaders/src/test_utils/bodies_client.rs index 2f3cf2f293fb..ac791742d009 100644 --- a/crates/net/downloaders/src/test_utils/bodies_client.rs +++ b/crates/net/downloaders/src/test_utils/bodies_client.rs @@ -92,7 +92,7 @@ impl BodiesClient for TestBodiesClient { Box::pin(async move { if should_respond_empty { - return Ok((PeerId::default(), vec![]).into()) + return Ok((PeerId::default(), vec![]).into()); } if should_delay { diff --git a/crates/net/eth-wire/src/capability.rs b/crates/net/eth-wire/src/capability.rs index d763c6ed412d..9f5d0c65d96b 100644 --- a/crates/net/eth-wire/src/capability.rs +++ b/crates/net/eth-wire/src/capability.rs @@ -252,7 +252,7 @@ impl SharedCapability { /// Returns an error if the offset is equal or less than [`MAX_RESERVED_MESSAGE_ID`]. pub(crate) fn new(name: &str, version: u8, offset: u8) -> Result { if offset <= MAX_RESERVED_MESSAGE_ID { - return Err(SharedCapabilityError::ReservedMessageIdOffset(offset)) + return Err(SharedCapabilityError::ReservedMessageIdOffset(offset)); } match name { @@ -350,7 +350,7 @@ impl SharedCapabilities { pub fn eth(&self) -> Result<&SharedCapability, P2PStreamError> { for cap in self.iter_caps() { if cap.is_eth() { - return Ok(cap) + return Ok(cap); } } Err(P2PStreamError::CapabilityNotShared) @@ -427,8 +427,8 @@ pub fn shared_capability_offsets( let version = shared_capabilities.get(&peer_capability.name).map(|v| v.version); // TODO(mattsse): simplify - if version.is_none() || - (version.is_some() && peer_capability.version > version.expect("is some; qed")) + if version.is_none() + || (version.is_some() && peer_capability.version > version.expect("is some; qed")) { shared_capabilities.insert( peer_capability.name.clone(), @@ -441,7 +441,7 @@ pub fn shared_capability_offsets( // disconnect if we don't share any capabilities if shared_capabilities.is_empty() { - return Err(P2PStreamError::HandshakeError(P2PHandshakeError::NoSharedCapabilities)) + return Err(P2PStreamError::HandshakeError(P2PHandshakeError::NoSharedCapabilities)); } // order versions based on capability name (alphabetical) and select offsets based on @@ -462,7 +462,7 @@ pub fn shared_capability_offsets( } if shared_with_offsets.is_empty() { - return Err(P2PStreamError::HandshakeError(P2PHandshakeError::NoSharedCapabilities)) + return Err(P2PStreamError::HandshakeError(P2PHandshakeError::NoSharedCapabilities)); } Ok(shared_with_offsets) diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index 8a920c0f8e74..d8357fc169cb 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -117,7 +117,7 @@ impl SnapshotSegment { ) -> Option<(Self, RangeInclusive, RangeInclusive)> { let mut parts = name.to_str()?.split('_'); if parts.next() != Some("snapshot") { - return None + return None; } let segment = Self::from_str(parts.next()?).ok()?; @@ -125,7 +125,7 @@ impl SnapshotSegment { let (tx_start, tx_end) = (parts.next()?.parse().ok()?, parts.next()?.parse().ok()?); if block_start >= block_end || tx_start > tx_end { - return None + return None; } Some((segment, block_start..=block_end, tx_start..=tx_end)) diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index ba9b2a1fcf0c..35c95c79c01a 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -685,7 +685,7 @@ impl FromStr for RpcModuleSelection { fn from_str(s: &str) -> Result { if s.is_empty() { - return Ok(Selection(vec![])) + return Ok(Selection(vec![])); } let mut modules = s.split(',').map(str::trim).peekable(); let first = modules.peek().copied().ok_or(ParseError::VariantNotFound)?; @@ -1116,9 +1116,8 @@ where }), ); - let c = cache.clone(); let fee_history_cache = - FeeHistoryCache::new(c, self.config.eth.fee_history_cache.clone()); + FeeHistoryCache::new(cache.clone(), self.config.eth.fee_history_cache.clone()); let new_canonical_blocks = self.events.canonical_state_stream(); let fhc = fee_history_cache.clone(); let provider_clone = self.provider.clone(); @@ -1381,9 +1380,9 @@ impl RpcServerConfig { /// /// If no server is configured, no server will be be launched on [RpcServerConfig::start]. pub fn has_server(&self) -> bool { - self.http_server_config.is_some() || - self.ws_server_config.is_some() || - self.ipc_server_config.is_some() + self.http_server_config.is_some() + || self.ws_server_config.is_some() + || self.ipc_server_config.is_some() } /// Returns the [SocketAddr] of the http server @@ -1424,9 +1423,9 @@ impl RpcServerConfig { .unwrap_or(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, DEFAULT_WS_RPC_PORT))); // If both are configured on the same port, we combine them into one server. - if self.http_addr == self.ws_addr && - self.http_server_config.is_some() && - self.ws_server_config.is_some() + if self.http_addr == self.ws_addr + && self.http_server_config.is_some() + && self.ws_server_config.is_some() { let cors = match (self.ws_cors_domains.as_ref(), self.http_cors_domains.as_ref()) { (Some(ws_cors), Some(http_cors)) => { @@ -1435,7 +1434,7 @@ impl RpcServerConfig { http_cors_domains: Some(http_cors.clone()), ws_cors_domains: Some(ws_cors.clone()), } - .into()) + .into()); } Some(ws_cors) } @@ -1472,7 +1471,7 @@ impl RpcServerConfig { ws_local_addr: Some(addr), server: WsHttpServers::SamePort(server), jwt_secret, - }) + }); } let mut http_local_addr = None; @@ -1671,7 +1670,7 @@ impl TransportRpcModules { other: impl Into, ) -> Result { if let Some(ref mut http) = self.http { - return http.merge(other.into()).map(|_| true) + return http.merge(other.into()).map(|_| true); } Ok(false) } @@ -1686,7 +1685,7 @@ impl TransportRpcModules { other: impl Into, ) -> Result { if let Some(ref mut ws) = self.ws { - return ws.merge(other.into()).map(|_| true) + return ws.merge(other.into()).map(|_| true); } Ok(false) } @@ -1701,7 +1700,7 @@ impl TransportRpcModules { other: impl Into, ) -> Result { if let Some(ref mut ipc) = self.ipc { - return ipc.merge(other.into()).map(|_| true) + return ipc.merge(other.into()).map(|_| true); } Ok(false) } @@ -1996,8 +1995,8 @@ impl RpcServerHandle { "Bearer {}", secret .encode(&Claims { - iat: (SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + - Duration::from_secs(60)) + iat: (SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + + Duration::from_secs(60)) .as_secs(), exp: None, }) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index ba3ac0e66614..135869ba1528 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -308,7 +308,7 @@ where ) -> EngineApiResult { let len = hashes.len() as u64; if len > MAX_PAYLOAD_BODIES_LIMIT { - return Err(EngineApiError::PayloadRequestTooLarge { len }) + return Err(EngineApiError::PayloadRequestTooLarge { len }); } let mut result = Vec::with_capacity(hashes.len()); @@ -348,7 +348,7 @@ where return Err(EngineApiError::TerminalTD { execution: merge_terminal_td, consensus: terminal_total_difficulty, - }) + }); } self.inner.beacon_consensus.transition_configuration_exchanged().await; @@ -358,7 +358,7 @@ where return Ok(TransitionConfiguration { terminal_total_difficulty: merge_terminal_td, ..Default::default() - }) + }); } // Attempt to look up terminal block hash @@ -413,7 +413,7 @@ where // 1. Client software **MUST** return `-38005: Unsupported fork` error if the // `timestamp` of payload or payloadAttributes greater or equal to the Cancun // activation timestamp. - return Err(EngineApiError::UnsupportedFork) + return Err(EngineApiError::UnsupportedFork); } if version == EngineApiMessageVersion::V3 && !is_cancun { @@ -423,7 +423,7 @@ where // 1. Client software **MUST** return `-38005: Unsupported fork` error if the // `timestamp` of the built payload does not fall within the time frame of the Cancun // fork. - return Err(EngineApiError::UnsupportedFork) + return Err(EngineApiError::UnsupportedFork); } Ok(()) } @@ -443,18 +443,18 @@ where match version { EngineApiMessageVersion::V1 => { if has_withdrawals { - return Err(EngineApiError::WithdrawalsNotSupportedInV1) + return Err(EngineApiError::WithdrawalsNotSupportedInV1); } if is_shanghai { - return Err(EngineApiError::NoWithdrawalsPostShanghai) + return Err(EngineApiError::NoWithdrawalsPostShanghai); } } EngineApiMessageVersion::V2 | EngineApiMessageVersion::V3 => { if is_shanghai && !has_withdrawals { - return Err(EngineApiError::NoWithdrawalsPostShanghai) + return Err(EngineApiError::NoWithdrawalsPostShanghai); } if !is_shanghai && has_withdrawals { - return Err(EngineApiError::HasWithdrawalsPreShanghai) + return Err(EngineApiError::HasWithdrawalsPreShanghai); } } }; @@ -498,12 +498,12 @@ where match version { EngineApiMessageVersion::V1 | EngineApiMessageVersion::V2 => { if has_parent_beacon_block_root { - return Err(EngineApiError::ParentBeaconBlockRootNotSupportedBeforeV3) + return Err(EngineApiError::ParentBeaconBlockRootNotSupportedBeforeV3); } } EngineApiMessageVersion::V3 => { if !has_parent_beacon_block_root { - return Err(EngineApiError::NoParentBeaconBlockRootPostCancun) + return Err(EngineApiError::NoParentBeaconBlockRootPostCancun); } } }; @@ -558,10 +558,10 @@ where let attr_validation_res = self.validate_version_specific_fields(version, &attrs.into()); #[cfg(feature = "optimism")] - if attrs.optimism_payload_attributes.gas_limit.is_none() && - self.inner.chain_spec.is_optimism() + if attrs.optimism_payload_attributes.gas_limit.is_none() + && self.inner.chain_spec.is_optimism() { - return Err(EngineApiError::MissingGasLimitInPayloadAttributes) + return Err(EngineApiError::MissingGasLimitInPayloadAttributes); } // From the engine API spec: @@ -582,9 +582,9 @@ where // TODO: decide if we want this branch - the FCU INVALID response might be more // useful than the payload attributes INVALID response if fcu_res.is_invalid() { - return Ok(fcu_res) + return Ok(fcu_res); } - return Err(err) + return Err(err); } } @@ -920,8 +920,8 @@ mod tests { blocks .iter() .filter(|b| { - !first_missing_range.contains(&b.number) && - !second_missing_range.contains(&b.number) + !first_missing_range.contains(&b.number) + && !second_missing_range.contains(&b.number) }) .map(|b| (b.hash(), b.clone().unseal())), ); @@ -950,8 +950,8 @@ mod tests { // ensure we still return trailing `None`s here because by-hash will not be aware // of the missing block's number, and cannot compare it to the current best block .map(|b| { - if first_missing_range.contains(&b.number) || - second_missing_range.contains(&b.number) + if first_missing_range.contains(&b.number) + || second_missing_range.contains(&b.number) { None } else { @@ -977,8 +977,8 @@ mod tests { let (handle, api) = setup_engine_api(); let transition_config = TransitionConfiguration { - terminal_total_difficulty: handle.chain_spec.fork(Hardfork::Paris).ttd().unwrap() + - U256::from(1), + terminal_total_difficulty: handle.chain_spec.fork(Hardfork::Paris).ttd().unwrap() + + U256::from(1), ..Default::default() }; diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 0e5a0c5359b5..1aade8c1d162 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -13,7 +13,7 @@ use reth_transaction_pool::TransactionPool; use tracing::debug; -use super::{calculate_reward_percentiles_for_block, predefined_percentiles, FeeHistoryEntry}; +use super::FeeHistoryEntry; impl EthApi where @@ -56,7 +56,7 @@ where reward_percentiles: Option>, ) -> EthResult { if block_count == 0 { - return Ok(FeeHistory::default()) + return Ok(FeeHistory::default()); } // See https://github.com/ethereum/go-ethereum/blob/2754b197c935ee63101cbbca2752338246384fec/eth/gasprice/feehistory.go#L218C8-L225 @@ -76,7 +76,7 @@ where } let Some(end_block) = self.provider().block_number_for_id(newest_block.into())? else { - return Err(EthApiError::UnknownBlockNumber) + return Err(EthApiError::UnknownBlockNumber); }; // need to add 1 to the end block to get the correct (inclusive) range @@ -92,7 +92,7 @@ where // Note: The types used ensure that the percentiles are never < 0 if let Some(percentiles) = &reward_percentiles { if percentiles.windows(2).any(|w| w[0] > w[1] || w[0] > 100.) { - return Err(EthApiError::InvalidRewardPercentiles) + return Err(EthApiError::InvalidRewardPercentiles); } } @@ -105,24 +105,18 @@ where let start_block = end_block_plus - block_count; let mut fee_entries = self.fee_history_cache().get_history(start_block, end_block).await?; + + let mut fee_cache_flag = true; if fee_entries.is_empty() { - for block in self.provider().block_range(start_block..=end_block)?.iter() { - let mut entry = FeeHistoryEntry::new(block, block.clone().seal_slow().hash); - let percentiles = predefined_percentiles(); - - entry.rewards = calculate_reward_percentiles_for_block( - &percentiles, - &entry, - self.cache().clone(), - ) - .await?; + for block in self.provider().block_range(start_block..=end_block)?.into_iter() { + let entry = FeeHistoryEntry::new(&block.seal_slow()); fee_entries.push(entry); } - // calculate rewards on the fly + fee_cache_flag = false; } if fee_entries.len() != block_count as usize { - return Err(EthApiError::InvalidBlockRange) + return Err(EthApiError::InvalidBlockRange); } // Collect base fees, gas usage ratios and (optionally) reward percentile data let mut base_fee_per_gas: Vec = Vec::new(); @@ -133,13 +127,15 @@ where base_fee_per_gas.push(U256::try_from(entry.base_fee_per_gas).unwrap()); gas_used_ratio.push(entry.gas_used_ratio); - if let Some(percentiles) = &reward_percentiles { - let mut block_rewards = Vec::new(); + if fee_cache_flag == true { + if let Some(percentiles) = &reward_percentiles { + let mut block_rewards = Vec::new(); - for &percentile in percentiles.iter() { - block_rewards.push(self.fetch_reward_for_percentile(entry, percentile)); + for &percentile in percentiles.iter() { + block_rewards.push(self.approximate_percentile(entry, percentile)); + } + rewards.push(block_rewards); } - rewards.push(block_rewards); } } @@ -167,16 +163,16 @@ where }) } - fn fetch_reward_for_percentile( - &self, - entry: &FeeHistoryEntry, - requested_percentile: f64, - ) -> U256 { - let rounded_percentile = (requested_percentile / 0.25).round() * 0.25; - let rounded_percentile = rounded_percentile.clamp(0.0, 100.0); + /// Approximates reward at a given percentile for a specific block + /// Based on the configured resolution + fn approximate_percentile(&self, entry: &FeeHistoryEntry, requested_percentile: f64) -> U256 { + let resolution = self.fee_history_cache().config.resolution; + let rounded_percentile = + (requested_percentile * resolution as f64).round() / resolution as f64; + let clamped_percentile = rounded_percentile.clamp(0.0, 100.0); // Calculate the index in the precomputed rewards array - let index = (rounded_percentile / 0.25) as usize; + let index = (clamped_percentile / (1.0 / resolution as f64)).round() as usize; // Fetch the reward from the FeeHistoryEntry entry.rewards.get(index).cloned().unwrap_or(U256::ZERO) } diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 83d342f66a30..094da1750cf1 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -15,8 +15,8 @@ use reth_interfaces::RethResult; use reth_network_api::NetworkInfo; use reth_primitives::{ revm_primitives::{BlockEnv, CfgEnv}, - Address, Block, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, SealedBlockWithSenders, - B256, U256, U64, + Address, BlockId, BlockNumberOrTag, ChainInfo, Receipt, SealedBlock, SealedBlockWithSenders, + TransactionSigned, B256, U256, U64, }; use serde::{Deserialize, Serialize}; @@ -303,7 +303,7 @@ where pub(crate) async fn local_pending_block(&self) -> EthResult> { let pending = self.pending_block_env_and_cfg()?; if pending.origin.is_actual_pending() { - return Ok(pending.origin.into_actual_pending()) + return Ok(pending.origin.into_actual_pending()); } // no pending block from the CL yet, so we need to build it ourselves via txpool @@ -314,17 +314,17 @@ where // check if the block is still good if let Some(pending_block) = lock.as_ref() { // this is guaranteed to be the `latest` header - if pending.block_env.number.to::() == pending_block.block.number && - pending.origin.header().hash == pending_block.block.parent_hash && - now <= pending_block.expires_at + if pending.block_env.number.to::() == pending_block.block.number + && pending.origin.header().hash == pending_block.block.parent_hash + && now <= pending_block.expires_at { - return Ok(Some(pending_block.block.clone())) + return Ok(Some(pending_block.block.clone())); } } // if we're currently syncing, we're unable to build a pending block if this.network().is_syncing() { - return Ok(None) + return Ok(None); } // we rebuild the block @@ -332,7 +332,7 @@ where Ok(block) => block, Err(err) => { tracing::debug!(target: "rpc", "Failed to build pending block: {:?}", err); - return Ok(None) + return Ok(None); } }; @@ -482,11 +482,15 @@ pub struct FeeHistoryCacheConfig { /// /// Default is 1024. pub max_blocks: u64, + /// Percentile approximation resolution + /// + /// Default is 4 which means 0.25 + pub resolution: u64, } impl Default for FeeHistoryCacheConfig { fn default() -> Self { - FeeHistoryCacheConfig { max_blocks: 1024 } + FeeHistoryCacheConfig { max_blocks: 1024, resolution: 4 } } } @@ -516,34 +520,38 @@ impl FeeHistoryCache { /// Processing of the arriving blocks pub async fn on_new_blocks<'a, I>(&self, blocks: I) where - I: Iterator, + I: Iterator, { let mut entries = self.entries.write().await; for block in blocks { - let mut fee_history_entry = FeeHistoryEntry::new(block, block.clone().seal_slow().hash); - - let percentiles = predefined_percentiles(); - - fee_history_entry.rewards = calculate_reward_percentiles_for_block( - &percentiles, - &fee_history_entry, - self.eth_cache.clone(), - ) - .await - .unwrap_or_default(); - - // calculate rewards - entries.insert(block.number, fee_history_entry); + let mut fee_history_entry = FeeHistoryEntry::new(&block); + let percentiles = self.predefined_percentiles(); + + if let Ok(Some((transactions, receipts))) = + self.eth_cache.get_transactions_and_receipts(fee_history_entry.header_hash).await + { + fee_history_entry.rewards = calculate_reward_percentiles_for_block( + &percentiles, + &fee_history_entry, + transactions, + receipts, + ) + .await + .unwrap_or_default(); + + entries.insert(block.number, fee_history_entry); + } else { + break; + } } - while entries.len() > self.config.max_blocks as usize { entries.pop_first(); } if entries.len() == 0 { self.upper_bound.store(0, SeqCst); self.lower_bound.store(0, SeqCst); - return + return; } let upper_bound = *entries.last_entry().expect("Contains at least one entry").key(); let lower_bound = *entries.first_entry().expect("Contains at least one entry").key(); @@ -561,26 +569,35 @@ impl FeeHistoryCache { self.lower_bound.load(SeqCst) } - /// Collect fee history for given range. It will try to use a cache to take the most recent - /// headers or if the range is out of caching config it will fallback to the database provider + /// Collect fee history for given range. + /// This function retrieves fee history entries from the cache for the specified range. + /// If the requested range (star_block to end_block) is within the cache bounds, + /// it returns the corresponding entries. + /// Otherwise it returns None. pub async fn get_history( &self, start_block: u64, end_block: u64, ) -> RethResult> { - let mut result = Vec::new(); - let lower_bound = self.lower_bound(); let upper_bound = self.upper_bound(); if start_block >= lower_bound && end_block <= upper_bound { let entries = self.entries.read().await; - result = entries + let result = entries .range(start_block..=end_block + 1) .map(|(_, fee_entry)| fee_entry.clone()) .collect(); + Ok(result) + } else { + Ok(Vec::new()) } + } - Ok(result) + /// Generates predefined set of percentiles + pub fn predefined_percentiles(&self) -> Vec { + (0..=100 * self.config.resolution) + .map(|p| p as f64 / self.config.resolution as f64) + .collect() } } @@ -605,8 +622,9 @@ pub async fn fee_history_cache_new_blocks_task( }; let blocks = provider.block_range(start_block..=last_block_number).unwrap_or_default(); + let sealed = blocks.into_iter().map(|block| block.seal_slow()).collect::>(); - fee_history_cache.on_new_block(blocks.iter()).await; + fee_history_cache.on_new_blocks(sealed.iter()).await; } while let Some(event) = events.next().await { @@ -614,21 +632,13 @@ pub async fn fee_history_cache_new_blocks_task( // we're only interested in new committed blocks let (blocks, _) = committed.inner(); - let blocks = blocks - .iter() - .map(|(_, block)| block.block.clone().unseal().clone()) - .collect::>(); + let blocks = blocks.iter().map(|(_, v)| v.block.clone()).collect::>(); - fee_history_cache.on_new_block(blocks.iter()).await; + fee_history_cache.on_new_blocks(blocks.iter()).await; } } } -/// Generates predefined set of percentiles -fn predefined_percentiles() -> Vec { - (0..=400).map(|p| p as f64 * 0.25).collect() -} - /// Calculates reward percentiles for transactions in a block header. /// Given a list of percentiles and a sealed block header, this function computes /// the corresponding rewards for the transactions at each percentile. @@ -637,12 +647,9 @@ fn predefined_percentiles() -> Vec { async fn calculate_reward_percentiles_for_block( percentiles: &[f64], fee_entry: &FeeHistoryEntry, - cache: EthStateCache, + transactions: Vec, + receipts: Vec, ) -> Result, EthApiError> { - let (transactions, receipts) = cache - .get_transactions_and_receipts(fee_entry.header_hash) - .await? - .ok_or(EthApiError::InvalidBlockRange)?; let mut transactions = transactions .into_iter() .zip(receipts) @@ -679,7 +686,7 @@ async fn calculate_reward_percentiles_for_block( // Empty blocks should return in a zero row if transactions.is_empty() { rewards_in_block.push(U256::ZERO); - continue + continue; } let threshold = (fee_entry.gas_used as f64 * percentile / 100.) as u64; @@ -704,12 +711,12 @@ pub struct FeeHistoryEntry { } impl FeeHistoryEntry { - fn new(block: &Block, hash: B256) -> Self { + fn new(block: &SealedBlock) -> Self { FeeHistoryEntry { base_fee_per_gas: block.base_fee_per_gas.unwrap_or_default(), gas_used_ratio: block.gas_used as f64 / block.gas_limit as f64, gas_used: block.gas_used, - header_hash: hash, + header_hash: block.hash, gas_limit: block.gas_limit, rewards: Vec::new(), } From b904b37f521794aa4e21a135831534b1b3b0af48 Mon Sep 17 00:00:00 2001 From: allnil Date: Mon, 20 Nov 2023 22:15:26 +0000 Subject: [PATCH 25/32] apply nightly rules --- .../src/test_utils/bodies_client.rs | 2 +- crates/net/eth-wire/src/capability.rs | 12 ++--- crates/primitives/src/snapshot/segment.rs | 4 +- crates/rpc/rpc-builder/src/lib.rs | 28 ++++++------ crates/rpc/rpc-engine-api/src/engine_api.rs | 44 +++++++++---------- crates/rpc/rpc/src/eth/api/fees.rs | 8 ++-- crates/rpc/rpc/src/eth/api/mod.rs | 20 ++++----- 7 files changed, 59 insertions(+), 59 deletions(-) diff --git a/crates/net/downloaders/src/test_utils/bodies_client.rs b/crates/net/downloaders/src/test_utils/bodies_client.rs index ac791742d009..2f3cf2f293fb 100644 --- a/crates/net/downloaders/src/test_utils/bodies_client.rs +++ b/crates/net/downloaders/src/test_utils/bodies_client.rs @@ -92,7 +92,7 @@ impl BodiesClient for TestBodiesClient { Box::pin(async move { if should_respond_empty { - return Ok((PeerId::default(), vec![]).into()); + return Ok((PeerId::default(), vec![]).into()) } if should_delay { diff --git a/crates/net/eth-wire/src/capability.rs b/crates/net/eth-wire/src/capability.rs index 9f5d0c65d96b..d763c6ed412d 100644 --- a/crates/net/eth-wire/src/capability.rs +++ b/crates/net/eth-wire/src/capability.rs @@ -252,7 +252,7 @@ impl SharedCapability { /// Returns an error if the offset is equal or less than [`MAX_RESERVED_MESSAGE_ID`]. pub(crate) fn new(name: &str, version: u8, offset: u8) -> Result { if offset <= MAX_RESERVED_MESSAGE_ID { - return Err(SharedCapabilityError::ReservedMessageIdOffset(offset)); + return Err(SharedCapabilityError::ReservedMessageIdOffset(offset)) } match name { @@ -350,7 +350,7 @@ impl SharedCapabilities { pub fn eth(&self) -> Result<&SharedCapability, P2PStreamError> { for cap in self.iter_caps() { if cap.is_eth() { - return Ok(cap); + return Ok(cap) } } Err(P2PStreamError::CapabilityNotShared) @@ -427,8 +427,8 @@ pub fn shared_capability_offsets( let version = shared_capabilities.get(&peer_capability.name).map(|v| v.version); // TODO(mattsse): simplify - if version.is_none() - || (version.is_some() && peer_capability.version > version.expect("is some; qed")) + if version.is_none() || + (version.is_some() && peer_capability.version > version.expect("is some; qed")) { shared_capabilities.insert( peer_capability.name.clone(), @@ -441,7 +441,7 @@ pub fn shared_capability_offsets( // disconnect if we don't share any capabilities if shared_capabilities.is_empty() { - return Err(P2PStreamError::HandshakeError(P2PHandshakeError::NoSharedCapabilities)); + return Err(P2PStreamError::HandshakeError(P2PHandshakeError::NoSharedCapabilities)) } // order versions based on capability name (alphabetical) and select offsets based on @@ -462,7 +462,7 @@ pub fn shared_capability_offsets( } if shared_with_offsets.is_empty() { - return Err(P2PStreamError::HandshakeError(P2PHandshakeError::NoSharedCapabilities)); + return Err(P2PStreamError::HandshakeError(P2PHandshakeError::NoSharedCapabilities)) } Ok(shared_with_offsets) diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index d8357fc169cb..8a920c0f8e74 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -117,7 +117,7 @@ impl SnapshotSegment { ) -> Option<(Self, RangeInclusive, RangeInclusive)> { let mut parts = name.to_str()?.split('_'); if parts.next() != Some("snapshot") { - return None; + return None } let segment = Self::from_str(parts.next()?).ok()?; @@ -125,7 +125,7 @@ impl SnapshotSegment { let (tx_start, tx_end) = (parts.next()?.parse().ok()?, parts.next()?.parse().ok()?); if block_start >= block_end || tx_start > tx_end { - return None; + return None } Some((segment, block_start..=block_end, tx_start..=tx_end)) diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 35c95c79c01a..cb12eb90069f 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -685,7 +685,7 @@ impl FromStr for RpcModuleSelection { fn from_str(s: &str) -> Result { if s.is_empty() { - return Ok(Selection(vec![])); + return Ok(Selection(vec![])) } let mut modules = s.split(',').map(str::trim).peekable(); let first = modules.peek().copied().ok_or(ParseError::VariantNotFound)?; @@ -1380,9 +1380,9 @@ impl RpcServerConfig { /// /// If no server is configured, no server will be be launched on [RpcServerConfig::start]. pub fn has_server(&self) -> bool { - self.http_server_config.is_some() - || self.ws_server_config.is_some() - || self.ipc_server_config.is_some() + self.http_server_config.is_some() || + self.ws_server_config.is_some() || + self.ipc_server_config.is_some() } /// Returns the [SocketAddr] of the http server @@ -1423,9 +1423,9 @@ impl RpcServerConfig { .unwrap_or(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, DEFAULT_WS_RPC_PORT))); // If both are configured on the same port, we combine them into one server. - if self.http_addr == self.ws_addr - && self.http_server_config.is_some() - && self.ws_server_config.is_some() + if self.http_addr == self.ws_addr && + self.http_server_config.is_some() && + self.ws_server_config.is_some() { let cors = match (self.ws_cors_domains.as_ref(), self.http_cors_domains.as_ref()) { (Some(ws_cors), Some(http_cors)) => { @@ -1434,7 +1434,7 @@ impl RpcServerConfig { http_cors_domains: Some(http_cors.clone()), ws_cors_domains: Some(ws_cors.clone()), } - .into()); + .into()) } Some(ws_cors) } @@ -1471,7 +1471,7 @@ impl RpcServerConfig { ws_local_addr: Some(addr), server: WsHttpServers::SamePort(server), jwt_secret, - }); + }) } let mut http_local_addr = None; @@ -1670,7 +1670,7 @@ impl TransportRpcModules { other: impl Into, ) -> Result { if let Some(ref mut http) = self.http { - return http.merge(other.into()).map(|_| true); + return http.merge(other.into()).map(|_| true) } Ok(false) } @@ -1685,7 +1685,7 @@ impl TransportRpcModules { other: impl Into, ) -> Result { if let Some(ref mut ws) = self.ws { - return ws.merge(other.into()).map(|_| true); + return ws.merge(other.into()).map(|_| true) } Ok(false) } @@ -1700,7 +1700,7 @@ impl TransportRpcModules { other: impl Into, ) -> Result { if let Some(ref mut ipc) = self.ipc { - return ipc.merge(other.into()).map(|_| true); + return ipc.merge(other.into()).map(|_| true) } Ok(false) } @@ -1995,8 +1995,8 @@ impl RpcServerHandle { "Bearer {}", secret .encode(&Claims { - iat: (SystemTime::now().duration_since(UNIX_EPOCH).unwrap() - + Duration::from_secs(60)) + iat: (SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + + Duration::from_secs(60)) .as_secs(), exp: None, }) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 135869ba1528..ba3ac0e66614 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -308,7 +308,7 @@ where ) -> EngineApiResult { let len = hashes.len() as u64; if len > MAX_PAYLOAD_BODIES_LIMIT { - return Err(EngineApiError::PayloadRequestTooLarge { len }); + return Err(EngineApiError::PayloadRequestTooLarge { len }) } let mut result = Vec::with_capacity(hashes.len()); @@ -348,7 +348,7 @@ where return Err(EngineApiError::TerminalTD { execution: merge_terminal_td, consensus: terminal_total_difficulty, - }); + }) } self.inner.beacon_consensus.transition_configuration_exchanged().await; @@ -358,7 +358,7 @@ where return Ok(TransitionConfiguration { terminal_total_difficulty: merge_terminal_td, ..Default::default() - }); + }) } // Attempt to look up terminal block hash @@ -413,7 +413,7 @@ where // 1. Client software **MUST** return `-38005: Unsupported fork` error if the // `timestamp` of payload or payloadAttributes greater or equal to the Cancun // activation timestamp. - return Err(EngineApiError::UnsupportedFork); + return Err(EngineApiError::UnsupportedFork) } if version == EngineApiMessageVersion::V3 && !is_cancun { @@ -423,7 +423,7 @@ where // 1. Client software **MUST** return `-38005: Unsupported fork` error if the // `timestamp` of the built payload does not fall within the time frame of the Cancun // fork. - return Err(EngineApiError::UnsupportedFork); + return Err(EngineApiError::UnsupportedFork) } Ok(()) } @@ -443,18 +443,18 @@ where match version { EngineApiMessageVersion::V1 => { if has_withdrawals { - return Err(EngineApiError::WithdrawalsNotSupportedInV1); + return Err(EngineApiError::WithdrawalsNotSupportedInV1) } if is_shanghai { - return Err(EngineApiError::NoWithdrawalsPostShanghai); + return Err(EngineApiError::NoWithdrawalsPostShanghai) } } EngineApiMessageVersion::V2 | EngineApiMessageVersion::V3 => { if is_shanghai && !has_withdrawals { - return Err(EngineApiError::NoWithdrawalsPostShanghai); + return Err(EngineApiError::NoWithdrawalsPostShanghai) } if !is_shanghai && has_withdrawals { - return Err(EngineApiError::HasWithdrawalsPreShanghai); + return Err(EngineApiError::HasWithdrawalsPreShanghai) } } }; @@ -498,12 +498,12 @@ where match version { EngineApiMessageVersion::V1 | EngineApiMessageVersion::V2 => { if has_parent_beacon_block_root { - return Err(EngineApiError::ParentBeaconBlockRootNotSupportedBeforeV3); + return Err(EngineApiError::ParentBeaconBlockRootNotSupportedBeforeV3) } } EngineApiMessageVersion::V3 => { if !has_parent_beacon_block_root { - return Err(EngineApiError::NoParentBeaconBlockRootPostCancun); + return Err(EngineApiError::NoParentBeaconBlockRootPostCancun) } } }; @@ -558,10 +558,10 @@ where let attr_validation_res = self.validate_version_specific_fields(version, &attrs.into()); #[cfg(feature = "optimism")] - if attrs.optimism_payload_attributes.gas_limit.is_none() - && self.inner.chain_spec.is_optimism() + if attrs.optimism_payload_attributes.gas_limit.is_none() && + self.inner.chain_spec.is_optimism() { - return Err(EngineApiError::MissingGasLimitInPayloadAttributes); + return Err(EngineApiError::MissingGasLimitInPayloadAttributes) } // From the engine API spec: @@ -582,9 +582,9 @@ where // TODO: decide if we want this branch - the FCU INVALID response might be more // useful than the payload attributes INVALID response if fcu_res.is_invalid() { - return Ok(fcu_res); + return Ok(fcu_res) } - return Err(err); + return Err(err) } } @@ -920,8 +920,8 @@ mod tests { blocks .iter() .filter(|b| { - !first_missing_range.contains(&b.number) - && !second_missing_range.contains(&b.number) + !first_missing_range.contains(&b.number) && + !second_missing_range.contains(&b.number) }) .map(|b| (b.hash(), b.clone().unseal())), ); @@ -950,8 +950,8 @@ mod tests { // ensure we still return trailing `None`s here because by-hash will not be aware // of the missing block's number, and cannot compare it to the current best block .map(|b| { - if first_missing_range.contains(&b.number) - || second_missing_range.contains(&b.number) + if first_missing_range.contains(&b.number) || + second_missing_range.contains(&b.number) { None } else { @@ -977,8 +977,8 @@ mod tests { let (handle, api) = setup_engine_api(); let transition_config = TransitionConfiguration { - terminal_total_difficulty: handle.chain_spec.fork(Hardfork::Paris).ttd().unwrap() - + U256::from(1), + terminal_total_difficulty: handle.chain_spec.fork(Hardfork::Paris).ttd().unwrap() + + U256::from(1), ..Default::default() }; diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 1aade8c1d162..4e056c71b802 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -56,7 +56,7 @@ where reward_percentiles: Option>, ) -> EthResult { if block_count == 0 { - return Ok(FeeHistory::default()); + return Ok(FeeHistory::default()) } // See https://github.com/ethereum/go-ethereum/blob/2754b197c935ee63101cbbca2752338246384fec/eth/gasprice/feehistory.go#L218C8-L225 @@ -76,7 +76,7 @@ where } let Some(end_block) = self.provider().block_number_for_id(newest_block.into())? else { - return Err(EthApiError::UnknownBlockNumber); + return Err(EthApiError::UnknownBlockNumber) }; // need to add 1 to the end block to get the correct (inclusive) range @@ -92,7 +92,7 @@ where // Note: The types used ensure that the percentiles are never < 0 if let Some(percentiles) = &reward_percentiles { if percentiles.windows(2).any(|w| w[0] > w[1] || w[0] > 100.) { - return Err(EthApiError::InvalidRewardPercentiles); + return Err(EthApiError::InvalidRewardPercentiles) } } @@ -116,7 +116,7 @@ where } if fee_entries.len() != block_count as usize { - return Err(EthApiError::InvalidBlockRange); + return Err(EthApiError::InvalidBlockRange) } // Collect base fees, gas usage ratios and (optionally) reward percentile data let mut base_fee_per_gas: Vec = Vec::new(); diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 094da1750cf1..db32d2403d9c 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -303,7 +303,7 @@ where pub(crate) async fn local_pending_block(&self) -> EthResult> { let pending = self.pending_block_env_and_cfg()?; if pending.origin.is_actual_pending() { - return Ok(pending.origin.into_actual_pending()); + return Ok(pending.origin.into_actual_pending()) } // no pending block from the CL yet, so we need to build it ourselves via txpool @@ -314,17 +314,17 @@ where // check if the block is still good if let Some(pending_block) = lock.as_ref() { // this is guaranteed to be the `latest` header - if pending.block_env.number.to::() == pending_block.block.number - && pending.origin.header().hash == pending_block.block.parent_hash - && now <= pending_block.expires_at + if pending.block_env.number.to::() == pending_block.block.number && + pending.origin.header().hash == pending_block.block.parent_hash && + now <= pending_block.expires_at { - return Ok(Some(pending_block.block.clone())); + return Ok(Some(pending_block.block.clone())) } } // if we're currently syncing, we're unable to build a pending block if this.network().is_syncing() { - return Ok(None); + return Ok(None) } // we rebuild the block @@ -332,7 +332,7 @@ where Ok(block) => block, Err(err) => { tracing::debug!(target: "rpc", "Failed to build pending block: {:?}", err); - return Ok(None); + return Ok(None) } }; @@ -542,7 +542,7 @@ impl FeeHistoryCache { entries.insert(block.number, fee_history_entry); } else { - break; + break } } while entries.len() > self.config.max_blocks as usize { @@ -551,7 +551,7 @@ impl FeeHistoryCache { if entries.len() == 0 { self.upper_bound.store(0, SeqCst); self.lower_bound.store(0, SeqCst); - return; + return } let upper_bound = *entries.last_entry().expect("Contains at least one entry").key(); let lower_bound = *entries.first_entry().expect("Contains at least one entry").key(); @@ -686,7 +686,7 @@ async fn calculate_reward_percentiles_for_block( // Empty blocks should return in a zero row if transactions.is_empty() { rewards_in_block.push(U256::ZERO); - continue; + continue } let threshold = (fee_entry.gas_used as f64 * percentile / 100.) as u64; From 3d4efe75aeb29cc15a7500e5fdcee26154c22c90 Mon Sep 17 00:00:00 2001 From: allnil Date: Mon, 20 Nov 2023 22:24:07 +0000 Subject: [PATCH 26/32] set proper and idiomatic semicolons --- crates/rpc/rpc-engine-api/src/engine_api.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index ba3ac0e66614..a000a00ab2e5 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -262,12 +262,12 @@ where self.inner.task_spawner.spawn_blocking(Box::pin(async move { if count > MAX_PAYLOAD_BODIES_LIMIT { tx.send(Err(EngineApiError::PayloadRequestTooLarge { len: count })).ok(); - return; + return } if start == 0 || count == 0 { tx.send(Err(EngineApiError::InvalidBodiesRange { start, count })).ok(); - return; + return } let mut result = Vec::with_capacity(count as usize); @@ -291,7 +291,7 @@ where } Err(err) => { tx.send(Err(EngineApiError::Internal(Box::new(err)))).ok(); - return; + return } }; } From 8e9328c84349ba2bde7c4b791c29a70f424d908e Mon Sep 17 00:00:00 2001 From: allnil Date: Wed, 22 Nov 2023 19:27:56 +0000 Subject: [PATCH 27/32] move types adjacent to fee history to separate file --- crates/rpc/rpc/src/eth/api/fee_history.rs | 270 ++++++++++++++++++++++ crates/rpc/rpc/src/eth/api/fees.rs | 2 +- crates/rpc/rpc/src/eth/api/mod.rs | 269 +-------------------- crates/rpc/rpc/src/eth/mod.rs | 5 +- 4 files changed, 283 insertions(+), 263 deletions(-) create mode 100644 crates/rpc/rpc/src/eth/api/fee_history.rs diff --git a/crates/rpc/rpc/src/eth/api/fee_history.rs b/crates/rpc/rpc/src/eth/api/fee_history.rs new file mode 100644 index 000000000000..9feab7b4af7c --- /dev/null +++ b/crates/rpc/rpc/src/eth/api/fee_history.rs @@ -0,0 +1,270 @@ +//! Consist of types adjacent to the fee history cache and its configs + +use crate::eth::{cache::EthStateCache, error::EthApiError}; + +use metrics::atomics::AtomicU64; +use reth_interfaces::RethResult; +use reth_primitives::{Receipt, SealedBlock, TransactionSigned, B256, U256}; +use serde::{Deserialize, Serialize}; + +use reth_provider::{BlockReaderIdExt, CanonStateNotification, ChainSpecProvider}; +use reth_rpc_types::TxGasAndReward; +use std::{ + collections::BTreeMap, + fmt::Debug, + sync::{atomic::Ordering::SeqCst, Arc}, +}; + +use futures::{Stream, StreamExt}; + +/// Settings for the [FeeHistoryCache](crate::eth::FeeHistoryCache). +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FeeHistoryCacheConfig { + /// Max number of blocks in cache. + /// + /// Default is 1024. + pub max_blocks: u64, + /// Percentile approximation resolution + /// + /// Default is 4 which means 0.25 + pub resolution: u64, +} + +impl Default for FeeHistoryCacheConfig { + fn default() -> Self { + FeeHistoryCacheConfig { max_blocks: 1024, resolution: 4 } + } +} + +/// Wrapper struct for BTreeMap +#[derive(Debug, Clone)] +pub struct FeeHistoryCache { + lower_bound: Arc, + upper_bound: Arc, + + /// Config for FeeHistoryCache, consists of resolution for percentile approximation + /// and max number of blocks + pub config: FeeHistoryCacheConfig, + entries: Arc>>, + eth_cache: EthStateCache, +} + +impl FeeHistoryCache { + /// Creates new FeeHistoryCache instance, initialize it with the mose recent data, set bounds + pub fn new(eth_cache: EthStateCache, config: FeeHistoryCacheConfig) -> Self { + let init_tree_map = BTreeMap::new(); + + let entries = Arc::new(tokio::sync::RwLock::new(init_tree_map)); + + let upper_bound = Arc::new(AtomicU64::new(0)); + let lower_bound = Arc::new(AtomicU64::new(0)); + + FeeHistoryCache { config, entries, upper_bound, lower_bound, eth_cache } + } + + /// Processing of the arriving blocks + pub async fn on_new_blocks<'a, I>(&self, blocks: I) + where + I: Iterator, + { + let mut entries = self.entries.write().await; + + for block in blocks { + let mut fee_history_entry = FeeHistoryEntry::new(&block); + let percentiles = self.predefined_percentiles(); + + if let Ok(Some((transactions, receipts))) = + self.eth_cache.get_transactions_and_receipts(fee_history_entry.header_hash).await + { + fee_history_entry.rewards = calculate_reward_percentiles_for_block( + &percentiles, + &fee_history_entry, + transactions, + receipts, + ) + .await + .unwrap_or_default(); + + entries.insert(block.number, fee_history_entry); + } else { + break + } + } + while entries.len() > self.config.max_blocks as usize { + entries.pop_first(); + } + if entries.len() == 0 { + self.upper_bound.store(0, SeqCst); + self.lower_bound.store(0, SeqCst); + return + } + let upper_bound = *entries.last_entry().expect("Contains at least one entry").key(); + let lower_bound = *entries.first_entry().expect("Contains at least one entry").key(); + self.upper_bound.store(upper_bound, SeqCst); + self.lower_bound.store(lower_bound, SeqCst); + } + + /// Get UpperBound value for FeeHistoryCache + pub fn upper_bound(&self) -> u64 { + self.upper_bound.load(SeqCst) + } + + /// Get LowerBound value for FeeHistoryCache + pub fn lower_bound(&self) -> u64 { + self.lower_bound.load(SeqCst) + } + + /// Collect fee history for given range. + /// This function retrieves fee history entries from the cache for the specified range. + /// If the requested range (star_block to end_block) is within the cache bounds, + /// it returns the corresponding entries. + /// Otherwise it returns None. + pub async fn get_history( + &self, + start_block: u64, + end_block: u64, + ) -> RethResult> { + let lower_bound = self.lower_bound(); + let upper_bound = self.upper_bound(); + if start_block >= lower_bound && end_block <= upper_bound { + let entries = self.entries.read().await; + let result = entries + .range(start_block..=end_block + 1) + .map(|(_, fee_entry)| fee_entry.clone()) + .collect(); + Ok(result) + } else { + Ok(Vec::new()) + } + } + + /// Generates predefined set of percentiles + pub fn predefined_percentiles(&self) -> Vec { + (0..=100 * self.config.resolution) + .map(|p| p as f64 / self.config.resolution as f64) + .collect() + } +} + +/// Awaits for new chain events and directly inserts them into the cache so they're available +/// immediately before they need to be fetched from disk. +pub async fn fee_history_cache_new_blocks_task( + fee_history_cache: FeeHistoryCache, + mut events: St, + provider: Provider, +) where + St: Stream + Unpin + 'static, + Provider: BlockReaderIdExt + ChainSpecProvider + 'static, +{ + // Init default state + if fee_history_cache.upper_bound() == 0 { + let last_block_number = provider.last_block_number().unwrap_or(0); + + let start_block = if last_block_number > fee_history_cache.config.max_blocks { + last_block_number - fee_history_cache.config.max_blocks + } else { + 0 + }; + + let blocks = provider.block_range(start_block..=last_block_number).unwrap_or_default(); + let sealed = blocks.into_iter().map(|block| block.seal_slow()).collect::>(); + + fee_history_cache.on_new_blocks(sealed.iter()).await; + } + + while let Some(event) = events.next().await { + if let Some(committed) = event.committed() { + // we're only interested in new committed blocks + let (blocks, _) = committed.inner(); + + let blocks = blocks.iter().map(|(_, v)| v.block.clone()).collect::>(); + + fee_history_cache.on_new_blocks(blocks.iter()).await; + } + } +} + +/// Calculates reward percentiles for transactions in a block header. +/// Given a list of percentiles and a sealed block header, this function computes +/// the corresponding rewards for the transactions at each percentile. +/// +/// The results are returned as a vector of U256 values. +async fn calculate_reward_percentiles_for_block( + percentiles: &[f64], + fee_entry: &FeeHistoryEntry, + transactions: Vec, + receipts: Vec, +) -> Result, EthApiError> { + let mut transactions = transactions + .into_iter() + .zip(receipts) + .scan(0, |previous_gas, (tx, receipt)| { + // Convert the cumulative gas used in the receipts + // to the gas usage by the transaction + // + // While we will sum up the gas again later, it is worth + // noting that the order of the transactions will be different, + // so the sum will also be different for each receipt. + let gas_used = receipt.cumulative_gas_used - *previous_gas; + *previous_gas = receipt.cumulative_gas_used; + + Some(TxGasAndReward { + gas_used, + reward: tx + .effective_tip_per_gas(Some(fee_entry.base_fee_per_gas)) + .unwrap_or_default(), + }) + }) + .collect::>(); + + // Sort the transactions by their rewards in ascending order + transactions.sort_by_key(|tx| tx.reward); + + // Find the transaction that corresponds to the given percentile + // + // We use a `tx_index` here that is shared across all percentiles, since we know + // the percentiles are monotonically increasing. + let mut tx_index = 0; + let mut cumulative_gas_used = transactions.first().map(|tx| tx.gas_used).unwrap_or_default(); + let mut rewards_in_block = Vec::new(); + for percentile in percentiles { + // Empty blocks should return in a zero row + if transactions.is_empty() { + rewards_in_block.push(U256::ZERO); + continue + } + + let threshold = (fee_entry.gas_used as f64 * percentile / 100.) as u64; + while cumulative_gas_used < threshold && tx_index < transactions.len() - 1 { + tx_index += 1; + cumulative_gas_used += transactions[tx_index].gas_used; + } + rewards_in_block.push(U256::from(transactions[tx_index].reward)); + } + + Ok(rewards_in_block) +} + +#[derive(Debug, Clone)] +pub struct FeeHistoryEntry { + pub base_fee_per_gas: u64, + pub gas_used_ratio: f64, + pub gas_used: u64, + pub gas_limit: u64, + pub header_hash: B256, + pub rewards: Vec, +} + +impl FeeHistoryEntry { + pub fn new(block: &SealedBlock) -> Self { + FeeHistoryEntry { + base_fee_per_gas: block.base_fee_per_gas.unwrap_or_default(), + gas_used_ratio: block.gas_used as f64 / block.gas_limit as f64, + gas_used: block.gas_used, + header_hash: block.hash, + gas_limit: block.gas_limit, + rewards: Vec::new(), + } + } +} diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 4e056c71b802..ea730b87c4ed 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -13,7 +13,7 @@ use reth_transaction_pool::TransactionPool; use tracing::debug; -use super::FeeHistoryEntry; +use crate::eth::api::fee_history::FeeHistoryEntry; impl EthApi where diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index db32d2403d9c..96830b9f3009 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -2,7 +2,10 @@ //! files. use crate::eth::{ - api::pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}, + api::{ + fee_history::FeeHistoryCache, + pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}, + }, cache::EthStateCache, error::{EthApiError, EthResult}, gas_oracle::GasPriceOracle, @@ -10,36 +13,31 @@ use crate::eth::{ }; use async_trait::async_trait; -use metrics::atomics::AtomicU64; use reth_interfaces::RethResult; use reth_network_api::NetworkInfo; use reth_primitives::{ revm_primitives::{BlockEnv, CfgEnv}, - Address, BlockId, BlockNumberOrTag, ChainInfo, Receipt, SealedBlock, SealedBlockWithSenders, - TransactionSigned, B256, U256, U64, + Address, BlockId, BlockNumberOrTag, ChainInfo, SealedBlockWithSenders, B256, U256, U64, }; -use serde::{Deserialize, Serialize}; use reth_provider::{ - BlockReaderIdExt, CanonStateNotification, ChainSpecProvider, EvmEnvProvider, StateProviderBox, - StateProviderFactory, + BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderBox, StateProviderFactory, }; -use reth_rpc_types::{SyncInfo, SyncStatus, TxGasAndReward}; +use reth_rpc_types::{SyncInfo, SyncStatus}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use reth_transaction_pool::TransactionPool; use std::{ - collections::BTreeMap, fmt::Debug, future::Future, - sync::{atomic::Ordering::SeqCst, Arc}, + sync::Arc, time::{Duration, Instant}, }; -use futures::{Stream, StreamExt}; use tokio::sync::{oneshot, Mutex}; mod block; mod call; +pub(crate) mod fee_history; mod fees; #[cfg(feature = "optimism")] mod optimism; @@ -473,252 +471,3 @@ struct EthApiInner { #[cfg(feature = "optimism")] http_client: reqwest::Client, } - -/// Settings for the [FeeHistoryCache](crate::eth::FeeHistoryCache). -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct FeeHistoryCacheConfig { - /// Max number of blocks in cache. - /// - /// Default is 1024. - pub max_blocks: u64, - /// Percentile approximation resolution - /// - /// Default is 4 which means 0.25 - pub resolution: u64, -} - -impl Default for FeeHistoryCacheConfig { - fn default() -> Self { - FeeHistoryCacheConfig { max_blocks: 1024, resolution: 4 } - } -} - -/// Wrapper struct for BTreeMap -#[derive(Debug, Clone)] -pub struct FeeHistoryCache { - lower_bound: Arc, - upper_bound: Arc, - config: FeeHistoryCacheConfig, - entries: Arc>>, - eth_cache: EthStateCache, -} - -impl FeeHistoryCache { - /// Creates new FeeHistoryCache instance, initialize it with the mose recent data, set bounds - pub fn new(eth_cache: EthStateCache, config: FeeHistoryCacheConfig) -> Self { - let init_tree_map = BTreeMap::new(); - - let entries = Arc::new(tokio::sync::RwLock::new(init_tree_map)); - - let upper_bound = Arc::new(AtomicU64::new(0)); - let lower_bound = Arc::new(AtomicU64::new(0)); - - FeeHistoryCache { config, entries, upper_bound, lower_bound, eth_cache } - } - - /// Processing of the arriving blocks - pub async fn on_new_blocks<'a, I>(&self, blocks: I) - where - I: Iterator, - { - let mut entries = self.entries.write().await; - - for block in blocks { - let mut fee_history_entry = FeeHistoryEntry::new(&block); - let percentiles = self.predefined_percentiles(); - - if let Ok(Some((transactions, receipts))) = - self.eth_cache.get_transactions_and_receipts(fee_history_entry.header_hash).await - { - fee_history_entry.rewards = calculate_reward_percentiles_for_block( - &percentiles, - &fee_history_entry, - transactions, - receipts, - ) - .await - .unwrap_or_default(); - - entries.insert(block.number, fee_history_entry); - } else { - break - } - } - while entries.len() > self.config.max_blocks as usize { - entries.pop_first(); - } - if entries.len() == 0 { - self.upper_bound.store(0, SeqCst); - self.lower_bound.store(0, SeqCst); - return - } - let upper_bound = *entries.last_entry().expect("Contains at least one entry").key(); - let lower_bound = *entries.first_entry().expect("Contains at least one entry").key(); - self.upper_bound.store(upper_bound, SeqCst); - self.lower_bound.store(lower_bound, SeqCst); - } - - /// Get UpperBound value for FeeHistoryCache - pub fn upper_bound(&self) -> u64 { - self.upper_bound.load(SeqCst) - } - - /// Get LowerBound value for FeeHistoryCache - pub fn lower_bound(&self) -> u64 { - self.lower_bound.load(SeqCst) - } - - /// Collect fee history for given range. - /// This function retrieves fee history entries from the cache for the specified range. - /// If the requested range (star_block to end_block) is within the cache bounds, - /// it returns the corresponding entries. - /// Otherwise it returns None. - pub async fn get_history( - &self, - start_block: u64, - end_block: u64, - ) -> RethResult> { - let lower_bound = self.lower_bound(); - let upper_bound = self.upper_bound(); - if start_block >= lower_bound && end_block <= upper_bound { - let entries = self.entries.read().await; - let result = entries - .range(start_block..=end_block + 1) - .map(|(_, fee_entry)| fee_entry.clone()) - .collect(); - Ok(result) - } else { - Ok(Vec::new()) - } - } - - /// Generates predefined set of percentiles - pub fn predefined_percentiles(&self) -> Vec { - (0..=100 * self.config.resolution) - .map(|p| p as f64 / self.config.resolution as f64) - .collect() - } -} - -/// Awaits for new chain events and directly inserts them into the cache so they're available -/// immediately before they need to be fetched from disk. -pub async fn fee_history_cache_new_blocks_task( - fee_history_cache: FeeHistoryCache, - mut events: St, - provider: Provider, -) where - St: Stream + Unpin + 'static, - Provider: BlockReaderIdExt + ChainSpecProvider + 'static, -{ - // Init default state - if fee_history_cache.upper_bound() == 0 { - let last_block_number = provider.last_block_number().unwrap_or(0); - - let start_block = if last_block_number > fee_history_cache.config.max_blocks { - last_block_number - fee_history_cache.config.max_blocks - } else { - 0 - }; - - let blocks = provider.block_range(start_block..=last_block_number).unwrap_or_default(); - let sealed = blocks.into_iter().map(|block| block.seal_slow()).collect::>(); - - fee_history_cache.on_new_blocks(sealed.iter()).await; - } - - while let Some(event) = events.next().await { - if let Some(committed) = event.committed() { - // we're only interested in new committed blocks - let (blocks, _) = committed.inner(); - - let blocks = blocks.iter().map(|(_, v)| v.block.clone()).collect::>(); - - fee_history_cache.on_new_blocks(blocks.iter()).await; - } - } -} - -/// Calculates reward percentiles for transactions in a block header. -/// Given a list of percentiles and a sealed block header, this function computes -/// the corresponding rewards for the transactions at each percentile. -/// -/// The results are returned as a vector of U256 values. -async fn calculate_reward_percentiles_for_block( - percentiles: &[f64], - fee_entry: &FeeHistoryEntry, - transactions: Vec, - receipts: Vec, -) -> Result, EthApiError> { - let mut transactions = transactions - .into_iter() - .zip(receipts) - .scan(0, |previous_gas, (tx, receipt)| { - // Convert the cumulative gas used in the receipts - // to the gas usage by the transaction - // - // While we will sum up the gas again later, it is worth - // noting that the order of the transactions will be different, - // so the sum will also be different for each receipt. - let gas_used = receipt.cumulative_gas_used - *previous_gas; - *previous_gas = receipt.cumulative_gas_used; - - Some(TxGasAndReward { - gas_used, - reward: tx - .effective_tip_per_gas(Some(fee_entry.base_fee_per_gas)) - .unwrap_or_default(), - }) - }) - .collect::>(); - - // Sort the transactions by their rewards in ascending order - transactions.sort_by_key(|tx| tx.reward); - - // Find the transaction that corresponds to the given percentile - // - // We use a `tx_index` here that is shared across all percentiles, since we know - // the percentiles are monotonically increasing. - let mut tx_index = 0; - let mut cumulative_gas_used = transactions.first().map(|tx| tx.gas_used).unwrap_or_default(); - let mut rewards_in_block = Vec::new(); - for percentile in percentiles { - // Empty blocks should return in a zero row - if transactions.is_empty() { - rewards_in_block.push(U256::ZERO); - continue - } - - let threshold = (fee_entry.gas_used as f64 * percentile / 100.) as u64; - while cumulative_gas_used < threshold && tx_index < transactions.len() - 1 { - tx_index += 1; - cumulative_gas_used += transactions[tx_index].gas_used; - } - rewards_in_block.push(U256::from(transactions[tx_index].reward)); - } - - Ok(rewards_in_block) -} - -#[derive(Debug, Clone)] -pub struct FeeHistoryEntry { - base_fee_per_gas: u64, - gas_used_ratio: f64, - gas_used: u64, - gas_limit: u64, - header_hash: B256, - rewards: Vec, -} - -impl FeeHistoryEntry { - fn new(block: &SealedBlock) -> Self { - FeeHistoryEntry { - base_fee_per_gas: block.base_fee_per_gas.unwrap_or_default(), - gas_used_ratio: block.gas_used as f64 / block.gas_limit as f64, - gas_used: block.gas_used, - header_hash: block.hash, - gas_limit: block.gas_limit, - rewards: Vec::new(), - } - } -} diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index 30c4b39f68e0..97f57af68855 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -14,9 +14,10 @@ mod signer; pub(crate) mod utils; pub use api::{ - fee_history_cache_new_blocks_task, EthApi, EthApiSpec, EthTransactions, FeeHistoryCache, - FeeHistoryCacheConfig, TransactionSource, RPC_DEFAULT_GAS_CAP, + fee_history::{fee_history_cache_new_blocks_task, FeeHistoryCache, FeeHistoryCacheConfig}, + EthApi, EthApiSpec, EthTransactions, TransactionSource, RPC_DEFAULT_GAS_CAP, }; + pub use bundle::EthBundle; pub use filter::{EthFilter, EthFilterConfig}; pub use id_provider::EthSubscriptionIdProvider; From 83f9c889935153e07f6158a81714bb50f00a6ea7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 24 Nov 2023 17:01:37 +0100 Subject: [PATCH 28/32] chore: touchups --- Cargo.lock | 1 + crates/rpc/rpc/src/eth/api/fee_history.rs | 32 +++++++++++++++-------- crates/rpc/rpc/src/eth/api/fees.rs | 4 +-- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2fc9c302f0cf..1e0f220a0715 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6378,6 +6378,7 @@ dependencies = [ "reth-rpc-api", "reth-rpc-types", "serde_json", + "similar-asserts", "tokio", ] diff --git a/crates/rpc/rpc/src/eth/api/fee_history.rs b/crates/rpc/rpc/src/eth/api/fee_history.rs index 9feab7b4af7c..890bb6542c03 100644 --- a/crates/rpc/rpc/src/eth/api/fee_history.rs +++ b/crates/rpc/rpc/src/eth/api/fee_history.rs @@ -2,22 +2,20 @@ use crate::eth::{cache::EthStateCache, error::EthApiError}; +use futures::{Stream, StreamExt}; use metrics::atomics::AtomicU64; use reth_interfaces::RethResult; use reth_primitives::{Receipt, SealedBlock, TransactionSigned, B256, U256}; -use serde::{Deserialize, Serialize}; - use reth_provider::{BlockReaderIdExt, CanonStateNotification, ChainSpecProvider}; use reth_rpc_types::TxGasAndReward; +use serde::{Deserialize, Serialize}; use std::{ collections::BTreeMap, fmt::Debug, sync::{atomic::Ordering::SeqCst, Arc}, }; -use futures::{Stream, StreamExt}; - -/// Settings for the [FeeHistoryCache](crate::eth::FeeHistoryCache). +/// Settings for the [FeeHistoryCache]. #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct FeeHistoryCacheConfig { @@ -40,12 +38,12 @@ impl Default for FeeHistoryCacheConfig { /// Wrapper struct for BTreeMap #[derive(Debug, Clone)] pub struct FeeHistoryCache { + /// Stores the lower bound of the cache lower_bound: Arc, upper_bound: Arc, - /// Config for FeeHistoryCache, consists of resolution for percentile approximation /// and max number of blocks - pub config: FeeHistoryCacheConfig, + config: FeeHistoryCacheConfig, entries: Arc>>, eth_cache: EthStateCache, } @@ -63,6 +61,17 @@ impl FeeHistoryCache { FeeHistoryCache { config, entries, upper_bound, lower_bound, eth_cache } } + /// How the cache is configured. + pub fn config(&self) -> &FeeHistoryCacheConfig { + &self.config + } + + /// Returns the configured resolution for percentile approximation. + #[inline] + pub fn resolution(&self) -> u64 { + self.config.resolution + } + /// Processing of the arriving blocks pub async fn on_new_blocks<'a, I>(&self, blocks: I) where @@ -71,7 +80,7 @@ impl FeeHistoryCache { let mut entries = self.entries.write().await; for block in blocks { - let mut fee_history_entry = FeeHistoryEntry::new(&block); + let mut fee_history_entry = FeeHistoryEntry::new(block); let percentiles = self.predefined_percentiles(); if let Ok(Some((transactions, receipts))) = @@ -140,10 +149,11 @@ impl FeeHistoryCache { } /// Generates predefined set of percentiles + /// + /// This returns 100 * resolution points pub fn predefined_percentiles(&self) -> Vec { - (0..=100 * self.config.resolution) - .map(|p| p as f64 / self.config.resolution as f64) - .collect() + let res = self.resolution() as f64; + (0..=100 * self.resolution()).map(|p| p as f64 / res).collect() } } diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index ea730b87c4ed..f97c7b35ef79 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -127,7 +127,7 @@ where base_fee_per_gas.push(U256::try_from(entry.base_fee_per_gas).unwrap()); gas_used_ratio.push(entry.gas_used_ratio); - if fee_cache_flag == true { + if fee_cache_flag { if let Some(percentiles) = &reward_percentiles { let mut block_rewards = Vec::new(); @@ -166,7 +166,7 @@ where /// Approximates reward at a given percentile for a specific block /// Based on the configured resolution fn approximate_percentile(&self, entry: &FeeHistoryEntry, requested_percentile: f64) -> U256 { - let resolution = self.fee_history_cache().config.resolution; + let resolution = self.fee_history_cache().resolution(); let rounded_percentile = (requested_percentile * resolution as f64).round() / resolution as f64; let clamped_percentile = rounded_percentile.clamp(0.0, 100.0); From 3cbd4638480cb1bbda2beb79ffbea16aad623f07 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 24 Nov 2023 17:30:47 +0100 Subject: [PATCH 29/32] refactor --- crates/rpc/rpc/src/eth/api/fee_history.rs | 24 ++--- crates/rpc/rpc/src/eth/api/fees.rs | 109 ++++++++++++++-------- 2 files changed, 84 insertions(+), 49 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fee_history.rs b/crates/rpc/rpc/src/eth/api/fee_history.rs index 890bb6542c03..543df661af78 100644 --- a/crates/rpc/rpc/src/eth/api/fee_history.rs +++ b/crates/rpc/rpc/src/eth/api/fee_history.rs @@ -88,11 +88,11 @@ impl FeeHistoryCache { { fee_history_entry.rewards = calculate_reward_percentiles_for_block( &percentiles, - &fee_history_entry, + fee_history_entry.gas_used, + fee_history_entry.base_fee_per_gas, transactions, receipts, ) - .await .unwrap_or_default(); entries.insert(block.number, fee_history_entry); @@ -100,14 +100,17 @@ impl FeeHistoryCache { break } } + while entries.len() > self.config.max_blocks as usize { entries.pop_first(); } + if entries.len() == 0 { self.upper_bound.store(0, SeqCst); self.lower_bound.store(0, SeqCst); return } + let upper_bound = *entries.last_entry().expect("Contains at least one entry").key(); let lower_bound = *entries.first_entry().expect("Contains at least one entry").key(); self.upper_bound.store(upper_bound, SeqCst); @@ -133,7 +136,7 @@ impl FeeHistoryCache { &self, start_block: u64, end_block: u64, - ) -> RethResult> { + ) -> RethResult>> { let lower_bound = self.lower_bound(); let upper_bound = self.upper_bound(); if start_block >= lower_bound && end_block <= upper_bound { @@ -142,9 +145,9 @@ impl FeeHistoryCache { .range(start_block..=end_block + 1) .map(|(_, fee_entry)| fee_entry.clone()) .collect(); - Ok(result) + Ok(Some(result)) } else { - Ok(Vec::new()) + Ok(None) } } @@ -200,9 +203,10 @@ pub async fn fee_history_cache_new_blocks_task( /// the corresponding rewards for the transactions at each percentile. /// /// The results are returned as a vector of U256 values. -async fn calculate_reward_percentiles_for_block( +pub(crate) fn calculate_reward_percentiles_for_block( percentiles: &[f64], - fee_entry: &FeeHistoryEntry, + gas_used: u64, + base_fee_per_gas: u64, transactions: Vec, receipts: Vec, ) -> Result, EthApiError> { @@ -221,9 +225,7 @@ async fn calculate_reward_percentiles_for_block( Some(TxGasAndReward { gas_used, - reward: tx - .effective_tip_per_gas(Some(fee_entry.base_fee_per_gas)) - .unwrap_or_default(), + reward: tx.effective_tip_per_gas(Some(base_fee_per_gas)).unwrap_or_default(), }) }) .collect::>(); @@ -245,7 +247,7 @@ async fn calculate_reward_percentiles_for_block( continue } - let threshold = (fee_entry.gas_used as f64 * percentile / 100.) as u64; + let threshold = (gas_used as f64 * percentile / 100.) as u64; while cumulative_gas_used < threshold && tx_index < transactions.len() - 1 { tx_index += 1; cumulative_gas_used += transactions[tx_index].gas_used; diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index f97c7b35ef79..baaa2713021f 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -13,7 +13,7 @@ use reth_transaction_pool::TransactionPool; use tracing::debug; -use crate::eth::api::fee_history::FeeHistoryEntry; +use crate::eth::api::fee_history::{calculate_reward_percentiles_for_block, FeeHistoryEntry}; impl EthApi where @@ -47,8 +47,10 @@ where self.gas_oracle().suggest_tip_cap().await } - /// Reports the fee history, for the given amount of blocks, up until the newest block - /// provided. + /// Reports the fee history, for the given amount of blocks, up until the given newest block. + /// + /// If `reward_percentiles` are provided the [FeeHistory] will include the _approximated_ + /// rewards for the requested range. pub(crate) async fn fee_history( &self, mut block_count: u64, @@ -101,65 +103,96 @@ where // Treat a request for 1 block as a request for `newest_block..=newest_block`, // otherwise `newest_block - 2 // SAFETY: We ensured that block count is capped - let start_block = end_block_plus - block_count; - let mut fee_entries = self.fee_history_cache().get_history(start_block, end_block).await?; - - let mut fee_cache_flag = true; - if fee_entries.is_empty() { - for block in self.provider().block_range(start_block..=end_block)?.into_iter() { - let entry = FeeHistoryEntry::new(&block.seal_slow()); - fee_entries.push(entry); - } - fee_cache_flag = false; - } + // Check if the requested range is within the cache bounds + let fee_entries = self.fee_history_cache().get_history(start_block, end_block).await?; - if fee_entries.len() != block_count as usize { - return Err(EthApiError::InvalidBlockRange) - } // Collect base fees, gas usage ratios and (optionally) reward percentile data let mut base_fee_per_gas: Vec = Vec::new(); let mut gas_used_ratio: Vec = Vec::new(); let mut rewards: Vec> = Vec::new(); - for entry in &fee_entries { - base_fee_per_gas.push(U256::try_from(entry.base_fee_per_gas).unwrap()); - gas_used_ratio.push(entry.gas_used_ratio); + if let Some(fee_entries) = fee_entries { + if fee_entries.len() != block_count as usize { + return Err(EthApiError::InvalidBlockRange) + } + + for entry in &fee_entries { + base_fee_per_gas.push(U256::from(entry.base_fee_per_gas)); + gas_used_ratio.push(entry.gas_used_ratio); - if fee_cache_flag { if let Some(percentiles) = &reward_percentiles { - let mut block_rewards = Vec::new(); - + let mut block_rewards = Vec::with_capacity(percentiles.len()); for &percentile in percentiles.iter() { block_rewards.push(self.approximate_percentile(entry, percentile)); } rewards.push(block_rewards); } } - } - - // The spec states that `base_fee_per_gas` "[..] includes the next block after the newest of - // the returned range, because this value can be derived from the newest block" - // - // The unwrap is safe since we checked earlier that we got at least 1 header. + let last_entry = fee_entries.last().unwrap(); + base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( + last_entry.gas_used, + last_entry.gas_limit, + last_entry.base_fee_per_gas, + self.provider().chain_spec().base_fee_params, + ))); + } else { + // read the requested header range + let headers = self.provider().sealed_headers_range(start_block..=end_block)?; + if headers.len() != block_count as usize { + return Err(EthApiError::InvalidBlockRange) + } - let last_entry = fee_entries.last().unwrap(); + for header in &headers { + base_fee_per_gas.push(U256::from(header.base_fee_per_gas.unwrap_or_default())); + gas_used_ratio.push(header.gas_used as f64 / header.gas_limit as f64); - let chain_spec = self.provider().chain_spec(); + // Percentiles were specified, so we need to collect reward percentile ino + if let Some(percentiles) = &reward_percentiles { + let (transactions, receipts) = self + .cache() + .get_transactions_and_receipts(header.hash) + .await? + .ok_or(EthApiError::InvalidBlockRange)?; + rewards.push( + calculate_reward_percentiles_for_block( + percentiles, + header.gas_used, + header.base_fee_per_gas.unwrap_or_default(), + transactions, + receipts, + ) + .unwrap_or_default(), + ); + } + } - base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( - last_entry.gas_used, - last_entry.gas_limit, - last_entry.base_fee_per_gas, - chain_spec.base_fee_params, - ))); + // The spec states that `base_fee_per_gas` "[..] includes the next block after the + // newest of the returned range, because this value can be derived from the + // newest block" + // + // The unwrap is safe since we checked earlier that we got at least 1 header. + + // The spec states that `base_fee_per_gas` "[..] includes the next block after the + // newest of the returned range, because this value can be derived from the + // newest block" + // + // The unwrap is safe since we checked earlier that we got at least 1 header. + let last_header = headers.last().unwrap(); + base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( + last_header.gas_used, + last_header.gas_limit, + last_header.base_fee_per_gas.unwrap_or_default(), + self.provider().chain_spec().base_fee_params, + ))); + }; Ok(FeeHistory { base_fee_per_gas, gas_used_ratio, oldest_block: U256::from(start_block), - reward: if rewards.is_empty() { None } else { Some(rewards) }, + reward: reward_percentiles.map(|_| rewards), }) } From 7dc0e835bb24fd0c967d2effd008bf741cbeb60d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 24 Nov 2023 17:38:51 +0100 Subject: [PATCH 30/32] wip: caching --- crates/rpc/rpc/src/eth/api/fee_history.rs | 24 +++++++++++------------ crates/rpc/rpc/src/eth/api/fees.rs | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fee_history.rs b/crates/rpc/rpc/src/eth/api/fee_history.rs index 543df661af78..3edfbf008c7c 100644 --- a/crates/rpc/rpc/src/eth/api/fee_history.rs +++ b/crates/rpc/rpc/src/eth/api/fee_history.rs @@ -6,7 +6,7 @@ use futures::{Stream, StreamExt}; use metrics::atomics::AtomicU64; use reth_interfaces::RethResult; use reth_primitives::{Receipt, SealedBlock, TransactionSigned, B256, U256}; -use reth_provider::{BlockReaderIdExt, CanonStateNotification, ChainSpecProvider}; +use reth_provider::{BlockReaderIdExt, CanonStateNotification, Chain, ChainSpecProvider}; use reth_rpc_types::TxGasAndReward; use serde::{Deserialize, Serialize}; use std::{ @@ -73,12 +73,16 @@ impl FeeHistoryCache { } /// Processing of the arriving blocks - pub async fn on_new_blocks<'a, I>(&self, blocks: I) + async fn on_new_chain<'a, I>(&self, chain: Arc) where I: Iterator, { let mut entries = self.entries.write().await; + // we're only interested in new committed blocks + let (blocks, state) = chain.inner(); + // TODO: need iterator + for block in blocks { let mut fee_history_entry = FeeHistoryEntry::new(block); let percentiles = self.predefined_percentiles(); @@ -101,6 +105,7 @@ impl FeeHistoryCache { } } + // enforce bounds while entries.len() > self.config.max_blocks as usize { entries.pop_first(); } @@ -180,20 +185,15 @@ pub async fn fee_history_cache_new_blocks_task( 0 }; - let blocks = provider.block_range(start_block..=last_block_number).unwrap_or_default(); - let sealed = blocks.into_iter().map(|block| block.seal_slow()).collect::>(); - - fee_history_cache.on_new_blocks(sealed.iter()).await; + // let blocks = provider.block_range(start_block..=last_block_number).unwrap_or_default(); + // let sealed = blocks.into_iter().map(|block| block.seal_slow()).collect::>(); + // + // fee_history_cache.on_new_chain(sealed.iter()).await; } while let Some(event) = events.next().await { if let Some(committed) = event.committed() { - // we're only interested in new committed blocks - let (blocks, _) = committed.inner(); - - let blocks = blocks.iter().map(|(_, v)| v.block.clone()).collect::>(); - - fee_history_cache.on_new_blocks(blocks.iter()).await; + fee_history_cache.on_new_chain(committed).await; } } } diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index baaa2713021f..01bea86e35e2 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -130,7 +130,7 @@ where rewards.push(block_rewards); } } - let last_entry = fee_entries.last().unwrap(); + let last_entry = fee_entries.last().expect("is present"); base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( last_entry.gas_used, last_entry.gas_limit, @@ -179,7 +179,7 @@ where // newest block" // // The unwrap is safe since we checked earlier that we got at least 1 header. - let last_header = headers.last().unwrap(); + let last_header = headers.last().expect("is present"); base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( last_header.gas_used, last_header.gas_limit, From 87bd4292279e4c6a17da9b05242ee9d4e17f03a7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 27 Nov 2023 10:59:52 +0100 Subject: [PATCH 31/32] cleanup --- crates/rpc/rpc/src/eth/api/fee_history.rs | 110 +++++++++++----------- crates/rpc/rpc/src/eth/api/fees.rs | 2 +- crates/rpc/rpc/src/eth/gas_oracle.rs | 11 ++- 3 files changed, 65 insertions(+), 58 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fee_history.rs b/crates/rpc/rpc/src/eth/api/fee_history.rs index 3edfbf008c7c..2e834256732a 100644 --- a/crates/rpc/rpc/src/eth/api/fee_history.rs +++ b/crates/rpc/rpc/src/eth/api/fee_history.rs @@ -1,12 +1,10 @@ //! Consist of types adjacent to the fee history cache and its configs -use crate::eth::{cache::EthStateCache, error::EthApiError}; - +use crate::eth::{cache::EthStateCache, error::EthApiError, gas_oracle::MAX_HEADER_HISTORY}; use futures::{Stream, StreamExt}; use metrics::atomics::AtomicU64; -use reth_interfaces::RethResult; use reth_primitives::{Receipt, SealedBlock, TransactionSigned, B256, U256}; -use reth_provider::{BlockReaderIdExt, CanonStateNotification, Chain, ChainSpecProvider}; +use reth_provider::{BlockReaderIdExt, CanonStateNotification, ChainSpecProvider}; use reth_rpc_types::TxGasAndReward; use serde::{Deserialize, Serialize}; use std::{ @@ -21,7 +19,8 @@ use std::{ pub struct FeeHistoryCacheConfig { /// Max number of blocks in cache. /// - /// Default is 1024. + /// Default is [MAX_HEADER_HISTORY] plus some change to also serve slightly older blocks from + /// cache, since fee_history supports the entire range pub max_blocks: u64, /// Percentile approximation resolution /// @@ -31,7 +30,7 @@ pub struct FeeHistoryCacheConfig { impl Default for FeeHistoryCacheConfig { fn default() -> Self { - FeeHistoryCacheConfig { max_blocks: 1024, resolution: 4 } + FeeHistoryCacheConfig { max_blocks: MAX_HEADER_HISTORY + 100, resolution: 4 } } } @@ -73,39 +72,29 @@ impl FeeHistoryCache { } /// Processing of the arriving blocks - async fn on_new_chain<'a, I>(&self, chain: Arc) + async fn insert_blocks(&self, blocks: I) where - I: Iterator, + I: Iterator, Vec)>, { let mut entries = self.entries.write().await; - // we're only interested in new committed blocks - let (blocks, state) = chain.inner(); - // TODO: need iterator - - for block in blocks { - let mut fee_history_entry = FeeHistoryEntry::new(block); - let percentiles = self.predefined_percentiles(); - - if let Ok(Some((transactions, receipts))) = - self.eth_cache.get_transactions_and_receipts(fee_history_entry.header_hash).await - { - fee_history_entry.rewards = calculate_reward_percentiles_for_block( - &percentiles, - fee_history_entry.gas_used, - fee_history_entry.base_fee_per_gas, - transactions, - receipts, - ) - .unwrap_or_default(); - - entries.insert(block.number, fee_history_entry); - } else { - break - } + let percentiles = self.predefined_percentiles(); + // Insert all new blocks and calculate approximated rewards + for (block, transactions, receipts) in blocks { + let mut fee_history_entry = FeeHistoryEntry::new(&block); + + fee_history_entry.rewards = calculate_reward_percentiles_for_block( + &percentiles, + fee_history_entry.gas_used, + fee_history_entry.base_fee_per_gas, + transactions, + receipts, + ) + .unwrap_or_default(); + entries.insert(block.number, fee_history_entry); } - // enforce bounds + // enforce bounds by popping the oldest entries while entries.len() > self.config.max_blocks as usize { entries.pop_first(); } @@ -141,7 +130,7 @@ impl FeeHistoryCache { &self, start_block: u64, end_block: u64, - ) -> RethResult>> { + ) -> Option> { let lower_bound = self.lower_bound(); let upper_bound = self.upper_bound(); if start_block >= lower_bound && end_block <= upper_bound { @@ -149,10 +138,15 @@ impl FeeHistoryCache { let result = entries .range(start_block..=end_block + 1) .map(|(_, fee_entry)| fee_entry.clone()) - .collect(); - Ok(Some(result)) + .collect::>(); + + if result.is_empty() { + return None + } + + Some(result) } else { - Ok(None) + None } } @@ -175,25 +169,25 @@ pub async fn fee_history_cache_new_blocks_task( St: Stream + Unpin + 'static, Provider: BlockReaderIdExt + ChainSpecProvider + 'static, { - // Init default state - if fee_history_cache.upper_bound() == 0 { - let last_block_number = provider.last_block_number().unwrap_or(0); - - let start_block = if last_block_number > fee_history_cache.config.max_blocks { - last_block_number - fee_history_cache.config.max_blocks - } else { - 0 - }; - - // let blocks = provider.block_range(start_block..=last_block_number).unwrap_or_default(); - // let sealed = blocks.into_iter().map(|block| block.seal_slow()).collect::>(); - // - // fee_history_cache.on_new_chain(sealed.iter()).await; - } + // // Init default state + // if fee_history_cache.upper_bound() == 0 { + // let last_block_number = provider.last_block_number().unwrap_or(0); + // + // let start_block = if last_block_number > fee_history_cache.config.max_blocks { + // last_block_number - fee_history_cache.config.max_blocks + // } else { + // 0 + // }; + // + // // let blocks = + // provider.block_range(start_block..=last_block_number).unwrap_or_default(); // let + // sealed = blocks.into_iter().map(|block| block.seal_slow()).collect::>(); // + // // fee_history_cache.on_new_chain(sealed.iter()).await; + // } while let Some(event) = events.next().await { if let Some(committed) = event.committed() { - fee_history_cache.on_new_chain(committed).await; + // fee_history_cache.insert_blocks(committed).await; } } } @@ -258,17 +252,27 @@ pub(crate) fn calculate_reward_percentiles_for_block( Ok(rewards_in_block) } +/// A cached entry for a block's fee history. #[derive(Debug, Clone)] pub struct FeeHistoryEntry { + /// The base fee per gas for this block. pub base_fee_per_gas: u64, + /// Gas used ratio this block. pub gas_used_ratio: f64, + /// Gas used by this block. pub gas_used: u64, + /// Gas limit by this block. pub gas_limit: u64, + /// Hash of the block. pub header_hash: B256, + /// Approximated rewards for the configured percentiles. pub rewards: Vec, } impl FeeHistoryEntry { + /// Creates a new entry from a sealed block. + /// + /// Note: This does not calculate the rewards for the block. pub fn new(block: &SealedBlock) -> Self { FeeHistoryEntry { base_fee_per_gas: block.base_fee_per_gas.unwrap_or_default(), diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 01bea86e35e2..9110a4cd4c3c 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -106,7 +106,7 @@ where let start_block = end_block_plus - block_count; // Check if the requested range is within the cache bounds - let fee_entries = self.fee_history_cache().get_history(start_block, end_block).await?; + let fee_entries = self.fee_history_cache().get_history(start_block, end_block).await; // Collect base fees, gas usage ratios and (optionally) reward percentile data let mut base_fee_per_gas: Vec = Vec::new(); diff --git a/crates/rpc/rpc/src/eth/gas_oracle.rs b/crates/rpc/rpc/src/eth/gas_oracle.rs index f3afc3ff7be3..4eaaa5cdcfc5 100644 --- a/crates/rpc/rpc/src/eth/gas_oracle.rs +++ b/crates/rpc/rpc/src/eth/gas_oracle.rs @@ -16,6 +16,9 @@ use tracing::warn; /// The number of transactions sampled in a block pub const SAMPLE_NUMBER: usize = 3_usize; +/// The default maximum number of blocks to use for the gas price oracle. +pub const MAX_HEADER_HISTORY: u64 = 1024; + /// The default maximum gas price to use for the estimate pub const DEFAULT_MAX_PRICE: U256 = U256::from_limbs([500_000_000_000u64, 0, 0, 0]); @@ -53,8 +56,8 @@ impl Default for GasPriceOracleConfig { GasPriceOracleConfig { blocks: 20, percentile: 60, - max_header_history: 1024, - max_block_history: 1024, + max_header_history: MAX_HEADER_HISTORY, + max_block_history: MAX_HEADER_HISTORY, default: None, max_price: Some(DEFAULT_MAX_PRICE), ignore_price: Some(DEFAULT_IGNORE_PRICE), @@ -73,8 +76,8 @@ impl GasPriceOracleConfig { Self { blocks: blocks.unwrap_or(20), percentile: percentile.unwrap_or(60), - max_header_history: 1024, - max_block_history: 1024, + max_header_history: MAX_HEADER_HISTORY, + max_block_history: MAX_HEADER_HISTORY, default: None, max_price: max_price.map(U256::from).or(Some(DEFAULT_MAX_PRICE)), ignore_price: ignore_price.map(U256::from).or(Some(DEFAULT_IGNORE_PRICE)), From ef27a2f3b9fdbb0b10e12b9dcf2861be9e2051fa Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 27 Nov 2023 11:33:13 +0100 Subject: [PATCH 32/32] cleanuo --- crates/rpc/rpc-builder/src/lib.rs | 4 ++ crates/rpc/rpc/src/eth/api/fee_history.rs | 46 ++++++++++------------- crates/rpc/rpc/src/eth/api/fees.rs | 21 +++++------ 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index e13fb0424a52..b2c7ff3af966 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1154,6 +1154,10 @@ where } /// Creates the [EthHandlers] type the first time this is called. + /// + /// This will spawn the required service tasks for [EthApi] for: + /// - [EthStateCache] + /// - [FeeHistoryCache] fn with_eth(&mut self, f: F) -> R where F: FnOnce(&EthHandlers) -> R, diff --git a/crates/rpc/rpc/src/eth/api/fee_history.rs b/crates/rpc/rpc/src/eth/api/fee_history.rs index 2e834256732a..f25ecc6e00e4 100644 --- a/crates/rpc/rpc/src/eth/api/fee_history.rs +++ b/crates/rpc/rpc/src/eth/api/fee_history.rs @@ -43,7 +43,9 @@ pub struct FeeHistoryCache { /// Config for FeeHistoryCache, consists of resolution for percentile approximation /// and max number of blocks config: FeeHistoryCacheConfig, + /// Stores the entries of the cache entries: Arc>>, + #[allow(unused)] eth_cache: EthStateCache, } @@ -74,21 +76,20 @@ impl FeeHistoryCache { /// Processing of the arriving blocks async fn insert_blocks(&self, blocks: I) where - I: Iterator, Vec)>, + I: Iterator)>, { let mut entries = self.entries.write().await; let percentiles = self.predefined_percentiles(); // Insert all new blocks and calculate approximated rewards - for (block, transactions, receipts) in blocks { + for (block, receipts) in blocks { let mut fee_history_entry = FeeHistoryEntry::new(&block); - fee_history_entry.rewards = calculate_reward_percentiles_for_block( &percentiles, fee_history_entry.gas_used, fee_history_entry.base_fee_per_gas, - transactions, - receipts, + &block.body, + &receipts, ) .unwrap_or_default(); entries.insert(block.number, fee_history_entry); @@ -122,8 +123,9 @@ impl FeeHistoryCache { } /// Collect fee history for given range. + /// /// This function retrieves fee history entries from the cache for the specified range. - /// If the requested range (star_block to end_block) is within the cache bounds, + /// If the requested range (start_block to end_block) is within the cache bounds, /// it returns the corresponding entries. /// Otherwise it returns None. pub async fn get_history( @@ -164,30 +166,22 @@ impl FeeHistoryCache { pub async fn fee_history_cache_new_blocks_task( fee_history_cache: FeeHistoryCache, mut events: St, - provider: Provider, + _provider: Provider, ) where St: Stream + Unpin + 'static, Provider: BlockReaderIdExt + ChainSpecProvider + 'static, { - // // Init default state - // if fee_history_cache.upper_bound() == 0 { - // let last_block_number = provider.last_block_number().unwrap_or(0); - // - // let start_block = if last_block_number > fee_history_cache.config.max_blocks { - // last_block_number - fee_history_cache.config.max_blocks - // } else { - // 0 - // }; - // - // // let blocks = - // provider.block_range(start_block..=last_block_number).unwrap_or_default(); // let - // sealed = blocks.into_iter().map(|block| block.seal_slow()).collect::>(); // - // // fee_history_cache.on_new_chain(sealed.iter()).await; - // } + // TODO: keep track of gaps in the chain and fill them from disk while let Some(event) = events.next().await { if let Some(committed) = event.committed() { - // fee_history_cache.insert_blocks(committed).await; + let (blocks, receipts): (Vec<_>, Vec<_>) = committed + .blocks_and_receipts() + .map(|(block, receipts)| { + (block.block.clone(), receipts.iter().flatten().cloned().collect::>()) + }) + .unzip(); + fee_history_cache.insert_blocks(blocks.into_iter().zip(receipts)).await; } } } @@ -201,11 +195,11 @@ pub(crate) fn calculate_reward_percentiles_for_block( percentiles: &[f64], gas_used: u64, base_fee_per_gas: u64, - transactions: Vec, - receipts: Vec, + transactions: &[TransactionSigned], + receipts: &[Receipt], ) -> Result, EthApiError> { let mut transactions = transactions - .into_iter() + .iter() .zip(receipts) .scan(0, |previous_gas, (tx, receipt)| { // Convert the cumulative gas used in the receipts diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index 9110a4cd4c3c..481a8d01a389 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -1,20 +1,19 @@ //! Contains RPC handler implementations for fee history. use crate::{ - eth::error::{EthApiError, EthResult}, + eth::{ + api::fee_history::{calculate_reward_percentiles_for_block, FeeHistoryEntry}, + error::{EthApiError, EthResult}, + }, EthApi, }; - use reth_network_api::NetworkInfo; use reth_primitives::{basefee::calculate_next_block_base_fee, BlockNumberOrTag, U256}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::FeeHistory; use reth_transaction_pool::TransactionPool; - use tracing::debug; -use crate::eth::api::fee_history::{calculate_reward_percentiles_for_block, FeeHistoryEntry}; - impl EthApi where Pool: TransactionPool + Clone + 'static, @@ -105,14 +104,14 @@ where // SAFETY: We ensured that block count is capped let start_block = end_block_plus - block_count; - // Check if the requested range is within the cache bounds - let fee_entries = self.fee_history_cache().get_history(start_block, end_block).await; - // Collect base fees, gas usage ratios and (optionally) reward percentile data let mut base_fee_per_gas: Vec = Vec::new(); let mut gas_used_ratio: Vec = Vec::new(); let mut rewards: Vec> = Vec::new(); + // Check if the requested range is within the cache bounds + let fee_entries = self.fee_history_cache().get_history(start_block, end_block).await; + if let Some(fee_entries) = fee_entries { if fee_entries.len() != block_count as usize { return Err(EthApiError::InvalidBlockRange) @@ -130,7 +129,7 @@ where rewards.push(block_rewards); } } - let last_entry = fee_entries.last().expect("is present"); + let last_entry = fee_entries.last().expect("is not empty"); base_fee_per_gas.push(U256::from(calculate_next_block_base_fee( last_entry.gas_used, last_entry.gas_limit, @@ -160,8 +159,8 @@ where percentiles, header.gas_used, header.base_fee_per_gas.unwrap_or_default(), - transactions, - receipts, + &transactions, + &receipts, ) .unwrap_or_default(), );