Skip to content

Commit

Permalink
[Epoch Sync] EpochSyncProof validation (#12020)
Browse files Browse the repository at this point in the history 
Issue: #11932

Implements EpochSyncProof validation, according to:
- [design
doc](https://docs.google.com/document/d/14Itc9Hs7ewTRmcGANid9UCaYcJzaLzzM7FsvYYIFKDY/edit#heading=h.p2oiicluwd6o)
-
[`epoch_sync.rs`](https://github.com/near/nearcore/blob/d6d6e4782f7d57a7b7acc2894e1668355012ff74/core/primitives/src/epoch_sync.rs)

Summary:
- Move block merkle proof retrieval logic from `Chain` to a separate
trait (`MerkleProofAccess`). Make `Store` implements this trait so that
we can `get_block_proof` while serving `EpochSyncRequest` from a
separate thread and having no access to `Chain`.
- Refactor `verify_approval` and `compute_bp_hash` so that we can use it
for proof validation.
- Derive `first_block_info_in_epoch` from block header.
- Implement the validation logic.
- Add 2 more fields to the proof
  - `EpochSyncProofPastEpochData::protocol_version`
- `merkle_proof_for_first_block` generated at height of final block of
current epoch.
- Apparently, we need to add even more to enable generating
`merkle_proof_for_first_block` at the bootstrapped node. That will be
done as a follow-up issue:
#12255.
- Create a bunch of follow-up issues (added to the [tracking
issue](near/near-one-project-tracking#73)):
  - #12255
  - #12256
  - #12258
  - #12259
  - #12260
  • Loading branch information
@staffik
staffik authored Oct 18, 2024
1 parent 4ddca0e commit 2c40e53
Show file tree
Hide file tree
Showing 12 changed files with 538 additions and 223 deletions.
5 changes: 5 additions & 0 deletions chain/chain-primitives/src/error.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,9 @@ pub enum Error {
/// Resharding error.
#[error("Resharding Error: {0}")]
ReshardingError(String),
/// EpochSyncProof validation error.
#[error("EpochSyncProof Validation Error: {0}")]
InvalidEpochSyncProof(String),
/// Anything else
#[error("Other Error: {0}")]
Other(String),
Expand Down Expand Up @@ -300,6 +303,7 @@ impl Error {
| Error::MaliciousChallenge
| Error::IncorrectNumberOfChunkHeaders
| Error::InvalidEpochHash
| Error::InvalidEpochSyncProof(_)
| Error::InvalidNextBPHash
| Error::NotEnoughApprovals
| Error::InvalidFinalityInfo
Expand Down Expand Up @@ -377,6 +381,7 @@ impl Error {
Error::MaliciousChallenge => "malicious_challenge",
Error::IncorrectNumberOfChunkHeaders => "incorrect_number_of_chunk_headers",
Error::InvalidEpochHash => "invalid_epoch_hash",
Error::InvalidEpochSyncProof(_) => "invalid_epoch_sync_proof",
Error::InvalidNextBPHash => "invalid_next_bp_hash",
Error::NotEnoughApprovals => "not_enough_approvals",
Error::InvalidFinalityInfo => "invalid_finality_info",
Expand Down
214 changes: 30 additions & 184 deletions chain/chain/src/chain.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use crate::state_snapshot_actor::SnapshotCallbacks;
use crate::stateless_validation::chunk_endorsement::{
validate_chunk_endorsements_in_block, validate_chunk_endorsements_in_header,
};
use crate::store::{ChainStore, ChainStoreAccess, ChainStoreUpdate};
use crate::store::{ChainStore, ChainStoreAccess, ChainStoreUpdate, MerkleProofAccess};
use crate::types::{
AcceptedBlock, ApplyChunkBlockContext, BlockEconomicsConfig, ChainConfig, RuntimeAdapter,
StorageDataSource,
Expand Down Expand Up @@ -58,9 +58,7 @@ use near_primitives::congestion_info::CongestionInfo;
use near_primitives::epoch_block_info::BlockInfo;
use near_primitives::errors::EpochError;
use near_primitives::hash::{hash, CryptoHash};
use near_primitives::merkle::{
combine_hash, merklize, verify_path, Direction, MerklePath, MerklePathItem, PartialMerkleTree,
};
use near_primitives::merkle::{merklize, verify_path, PartialMerkleTree};
use near_primitives::receipt::Receipt;
use near_primitives::sandbox::state_patch::SandboxStatePatch;
use near_primitives::shard_layout::{account_id_to_shard_id, ShardLayout, ShardUId};
Expand All @@ -78,6 +76,7 @@ use near_primitives::stateless_validation::state_witness::{
};
use near_primitives::transaction::{ExecutionOutcomeWithIdAndProof, SignedTransaction};
use near_primitives::types::chunk_extra::ChunkExtra;
use near_primitives::types::validator_stake::ValidatorStake;
use near_primitives::types::{
AccountId, Balance, BlockExtra, BlockHeight, BlockHeightDelta, EpochId, Gas, MerkleHash,
NumBlocks, ShardId, StateRoot,
Expand Down Expand Up @@ -586,13 +585,20 @@ impl Chain {
last_known_hash: &CryptoHash,
) -> Result<CryptoHash, Error> {
let bps = epoch_manager.get_epoch_block_producers_ordered(&epoch_id, last_known_hash)?;
let validator_stakes = bps.into_iter().map(|(bp, _)| bp).collect_vec();
let protocol_version = epoch_manager.get_epoch_protocol_version(&prev_epoch_id)?;
Self::compute_bp_hash_from_validator_stakes(&validator_stakes, protocol_version)
}

pub fn compute_bp_hash_from_validator_stakes(
validator_stakes: &Vec<ValidatorStake>,
protocol_version: ProtocolVersion,
) -> Result<CryptoHash, Error> {
if checked_feature!("stable", BlockHeaderV3, protocol_version) {
let validator_stakes = bps.into_iter().map(|(bp, _)| bp);
Ok(CryptoHash::hash_borsh_iter(validator_stakes))
} else {
let validator_stakes = bps.into_iter().map(|(bp, _)| bp.into_v1());
Ok(CryptoHash::hash_borsh_iter(validator_stakes))
let stakes = validator_stakes.into_iter().map(|stake| stake.clone().into_v1());
Ok(CryptoHash::hash_borsh_iter(stakes))
}
}

Expand Down Expand Up @@ -1521,6 +1527,16 @@ impl Chain {
(accepted_blocks, errors)
}

fn chain_update(&mut self) -> ChainUpdate {
ChainUpdate::new(
&mut self.chain_store,
self.epoch_manager.clone(),
self.runtime_adapter.clone(),
self.doomslug_threshold_mode,
self.transaction_validity_period,
)
}

/// Process challenge to invalidate chain. This is done between blocks to unroll the chain as
/// soon as possible and allow next block producer to skip invalid blocks.
pub fn process_challenge(&mut self, challenge: &Challenge) {
Expand Down Expand Up @@ -4011,186 +4027,16 @@ fn get_should_apply_chunk(
}
}

/// Implement block merkle proof retrieval.
impl Chain {
fn combine_maybe_hashes(
hash1: Option<MerkleHash>,
hash2: Option<MerkleHash>,
) -> Option<MerkleHash> {
match (hash1, hash2) {
(Some(h1), Some(h2)) => Some(combine_hash(&h1, &h2)),
(Some(h1), None) => Some(h1),
(None, Some(_)) => {
debug_assert!(false, "Inconsistent state in merkle proof computation: left node is None but right node exists");
None
}
_ => None,
}
}

fn chain_update(&mut self) -> ChainUpdate {
ChainUpdate::new(
&mut self.chain_store,
self.epoch_manager.clone(),
self.runtime_adapter.clone(),
self.doomslug_threshold_mode,
self.transaction_validity_period,
)
}

/// Get node at given position (index, level). If the node does not exist, return `None`.
fn get_merkle_tree_node(
&self,
index: u64,
level: u64,
counter: u64,
tree_size: u64,
tree_nodes: &mut HashMap<(u64, u64), Option<MerkleHash>>,
) -> Result<Option<MerkleHash>, Error> {
if let Some(hash) = tree_nodes.get(&(index, level)) {
Ok(*hash)
} else {
if level == 0 {
let maybe_hash = if index >= tree_size {
None
} else {
Some(self.chain_store().get_block_hash_from_ordinal(index)?)
};
tree_nodes.insert((index, level), maybe_hash);
Ok(maybe_hash)
} else {
let cur_tree_size = (index + 1) * counter;
let maybe_hash = if cur_tree_size > tree_size {
if index * counter <= tree_size {
let left_hash = self.get_merkle_tree_node(
index * 2,
level - 1,
counter / 2,
tree_size,
tree_nodes,
)?;
let right_hash = self.reconstruct_merkle_tree_node(
index * 2 + 1,
level - 1,
counter / 2,
tree_size,
tree_nodes,
)?;
Self::combine_maybe_hashes(left_hash, right_hash)
} else {
None
}
} else {
Some(
*self
.chain_store()
.get_block_merkle_tree_from_ordinal(cur_tree_size)?
.get_path()
.last()
.ok_or_else(|| Error::Other("Merkle tree node missing".to_string()))?,
)
};
tree_nodes.insert((index, level), maybe_hash);
Ok(maybe_hash)
}
}
}

/// Reconstruct node at given position (index, level). If the node does not exist, return `None`.
fn reconstruct_merkle_tree_node(
impl MerkleProofAccess for Chain {
fn get_block_merkle_tree(
&self,
index: u64,
level: u64,
counter: u64,
tree_size: u64,
tree_nodes: &mut HashMap<(u64, u64), Option<MerkleHash>>,
) -> Result<Option<MerkleHash>, Error> {
if let Some(hash) = tree_nodes.get(&(index, level)) {
Ok(*hash)
} else {
if level == 0 {
let maybe_hash = if index >= tree_size {
None
} else {
Some(self.chain_store().get_block_hash_from_ordinal(index)?)
};
tree_nodes.insert((index, level), maybe_hash);
Ok(maybe_hash)
} else {
let left_hash = self.get_merkle_tree_node(
index * 2,
level - 1,
counter / 2,
tree_size,
tree_nodes,
)?;
let right_hash = self.reconstruct_merkle_tree_node(
index * 2 + 1,
level - 1,
counter / 2,
tree_size,
tree_nodes,
)?;
let maybe_hash = Self::combine_maybe_hashes(left_hash, right_hash);
tree_nodes.insert((index, level), maybe_hash);

Ok(maybe_hash)
}
}
block_hash: &CryptoHash,
) -> Result<Arc<PartialMerkleTree>, Error> {
ChainStoreAccess::get_block_merkle_tree(self.chain_store(), block_hash)
}

/// Get merkle proof for block with hash `block_hash` in the merkle tree of `head_block_hash`.
pub fn get_block_proof(
&self,
block_hash: &CryptoHash,
head_block_hash: &CryptoHash,
) -> Result<MerklePath, Error> {
let leaf_index = self.chain_store().get_block_merkle_tree(block_hash)?.size();
let tree_size = self.chain_store().get_block_merkle_tree(head_block_hash)?.size();
if leaf_index >= tree_size {
if block_hash == head_block_hash {
// special case if the block to prove is the same as head
return Ok(vec![]);
}
return Err(Error::Other(format!(
"block {} is ahead of head block {}",
block_hash, head_block_hash
)));
}
let mut level = 0;
let mut counter = 1;
let mut cur_index = leaf_index;
let mut path = vec![];
let mut tree_nodes = HashMap::new();
let mut iter = tree_size;
while iter > 1 {
if cur_index % 2 == 0 {
cur_index += 1
} else {
cur_index -= 1;
}
let direction = if cur_index % 2 == 0 { Direction::Left } else { Direction::Right };
let maybe_hash = if cur_index % 2 == 1 {
// node not immediately available. Needs to be reconstructed
self.reconstruct_merkle_tree_node(
cur_index,
level,
counter,
tree_size,
&mut tree_nodes,
)?
} else {
self.get_merkle_tree_node(cur_index, level, counter, tree_size, &mut tree_nodes)?
};
if let Some(hash) = maybe_hash {
path.push(MerklePathItem { hash, direction });
}
cur_index /= 2;
iter = (iter + 1) / 2;
level += 1;
counter *= 2;
}
Ok(path)
fn get_block_hash_from_ordinal(&self, block_ordinal: NumBlocks) -> Result<CryptoHash, Error> {
ChainStoreAccess::get_block_hash_from_ordinal(self.chain_store(), block_ordinal)
}
}

Expand Down
4 changes: 3 additions & 1 deletion chain/chain/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ pub use doomslug::{Doomslug, DoomslugBlockProductionReadiness, DoomslugThreshold
pub use lightclient::{create_light_client_block_view, get_epoch_block_producers_view};
pub use near_chain_primitives::{self, Error};
pub use near_primitives::receipt::ReceiptResult;
pub use store::{ChainStore, ChainStoreAccess, ChainStoreUpdate, LatestWitnessesInfo};
pub use store::{
ChainStore, ChainStoreAccess, ChainStoreUpdate, LatestWitnessesInfo, MerkleProofAccess,
};
pub use store_validator::{ErrorMessage, StoreValidator};
pub use types::{Block, BlockHeader, BlockStatus, ChainGenesis, LatestKnown, Provenance};

Expand Down
Loading

0 comments on commit 2c40e53

Please sign in to comment.