Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Domains: Move trace root capture from runtime to client for efficiency #3338

Merged
merged 16 commits into from
Jan 24, 2025
Merged

Domains: Move trace root capture from runtime to client for efficiency #3338

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
8a56468
move delta backend from execution prover to custom api
vedhavyas Jan 6, 2025
1b578e7
add trie backend api
vedhavyas Jan 6, 2025
ac1d95c
cleanup block builder
vedhavyas Jan 7, 2025
90a28c2
collect intermediate roots and add missing inherent_extrinsics api call
vedhavyas Jan 7, 2025
f7026d3
intergate TrieBackendApi into domain block builder
vedhavyas Jan 7, 2025
c34427f
replace runtime intermediate roots with roots derived from TrieBacken…
vedhavyas Jan 8, 2025
fa4fce9
remove Intermediate roots from pallet executive
vedhavyas Jan 9, 2025
dfc3827
avoid unnecessary clone of InmemoryDB
vedhavyas Jan 9, 2025
cae4f90
deduplicate storage changes instead of simp,y appending them
vedhavyas Jan 9, 2025
cc6f251
hold runtime code and some small cleanup
vedhavyas Jan 10, 2025
ae83448
update struct comment and remove debug points
vedhavyas Jan 14, 2025
c3ee9a3
add storage overlay checks pallet
vedhavyas Jan 17, 2025
3019471
purge all the zero referenced nodes post consolidation.
vedhavyas Jan 17, 2025
35aacb5
Add test test_custom_api_storage_root_match_upstream_root
NingLin-P Jan 23, 2025
564c648
Merge pull request #3357 from autonomys/add-test-for-custom-api
vedhavyas Jan 24, 2025
f4416f9
Merge branch 'main' into storage_root_changes
vedhavyas Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 2 additions & 3 deletions crates/sp-domains-fraud-proof/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ include = [

[dependencies]
codec = { package = "parity-scale-codec", version = "3.6.12", default-features = false, features = ["derive"] }
domain-block-builder = { version = "0.1.0", path = "../../domains/client/block-builder", optional = true, default-features = false }
domain-block-preprocessor = { version = "0.1.0", default-features = false, path = "../../domains/client/block-preprocessor", optional = true }
domain-runtime-primitives = { version = "0.1.0", default-features = false, path = "../../domains/primitives/runtime" }
frame-support = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
Expand Down Expand Up @@ -41,7 +42,6 @@ trie-db = { version = "0.29.1", default-features = false }
thiserror = { version = "2.0.0", default-features = false }

[dev-dependencies]
domain-block-builder = { version = "0.1.0", path = "../../domains/client/block-builder" }
domain-block-preprocessor = { version = "0.1.0", path = "../../domains/client/block-preprocessor" }
domain-test-service = { version = "0.1.0", path = "../../domains/test/service" }
ethereum = "0.15.0"
Expand All @@ -54,13 +54,11 @@ libsecp256k1 = { version = "0.7.1", features = ["static-context", "hmac"] }
pallet-balances = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
pallet-ethereum = { git = "https://github.com/autonomys/frontier", rev = "f80f9e2bad338f3bf3854b256b3c4edea23e5968", features = ['default'] }
pallet-evm = { version = "6.0.0-dev", git = "https://github.com/autonomys/frontier", rev = "f80f9e2bad338f3bf3854b256b3c4edea23e5968", default-features = false }
parking_lot = "0.12.2"
rand = { version = "0.8.5", features = ["min_const_gen"] }
rlp = "0.5.2"
sp-core = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
sc-cli = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305", default-features = false }
sc-service = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305", default-features = false }
subspace-test-client = { version = "0.1.0", path = "../../test/subspace-test-client" }
subspace-test-service = { version = "0.1.0", path = "../../test/subspace-test-service" }
subspace-runtime-primitives = { version = "0.1.0", path = "../../crates/subspace-runtime-primitives" }
tempfile = "3.13.0"
Expand All @@ -70,6 +68,7 @@ tokio = "1.40.0"
default = ["std"]
std = [
"codec/std",
"domain-block-builder",
"domain-block-preprocessor",
"domain-runtime-primitives/std",
"frame-support/std",
Expand Down
61 changes: 5 additions & 56 deletions crates/sp-domains-fraud-proof/src/execution_prover.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,13 @@
//! (`initialize_block` and `finalize_block`) and any specific extrinsic execution are supported.

use crate::fraud_proof::ExecutionPhase;
use codec::Codec;
use hash_db::{HashDB, Hasher, Prefix};
use domain_block_builder::create_delta_backend;
use sc_client_api::backend::Backend;
use sp_api::StorageProof;
use sp_core::traits::CodeExecutor;
use sp_runtime::traits::{Block as BlockT, HashingFor};
use sp_state_machine::backend::AsTrieBackend;
use sp_state_machine::{TrieBackend, TrieBackendBuilder, TrieBackendStorage};
use sp_trie::DBValue;
use sp_state_machine::BackendTransaction;
use std::marker::PhantomData;
use std::sync::Arc;

Expand Down Expand Up @@ -41,12 +39,12 @@ where

/// Returns a storage proof which can be used to reconstruct a partial state trie to re-run
/// the execution by someone who does not own the whole state.
pub fn prove_execution<DB: HashDB<HashingFor<Block>, DBValue>>(
pub fn prove_execution(
&self,
at: Block::Hash,
execution_phase: &ExecutionPhase,
call_data: &[u8],
delta_changes: Option<(DB, Block::Hash)>,
delta_changes: Option<(BackendTransaction<HashingFor<Block>>, Block::Hash)>,
) -> sp_blockchain::Result<StorageProof> {
let state = self.backend.state_at(at)?;

Expand All @@ -60,7 +58,7 @@ where
// TODO: avoid using the String API specified by `execution_method()`
// https://github.com/paritytech/substrate/discussions/11095
if let Some((delta, post_delta_root)) = delta_changes {
let delta_backend = create_delta_backend(trie_backend, delta, post_delta_root);
let delta_backend = create_delta_backend(trie_backend, &delta, post_delta_root);
sp_state_machine::prove_execution_on_trie_backend(
&delta_backend,
&mut Default::default(),
Expand All @@ -87,52 +85,3 @@ where
}
}
}

/// Create a new trie backend with memory DB delta changes.
///
/// This can be used to verify any extrinsic-specific execution on the combined state of `backend`
/// and `delta`.
fn create_delta_backend<'a, S, H, DB>(
backend: &'a TrieBackend<S, H>,
delta: DB,
post_delta_root: H::Out,
) -> TrieBackend<DeltaBackend<'a, S, H, DB>, H>
where
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher,
H::Out: Codec,
DB: HashDB<H, DBValue>,
{
let essence = backend.essence();
let delta_backend = DeltaBackend {
backend: essence.backend_storage(),
delta,
_phantom: PhantomData::<H>,
};
TrieBackendBuilder::new(delta_backend, post_delta_root).build()
}

struct DeltaBackend<'a, S, H, DB>
where
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher,
DB: HashDB<H, DBValue>,
{
backend: &'a S,
delta: DB,
_phantom: PhantomData<H>,
}

impl<'a, S, H, DB> TrieBackendStorage<H> for DeltaBackend<'a, S, H, DB>
where
S: 'a + TrieBackendStorage<H>,
H: 'a + Hasher,
DB: HashDB<H, DBValue>,
{
fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
match HashDB::get(&self.delta, key, prefix) {
Some(v) => Ok(Some(v)),
None => Ok(self.backend.get(key, prefix)?),
}
}
}
3 changes: 0 additions & 3 deletions crates/sp-domains/src/core_api.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,6 @@ sp_api::decl_runtime_apis! {
tx_range: &U256,
) -> bool;

/// Returns the intermediate storage roots in an encoded form.
fn intermediate_roots() -> Vec<[u8; 32]>;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to bump the API version when removing a method?

I think this means old client code will refuse to run with the new runtime due to a missing function, but that’s what we want, right? We want users to upgrade their nodes.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we do client upgrades first before runtime releases. So all the clients will use client side intermediate roots and once runtime is upgraded the function completely disappears.

I still like to check running the client with new change on taurus to ensure both give same roots. I have verified these locally but verifying on taurus is never a bad thing :)


/// Returns the storage root after initializing the block.
fn initialize_block_with_post_state_root(header: &Block::Header) -> Vec<u8>;

Expand Down
2 changes: 2 additions & 0 deletions domains/client/block-builder/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@ targets = ["x86_64-unknown-linux-gnu"]

[dependencies]
codec = { package = "parity-scale-codec", version = "3.6.12", features = ["derive"] }
hash-db = { version = "0.16.0", default-features = false }
sc-client-api = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
sp-api = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
sp-blockchain = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
sp-block-builder = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
sp-core = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
sp-externalities = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
sp-inherents = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
sp-runtime = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
sp-state-machine = { git = "https://github.com/subspace/polkadot-sdk", rev = "94a1a8143a89bbe9f938c1939ff68abc1519a305" }
Expand Down
Loading
Loading