From 7ad89dedc2eae4264883b7f81775cbaeeb83d94b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 15 Jul 2024 16:49:51 +0800 Subject: [PATCH] Preview --- .cargo/config.toml | 2 +- Cargo.lock | 74 ++++++ Cargo.toml | 29 ++- evm_arithmetization/Cargo.toml | 6 +- .../benches/fibonacci_25m_gas.rs | 1 + .../src/cpu/kernel/interpreter.rs | 63 +++++- .../src/cpu/kernel/tests/add11.rs | 2 + .../kernel/tests/core/jumpdest_analysis.rs | 10 + evm_arithmetization/src/generation/mod.rs | 7 +- .../src/generation/prover_input.rs | 43 +++- evm_arithmetization/src/generation/state.rs | 5 +- evm_arithmetization/src/lib.rs | 2 + evm_arithmetization/src/witness/operation.rs | 1 + evm_arithmetization/tests/add11_yml.rs | 1 + .../tests/basic_smart_contract.rs | 1 + evm_arithmetization/tests/empty_txn_list.rs | 1 + evm_arithmetization/tests/erc20.rs | 1 + evm_arithmetization/tests/erc721.rs | 1 + evm_arithmetization/tests/log_opcode.rs | 4 + .../tests/self_balance_gas_cost.rs | 1 + evm_arithmetization/tests/selfdestruct.rs | 1 + evm_arithmetization/tests/simple_transfer.rs | 1 + evm_arithmetization/tests/two_to_one_block.rs | 1 + evm_arithmetization/tests/withdrawals.rs | 1 + trace_decoder/src/decoding.rs | 2 + trace_decoder/src/lib.rs | 13 +- trace_decoder/src/processed_block_trace.rs | 3 + zero_bin/common/Cargo.toml | 1 + zero_bin/rpc/Cargo.toml | 2 + zero_bin/rpc/src/native/txn.rs | 210 ++++++++++++++++-- zero_bin/tools/prove_stdio.sh | 3 +- 31 files changed, 459 insertions(+), 34 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 6340ce34a..a71f56369 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,4 @@ [build] # https://github.com/rust-lang/rust/pull/124129 # https://github.com/dtolnay/linkme/pull/88 -rustflags = ["-Z", "linker-features=-lld"] +rustflags = ["-C", "target-cpu=native", "-Z", "linker-features=-lld"] diff --git a/Cargo.lock b/Cargo.lock index 0302d8d8d..da841d3dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1674,6 +1674,12 @@ dependencies = [ "cipher", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.9.0" @@ -1901,10 +1907,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "evm-disassembler" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded685d9f07315ff689ba56e7d84e6f1e782db19b531a46c34061a733bba7258" +dependencies = [ + "eyre", + "hex", +] + [[package]] name = "evm_arithmetization" version = "0.2.0" dependencies = [ + "alloy", "anyhow", "bytes", "criterion", @@ -1925,6 +1942,7 @@ dependencies = [ "plonky2", "plonky2_maybe_rayon", "plonky2_util", + "pretty_assertions", "rand", "rand_chacha", "ripemd", @@ -1948,6 +1966,16 @@ dependencies = [ "async-trait", ] +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + [[package]] name = "fastrand" version = "1.9.0" @@ -1984,6 +2012,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "filedescriptor" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7199d965852c3bac31f779ef99cbb4537f80e952e2d6aa0ffeb30cce00f4f46e" +dependencies = [ + "libc", + "thiserror", + "winapi", +] + [[package]] name = "fixed-hash" version = "0.7.0" @@ -2187,6 +2226,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42012b0f064e01aa58b545fe3727f90f7dd4020f4a3ea735b50344965f5a57e9" +[[package]] +name = "gag" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a713bee13966e9fbffdf7193af71d54a6b35a0bb34997cd6c9519ebeb5005972" +dependencies = [ + "filedescriptor", + "tempfile", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -2503,6 +2552,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + [[package]] name = "indexmap" version = "2.2.6" @@ -3611,6 +3666,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "pretty_env_logger" version = "0.5.0" @@ -4002,8 +4067,10 @@ dependencies = [ "anyhow", "clap", "compat", + "evm-disassembler", "evm_arithmetization", "futures", + "gag", "hex", "lru", "mpt_trie", @@ -5581,6 +5648,12 @@ dependencies = [ "time", ] +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zero_bin_common" version = "0.1.0" @@ -5592,6 +5665,7 @@ dependencies = [ "evm_arithmetization", "futures", "plonky2", + "pretty_assertions", "proof_gen", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 3d9a44604..e0b75d74c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,6 @@ criterion = "0.5.1" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" -enumn = "0.1.13" env_logger = "0.11.3" eth_trie = "0.4.0" ethereum-types = "0.14.1" @@ -94,7 +93,6 @@ ruint = "1.12.3" serde = "1.0.203" serde_json = "1.0.118" serde_path_to_error = "0.1.16" -serde_with = "3.8.1" sha2 = "0.10.8" smt_trie = { path = "smt_trie", version = "0.1.0" } static_assertions = "1.1.0" @@ -110,6 +108,8 @@ u4 = "0.1.0" uint = "0.9.5" url = "2.5.2" winnow = "0.6.13" +evm-disassembler = "0.5.0" +pretty_assertions = "1.4.0" # zero-bin related dependencies ops = { path = "zero_bin/ops" } @@ -128,4 +128,29 @@ proc-macro2 = "1.0" quote = "1.0" syn = "2.0" trybuild = "1.0" + zk_evm_proc_macro = { path = "proc_macro" } + +[profile.release] +opt-level=3 +debug=true +incremental=true +debug-assertions=true +lto=false +overflow-checks=false + +[profile.test] +opt-level=3 +debug=true +incremental=true +debug-assertions=true +lto=false +overflow-checks=false + +[profile.dev] +opt-level=3 +debug=true +incremental=true +debug-assertions=true +lto=false +overflow-checks=false diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index 8d98c214b..0aafea55b 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -19,7 +19,7 @@ anyhow = { workspace = true } bytes = { workspace = true } env_logger = { workspace = true } ethereum-types = { workspace = true } -hex = { workspace = true, optional = true } +hex = { workspace = true } hex-literal = { workspace = true } itertools = { workspace = true } keccak-hash = { workspace = true } @@ -46,6 +46,8 @@ serde_json = { workspace = true } # Local dependencies mpt_trie = { workspace = true } zk_evm_proc_macro = { workspace = true } +pretty_assertions = { workspace = true } +alloy.workspace = true [dev-dependencies] criterion = { workspace = true } @@ -55,7 +57,7 @@ sha2 = { workspace = true } [features] default = ["parallel"] -asmtools = ["hex"] +#asmtools = ["hex"] parallel = [ "plonky2/parallel", "plonky2_maybe_rayon/parallel", diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index d27b7e5ce..e49ceed57 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -173,6 +173,7 @@ fn prepare_setup() -> anyhow::Result { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index e47d48e92..91e126045 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -9,10 +9,11 @@ use core::cmp::Ordering; use std::collections::{BTreeSet, HashMap}; use anyhow::anyhow; -use ethereum_types::{BigEndianHash, U256}; +use ethereum_types::{BigEndianHash, U256, H256}; use log::Level; use mpt_trie::partial_trie::PartialTrie; use plonky2::field::types::Field; +//use alloy::primitives::Address; use crate::byte_packing::byte_packing_stark::BytePackingOp; use crate::cpu::columns::CpuColumnsView; @@ -20,6 +21,7 @@ use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::generation::debug_inputs; use crate::generation::mpt::load_all_mpts; +use crate::generation::prover_input::{get_proofs_and_jumpdests, JumpDestTableProcessed, JumpDestTableRaw}; use crate::generation::rlp::all_rlp_prover_inputs_reversed; use crate::generation::state::{ all_withdrawals_prover_inputs_reversed, GenerationState, GenerationStateCheckpoint, @@ -62,10 +64,12 @@ pub(crate) struct Interpreter { /// Simulates the CPU execution from `state` until the program counter reaches /// `final_label` in the current context. +/// TODO: main fun +#[deprecated(note = "use rpc to get jumpdests instead")] pub(crate) fn simulate_cpu_and_get_user_jumps( final_label: &str, state: &GenerationState, -) -> Option>> { +) -> Option>> { match state.jumpdest_table { Some(_) => None, None => { @@ -90,6 +94,58 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( } } +// todo: remove. for easy comparison only +#[deprecated(note = "use rpc to get jumpdests instead")] +pub(crate) fn simulate_cpu_and_get_user_jumps_rpc( + final_label: &str, + state: &GenerationState, +) -> Option { + match state.jumpdest_table { + Some(_) => None, + None => { + let halt_pc = KERNEL.global_labels[final_label]; + let initial_context = state.registers.context; + let mut interpreter = + Interpreter::new_with_state_and_halt_condition(state, halt_pc, initial_context); + + log::debug!("Simulating CPU for jumpdest analysis."); + + let _ = interpreter.run(); + + log::trace!("jumpdest table = {:?}", interpreter.jumpdest_table); + + //debug_assert_eq!(&interpreter.jumpdest_table, &state.inputs.jumpdest_table); + let jumpdest_table = set_jumpdest_analysis_inputs_rpc(&state.inputs.jumpdest_table, &state.inputs.contract_code); + + Some(jumpdest_table) + } + } +} + +/// Given a HashMap containing the contexts and the jumpdest addresses, compute +/// their respective proofs, by calling `get_proofs_and_jumpdests` +pub(crate) fn set_jumpdest_analysis_inputs_rpc( + jumpdest_table_rpc: &JumpDestTableRaw, + codes: &HashMap>, +) -> JumpDestTableProcessed { + JumpDestTableProcessed(jumpdest_table_rpc.0.iter().flat_map(|(code_address, ctx_table)| { + ctx_table.0.iter().map(|(&ctx, jumpdest_table)| { + let code = &codes[code_address]; + if let Some(&largest_address) = jumpdest_table.last() { + // maybe just serialize it as BTreeSet and avoid conversion + // this is consistent with: + // https://doc.rust-lang.org/std/collections/struct.BTreeSet.html#method.last + // let jdt: BTreeSet = jumpdest_table.into_iter().collect(); + let proofs = get_proofs_and_jumpdests(code, largest_address, jumpdest_table.clone()); + (ctx, proofs) + } else { + (ctx, vec![]) + } + }).collect::>>() + }).collect()) +} + + impl Interpreter { /// Returns an instance of `Interpreter` given `GenerationInputs`, and /// assuming we are initializing with the `KERNEL` code. @@ -350,6 +406,7 @@ impl Interpreter { } pub(crate) fn add_jumpdest_offset(&mut self, offset: usize) { + println!("SIM: ({:?}, {:?})", &self.generation_state.registers.context, offset); if let Some(jumpdest_table) = self .jumpdest_table .get_mut(&self.generation_state.registers.context) @@ -506,7 +563,7 @@ impl State for Interpreter { let registers = self.generation_state.registers; let (mut row, opcode) = self.base_row(); - let op = decode(registers, opcode)?; + let op: Operation = decode(registers, opcode)?; fill_op_flag(op, &mut row); diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index c2be6a0bd..511bfbca1 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -152,6 +152,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let initial_stack = vec![]; @@ -293,6 +294,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index dd9a295e2..de578d707 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -210,3 +210,13 @@ fn test_verify_non_jumpdest() -> Result<()> { } Ok(()) } + +#[test] +fn compare_jumpdest_tables() -> Result<()> { + let jumpdest_tbl_sim = todo!(); + let jumpdest_tbl_rpc = todo!(); + + assert_eq!(jumpdest_tbl_sim, jumpdest_tbl_rpc); + + Ok(()) +} \ No newline at end of file diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 84da8ceb2..396a4ae29 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -1,7 +1,8 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use anyhow::anyhow; use ethereum_types::{Address, BigEndianHash, H256, U256}; +// use hex::ToHex; use log::log_enabled; use mpt_trie::partial_trie::{HashedPartialTrie, PartialTrie}; use plonky2::field::extension::Extendable; @@ -10,6 +11,7 @@ use plonky2::field::types::Field; use plonky2::hash::hash_types::RichField; use plonky2::timed; use plonky2::util::timing::TimingTree; +use prover_input::JumpDestTableRaw; use serde::{Deserialize, Serialize}; use starky::config::StarkConfig; use GlobalMetadata::{ @@ -76,6 +78,8 @@ pub struct GenerationInputs { /// The hash of the current block, and a list of the 256 previous block /// hashes. pub block_hashes: BlockHashes, + + pub jumpdest_table: JumpDestTableRaw, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] @@ -223,6 +227,7 @@ pub fn generate_traces, const D: usize>( timing: &mut TimingTree, ) -> anyhow::Result<([Vec>; NUM_TABLES], PublicValues)> { debug_inputs(&inputs); + // TODO let mut state = GenerationState::::new(inputs.clone(), &KERNEL.code) .map_err(|err| anyhow!("Failed to parse all the initial prover inputs: {:?}", err))?; diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 904524be2..558235ee4 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -4,14 +4,16 @@ use std::str::FromStr; use anyhow::{bail, Error}; use ethereum_types::{BigEndianHash, H256, U256, U512}; +use hex::{FromHex, ToHex}; use itertools::Itertools; use num_bigint::BigUint; use plonky2::field::types::Field; use serde::{Deserialize, Serialize}; +use alloy::primitives::Address; use crate::cpu::kernel::constants::context_metadata::ContextMetadata; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; -use crate::cpu::kernel::interpreter::simulate_cpu_and_get_user_jumps; +use crate::cpu::kernel::interpreter::{set_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps}; use crate::extension_tower::{FieldExt, Fp12, BLS381, BN254}; use crate::generation::prover_input::EvmField::{ Bls381Base, Bls381Scalar, Bn254Base, Bn254Scalar, Secp256k1Base, Secp256k1Scalar, @@ -27,6 +29,22 @@ use crate::witness::memory::MemoryAddress; use crate::witness::operation::CONTEXT_SCALING_FACTOR; use crate::witness::util::{current_context_peek, stack_peek}; +use pretty_assertions::assert_eq; + + +/// Map from `CodeAddress -> (Context -> [JumpDests])` +/// Each `CodeAddress` can be called one or more times, each time creating a new `Context`. +/// Each `Context` may will have zero or more `JumpDests`. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct ContextTable(pub HashMap>); + +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct JumpDestTableRaw(pub HashMap); + +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct JumpDestTableProcessed(pub HashMap>); + + /// Prover input function represented as a scoped function name. /// Example: `PROVER_INPUT(ff::bn254_base::inverse)` is represented as /// `ProverInputFn([ff, bn254_base, inverse])`. @@ -266,6 +284,7 @@ impl GenerationState { fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + // This should take care of itself when the table is already filled in.. if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } @@ -386,7 +405,24 @@ impl GenerationState { fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - self.jumpdest_table = simulate_cpu_and_get_user_jumps("terminate_common", self); + // TODO! + // let rpc = simulate_cpu_and_get_user_jumps_rpc("terminate_common", self); + let sim = simulate_cpu_and_get_user_jumps("terminate_common", self); + println!("SIM JUMPDEST table"); + dbg!(&sim); + + let rpc = { + //debug_assert_eq!(&interpreter.jumpdest_table, &state.inputs.jumpdest_table); + //let code = self.inputs.contract_code; + let jumpdest_table = set_jumpdest_analysis_inputs_rpc(&self.inputs.jumpdest_table, &self.inputs.contract_code); + Some(jumpdest_table.0) + }; + println!("RPC JUMPDEST table"); + dbg!(&rpc); + + assert_eq!(sim, rpc); + assert!(false, "success"); + self.jumpdest_table = rpc; Ok(()) } @@ -495,7 +531,7 @@ impl GenerationState { /// for which none of the previous 32 bytes in the code (including opcodes /// and pushed bytes) is a PUSHXX and the address is in its range. It returns /// a vector of even size containing proofs followed by their addresses. -fn get_proofs_and_jumpdests( +pub(crate) fn get_proofs_and_jumpdests( code: &[u8], largest_address: usize, jumpdest_table: std::collections::BTreeSet, @@ -515,6 +551,7 @@ fn get_proofs_and_jumpdests( false }; let last_proof = if has_prefix { addr - 32 } else { last_proof }; + // todo if jumpdest_table.contains(&addr) { // Push the proof proofs.push(last_proof); diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index 6e08c5391..30da89ffb 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::mem::size_of; use anyhow::{anyhow, bail}; @@ -302,6 +302,8 @@ pub(crate) struct GenerationState { /// Pointers, within the `TrieData` segment, of the three MPTs. pub(crate) trie_root_ptrs: TrieRootPtrs, + //pub(crate) jumpdest_table_rpc: HashMap>, + /// A hash map where the key is a context in the user's code and the value /// is the set of jump destinations with its corresponding "proof". A /// "proof" for a jump destination is either 0 or an address i > 32 in @@ -340,6 +342,7 @@ impl GenerationState { txn_root_ptr: 0, receipt_root_ptr: 0, }, + // TODO jumpdest_table: None, }; let trie_root_ptrs = state.preinitialize_mpts(&inputs.tries); diff --git a/evm_arithmetization/src/lib.rs b/evm_arithmetization/src/lib.rs index b9236ae96..fc9ebaa94 100644 --- a/evm_arithmetization/src/lib.rs +++ b/evm_arithmetization/src/lib.rs @@ -206,6 +206,8 @@ pub mod verifier; pub mod generation; pub mod witness; +pub use generation::prover_input::{JumpDestTableProcessed, JumpDestTableRaw}; + // Utility modules pub mod curve_pairings; pub mod extension_tower; diff --git a/evm_arithmetization/src/witness/operation.rs b/evm_arithmetization/src/witness/operation.rs index 07d48819b..92be90c76 100644 --- a/evm_arithmetization/src/witness/operation.rs +++ b/evm_arithmetization/src/witness/operation.rs @@ -286,6 +286,7 @@ pub(crate) fn generate_set_context>( let old_ctx = generation_state.registers.context; // The popped value needs to be scaled down. let new_ctx = u256_to_usize(ctx >> CONTEXT_SCALING_FACTOR)?; + dbg!(&old_ctx, &new_ctx); let sp_field = ContextMetadata::StackSize.unscale(); let old_sp_addr = MemoryAddress::new(old_ctx, Segment::ContextMetadata, sp_field); diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 00d7e56b4..75148ebb6 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -163,6 +163,7 @@ fn add11_yml() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/basic_smart_contract.rs b/evm_arithmetization/tests/basic_smart_contract.rs index fd0948d80..9a41c4402 100644 --- a/evm_arithmetization/tests/basic_smart_contract.rs +++ b/evm_arithmetization/tests/basic_smart_contract.rs @@ -195,6 +195,7 @@ fn test_basic_smart_contract() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/empty_txn_list.rs b/evm_arithmetization/tests/empty_txn_list.rs index 1205414f6..031b7232c 100644 --- a/evm_arithmetization/tests/empty_txn_list.rs +++ b/evm_arithmetization/tests/empty_txn_list.rs @@ -69,6 +69,7 @@ fn test_empty_txn_list() -> anyhow::Result<()> { prev_hashes: initial_block_hashes, cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; // Initialize the preprocessed circuits for the zkEVM. diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index 609579af9..b6f83072f 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -171,6 +171,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 86dd34002..40cec1b3a 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -174,6 +174,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index 75cdd44f5..129dd0120 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -233,6 +233,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); @@ -443,6 +444,7 @@ fn test_log_with_aggreg() -> anyhow::Result<()> { prev_hashes: block_hashes.clone(), cur_hash: block_1_hash, }, + jumpdest_table: Default::default(), }; // Preprocess all circuits. @@ -573,6 +575,7 @@ fn test_log_with_aggreg() -> anyhow::Result<()> { prev_hashes: block_hashes.clone(), cur_hash: block_1_hash, }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove root second", log::Level::Info); @@ -639,6 +642,7 @@ fn test_log_with_aggreg() -> anyhow::Result<()> { prev_hashes: block_hashes, cur_hash: block_2_hash, }, + jumpdest_table: Default::default(), }; let (root_proof, public_values) = diff --git a/evm_arithmetization/tests/self_balance_gas_cost.rs b/evm_arithmetization/tests/self_balance_gas_cost.rs index f4aa8a606..c7eb32a06 100644 --- a/evm_arithmetization/tests/self_balance_gas_cost.rs +++ b/evm_arithmetization/tests/self_balance_gas_cost.rs @@ -177,6 +177,7 @@ fn self_balance_gas_cost() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 0ef48d2f4..07a04370d 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -134,6 +134,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 8adbc1c42..812ede0c1 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -150,6 +150,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/two_to_one_block.rs b/evm_arithmetization/tests/two_to_one_block.rs index fe387cee2..aa3edf60a 100644 --- a/evm_arithmetization/tests/two_to_one_block.rs +++ b/evm_arithmetization/tests/two_to_one_block.rs @@ -194,6 +194,7 @@ fn get_test_block_proof( contract_code: Default::default(), block_metadata: inputs.block_metadata.clone(), block_hashes: inputs.block_hashes.clone(), + jumpdest_table: Default::default(), }; let timing = &mut TimingTree::new(&format!("Blockproof {timestamp}"), log::Level::Info); diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index 7f133518b..5c63db393 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -80,6 +80,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/trace_decoder/src/decoding.rs b/trace_decoder/src/decoding.rs index 40a17c887..727a29bee 100644 --- a/trace_decoder/src/decoding.rs +++ b/trace_decoder/src/decoding.rs @@ -678,6 +678,7 @@ impl ProcessedBlockTrace { contract_code: txn_info.contract_code_accessed, block_metadata: other_data.b_data.b_meta.clone(), block_hashes: other_data.b_data.b_hashes.clone(), + jumpdest_table: txn_info.meta.jumpdest_table, }; // After processing a transaction, we update the remaining accumulators @@ -792,6 +793,7 @@ fn create_dummy_gen_input_common( gas_used_after: extra_data.gas_used_after, contract_code: HashMap::default(), withdrawals: vec![], // this is set after creating dummy payloads + jumpdest_table: Default::default(), } } diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index d5603012b..ff620cf53 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -35,7 +35,7 @@ //! if we can get this information sent to us instead. //! //! This library generates an Intermediary Representation (IR) of -//! a block's transactions, given a [BlockTrace] and some additional +//! a block's transactions, given a [`BlockTrace`] and some additional //! data represented by [OtherBlockData]. //! //! It first preprocesses the [BlockTrace] to provide transaction, @@ -97,7 +97,7 @@ use std::collections::HashMap; use ethereum_types::{Address, U256}; use evm_arithmetization::proof::{BlockHashes, BlockMetadata}; -use evm_arithmetization::GenerationInputs; +use evm_arithmetization::{GenerationInputs, JumpDestTableRaw}; use keccak_hash::H256; use mpt_trie::partial_trie::HashedPartialTrie; use serde::{Deserialize, Serialize}; @@ -122,6 +122,7 @@ pub struct BlockTrace { /// Traces and other info per transaction. The index of the transaction /// within the block corresponds to the slot in this vec. pub txn_info: Vec, + } /// Minimal hashed out tries needed by all txns in the block. @@ -205,6 +206,10 @@ pub struct TxnMeta { /// Gas used by this txn (Note: not cumulative gas used). pub gas_used: u64, + + /// JumpDest table + //#[serde(with = "crate::hex")] + pub jumpdest_table: JumpDestTableRaw, } /// A "trace" specific to an account for a txn. @@ -258,7 +263,7 @@ pub enum ContractCodeUsage { } impl ContractCodeUsage { - fn get_code_hash(&self) -> H256 { + pub fn get_code_hash(&self) -> H256 { match self { ContractCodeUsage::Read(hash) => *hash, ContractCodeUsage::Write(bytes) => hash(bytes), @@ -402,6 +407,8 @@ pub fn entrypoint( }) .collect::>(); + //other.jumpdest_tables = Some(jumpdest_tables); + Ok(ProcessedBlockTrace { tries: pre_images.tries, txn_info, diff --git a/trace_decoder/src/processed_block_trace.rs b/trace_decoder/src/processed_block_trace.rs index 7c9e89757..d68c298ec 100644 --- a/trace_decoder/src/processed_block_trace.rs +++ b/trace_decoder/src/processed_block_trace.rs @@ -4,6 +4,7 @@ use std::iter::once; use ethereum_types::{Address, H256, U256}; use evm_arithmetization::generation::mpt::{AccountRlp, LegacyReceiptRlp}; +use evm_arithmetization::JumpDestTableRaw; use mpt_trie::nibbles::Nibbles; use crate::hash; @@ -188,6 +189,7 @@ impl TxnInfo { txn_bytes, receipt_node_bytes, gas_used: self.meta.gas_used, + jumpdest_table: self.meta.jumpdest_table, }; ProcessedTxnInfo { @@ -241,4 +243,5 @@ pub(crate) struct TxnMetaState { pub(crate) txn_bytes: Option>, pub(crate) receipt_node_bytes: Vec, pub(crate) gas_used: u64, + pub(crate) jumpdest_table: JumpDestTableRaw, } diff --git a/zero_bin/common/Cargo.toml b/zero_bin/common/Cargo.toml index 9ee2ac5fe..f1c37de53 100644 --- a/zero_bin/common/Cargo.toml +++ b/zero_bin/common/Cargo.toml @@ -22,3 +22,4 @@ futures = { workspace = true } tokio = { workspace = true } alloy = { workspace = true } async-stream = { workspace = true } +pretty_assertions = "1.4.0" diff --git a/zero_bin/rpc/Cargo.toml b/zero_bin/rpc/Cargo.toml index 6724c16fe..77e3f5d6f 100644 --- a/zero_bin/rpc/Cargo.toml +++ b/zero_bin/rpc/Cargo.toml @@ -25,8 +25,10 @@ tower = { workspace = true, features = ["retry"] } trace_decoder = { workspace = true } tracing-subscriber = { workspace = true } url = { workspace = true } +evm-disassembler = { workspace = true } # Local dependencies compat = { workspace = true } zero_bin_common = { workspace = true } prover = { workspace = true } +gag = "1.0.0" diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 61bcf5f4a..7b0414c68 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -1,26 +1,30 @@ -use std::collections::{HashMap, HashSet}; +use std::{collections::{BTreeSet, HashMap, HashSet}}; use __compat_primitive_types::{H256, U256}; use alloy::{ - primitives::{keccak256, Address, B256}, + primitives::{fixed_bytes, keccak256, Address, FixedBytes, B256}, providers::{ ext::DebugApi as _, network::{eip2718::Encodable2718, Ethereum, Network}, Provider, }, rpc::types::{ - eth::Transaction, - eth::{AccessList, Block}, + eth::{AccessList, Block, Transaction}, trace::geth::{ - AccountState, DiffMode, GethDebugBuiltInTracerType, GethTrace, PreStateConfig, + AccountState, CallConfig, CallFrame, DefaultFrame, DiffMode, + GethDebugBuiltInTracerType, GethDebugTracerConfig, GethDebugTracerType, + GethDebugTracingOptions, GethDefaultTracingOptions, GethTrace, PreStateConfig, PreStateFrame, PreStateMode, }, - trace::geth::{GethDebugTracerType, GethDebugTracingOptions}, }, transports::Transport, }; use anyhow::Context as _; +use evm_arithmetization::JumpDestTableRaw; +use evm_disassembler::{disassemble_bytes, Opcode}; use futures::stream::{FuturesOrdered, TryStreamExt}; +use gag::Gag; +use serde_json::json; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use super::CodeDb; @@ -63,16 +67,12 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let (tx_receipt, pre_trace, diff_trace) = fetch_tx_data(provider, &tx.hash).await?; + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = + fetch_tx_data(provider, &tx.hash).await?; + let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); - let tx_meta = TxnMeta { - byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), - new_txn_trie_node_byte: vec![], - new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), - gas_used: tx_receipt.gas_used as u64, - }; let (code_db, tx_traces) = match (pre_trace, diff_trace) { ( @@ -82,6 +82,24 @@ where _ => unreachable!(), }; + let jumpdest_table: JumpDestTableRaw = if let GethTrace::Default(structlog_frame) = structlog_trace { + get_jumpdest_table(tx, provider, &structlog_frame, &tx_traces).await? + } else { + Default::default() + }; + + //let test_hash: alloy::primitives::TxHash = + // fixed_bytes!(" + // 9e63085271890a141297039b3b711913699f1ee4db1acb667ad7ce304772036b"); + + let tx_meta = TxnMeta { + byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), + new_txn_trie_node_byte: vec![], + new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), + gas_used: tx_receipt.gas_used as u64, + jumpdest_table, + }; + Ok(( code_db, TxnInfo { @@ -98,7 +116,15 @@ where async fn fetch_tx_data( provider: &ProviderT, tx_hash: &B256, -) -> anyhow::Result<(::ReceiptResponse, GethTrace, GethTrace), anyhow::Error> +) -> anyhow::Result< + ( + ::ReceiptResponse, + GethTrace, + GethTrace, + GethTrace, + ), + anyhow::Error, +> where ProviderT: Provider, TransportT: Transport + Clone, @@ -106,14 +132,39 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); + let k = structlog_tracing_options(); + let structlog_trace_fut = provider.debug_trace_transaction(*tx_hash, k.clone()); + + let structlog_trace = structlog_trace_fut.await; + + let test_hash = + fixed_bytes!("531c6cb3fb57b76826dc1bbb25e4d1aab3c9d4a190f5f31dea23210f05988e40"); + // let test_hash: alloy::primitives::TxHash = + // fixed_bytes!(" + // 9e63085271890a141297039b3b711913699f1ee4db1acb667ad7ce304772036b"); + if tx_hash == &test_hash { + // this could happen in a separate thread returning a future or when + //dbg!(&code_db); + let json1 = serde_json::to_string_pretty(&prestate_tracing_options(true)).unwrap(); + println!("prestate: {}", json1); + let json = serde_json::to_string_pretty(&k).unwrap(); + println!("{}", json); + }; let (tx_receipt, pre_trace, diff_trace) = - futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; + futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut)?; + + if tx_hash == &test_hash { + dbg!(structlog_trace?); + panic!(); + }; Ok(( tx_receipt.context("Transaction receipt not found.")?, pre_trace, + diff_trace.clone(), diff_trace, + // structlog_trace, )) } @@ -163,6 +214,7 @@ async fn process_tx_traces( let balance = post_state.and_then(|x| x.balance.map(Compat::compat)); let (storage_read, storage_written) = process_storage( + // Q: doesn't this remove the address from warm addresses? access_list.remove(&address).unwrap_or_default(), read_state, post_state, @@ -183,7 +235,6 @@ async fn process_tx_traces( traces.insert(address, result); } - Ok((code_db, traces)) } @@ -270,7 +321,6 @@ async fn process_code( (_, Some(read_code)) => { let code_hash = keccak256(read_code).compat(); code_db.insert(code_hash, read_code.to_vec()); - Some(ContractCodeUsage::Read(code_hash)) } _ => None, @@ -340,3 +390,129 @@ fn prestate_tracing_options(diff_mode: bool) -> GethDebugTracingOptions { ..GethDebugTracingOptions::default() } } + +/// Tracing options for the debug_traceTransaction call. +fn structlog_tracing_options() -> GethDebugTracingOptions { + GethDebugTracingOptions { + // config: GethDefaultTracingOptions { + // disable_memory: Some(true), + // disable_stack: Some(true), + // disable_storage: Some(true), + // ..GethDefaultTracingOptions::default() + // }, + tracer: None, + // tracer_config: GethDebugTracerConfig(serde_json::Value::Null), + ..GethDebugTracingOptions::default() + } +} + +async fn get_jumpdest_table( + tx: &Transaction, + provider: &ProviderT, + structlog_trace: &DefaultFrame, + tx_traces: &HashMap, +) -> anyhow::Result +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let mut jumpdest_table = JumpDestTableRaw::default(); + + let test_hash: alloy::primitives::TxHash = + fixed_bytes!("531c6cb3fb57b76826dc1bbb25e4d1aab3c9d4a190f5f31dea23210f05988e40"); + if tx.hash == test_hash { + // eprintln!("({}: {:?})", context_next-1, op); + eprintln!("SDFDSAFDSAFS"); + dbg!(structlog_trace); + //panic!("done"); + } + + // A mapping between an Address and its Code section. + let mut addr_to_code_hash = HashMap::::default(); + + for (&a, t) in tx_traces { + if let Some(code) = &t.code_usage { + addr_to_code_hash.insert(a, code.get_code_hash()); + } + }; + + let to_address: Address = tx.to.unwrap(); + let curr_code_hash: H256 = addr_to_code_hash[&to_address]; + + + // the next available context. Increments only. + let mut next_ctx = 1; + // use context 1; + let mut stack = vec![(curr_code_hash, next_ctx)]; + next_ctx += 1; + + for entry in structlog_trace.struct_logs.iter() { + debug_assert!(entry.depth as usize <= next_ctx); + // Question: should/does trace decoder provide an Enum variant for this? + match entry.op.as_str() { + "JUMPDEST" => { + let (code, ctx) = stack[stack.len() - 1]; + // add_jumpdest_offset(&mut jumpdest_table, ctx, entry.pc as usize) + //let e = jumpdest_table.entry(code).or_insert_with(ContextTable::default()). + // entry(ctx).or_insert(vec![]); + jumpdest_table + .0 + .entry(code) + .or_default() + .0 + .entry(ctx) + .or_default() + .insert(entry.pc as usize); + } + "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { + let callee_address = { + // This is the same stack index for all four opcodes. See https://ethervm.io/#F1 + let callee_addr_stack_index = 1; + // TODO: is there a more elegant way to do this? + let callee_raw = entry.stack.as_ref().expect("No stack found in structLog.") + [callee_addr_stack_index]; + // let b = U256::from(callee_addr); + let callee_bytes = FixedBytes::<32>::new(callee_raw.to_le_bytes()); + let callee_address = Address::from_word(callee_bytes); + dbg!(&callee_raw); + dbg!(&callee_address); + callee_address + }; + // TODO: check for calls to preocompiles + let code_hash = addr_to_code_hash[&callee_address]; + stack.push((code_hash, next_ctx)); + next_ctx += 1; + } + "RETURN" => { + stack.pop().expect("Call stack was empty at pop"); + } + _ => (), + } + } + + if tx.hash == test_hash { + //eprintln!("({}: {:?})", context_next-1, op); + dbg!(&jumpdest_table); + panic!(); + } + // } + // println!("JumpDest Table for tx {}:", tx.hash); + // for j in &jumpdest_table { + // println!("{:?}", j); + // } + Ok(jumpdest_table) +} + +// Non-member version of [`evm_arithmetization/src/cpu/kernel/interpreter.rs`] +// todo: should probably be merged +// pub(crate) fn add_jumpdest_offset( +// jumpdest_table: &mut JumpDestTableRaw, +// context: usize, +// offset: usize, +// ) { +// if let Some(context_set) = jumpdest_table.0.get_mut(&context) { +// context_set.insert(offset); +// } else { +// jumpdest_table.0.insert(context, BTreeSet::from([offset])); +// } +// } diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index 76af3b0dc..1a91436bc 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -33,7 +33,7 @@ export RUST_BACKTRACE=full export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 @@ -89,6 +89,7 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then + echo $RUSTFLAGS cargo run --release --features test_only --bin leader -- --runtime in-memory --load-strategy on-demand stdio < $INPUT_FILE | tee $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof"