From 7f7b1a67818676de7f5c81cb57b14d24171819fa Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 12 Nov 2024 17:22:25 +0900 Subject: [PATCH 1/7] feat: wip data imex initial --- Cargo.lock | 20 +++++ Cargo.toml | 2 +- core/bin/data_imex/Cargo.toml | 27 ++++++ core/bin/data_imex/src/main.rs | 151 +++++++++++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 core/bin/data_imex/Cargo.toml create mode 100644 core/bin/data_imex/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 04a863448d69..e08b12038084 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2373,6 +2373,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "data_imex" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "clap 4.5.20", + "sqlx", + "tokio", + "tracing", + "zksync_config", + "zksync_contract_verifier_lib", + "zksync_core_leftovers", + "zksync_dal", + "zksync_queued_job_processor", + "zksync_types", + "zksync_utils", + "zksync_vlog", +] + [[package]] name = "debugid" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index e491c64605bc..89ab3d885f1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,7 +78,7 @@ members = [ # Test infrastructure "core/tests/test_account", "core/tests/loadnext", - "core/tests/vm-benchmark", + "core/tests/vm-benchmark", "core/bin/data_imex", ] resolver = "2" diff --git a/core/bin/data_imex/Cargo.toml b/core/bin/data_imex/Cargo.toml new file mode 100644 index 000000000000..cf8cf907460e --- /dev/null +++ b/core/bin/data_imex/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "data_imex" +version.workspace = true +edition.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +zksync_dal.workspace = true +zksync_config = { workspace = true, features = ["observability_ext"] } +zksync_contract_verifier_lib.workspace = true +zksync_queued_job_processor.workspace = true +zksync_types.workspace = true +zksync_utils.workspace = true +zksync_vlog.workspace = true +zksync_core_leftovers.workspace = true + +anyhow.workspace = true +bincode.workspace = true +clap = { workspace = true, features = ["derive"] } +sqlx.workspace = true +tokio = { workspace = true, features = ["full"] } +tracing.workspace = true diff --git a/core/bin/data_imex/src/main.rs b/core/bin/data_imex/src/main.rs new file mode 100644 index 000000000000..f9ff964f669b --- /dev/null +++ b/core/bin/data_imex/src/main.rs @@ -0,0 +1,151 @@ +use std::collections::HashMap; + +use sqlx::prelude::*; +use sqlx::PgConnection; +use sqlx::QueryBuilder; +use zksync_types::{AccountTreeId, StorageKey, StorageLog, StorageLogKind, H256}; + +const BIND_LIMIT: usize = 65535; // postgres limit = 65535 + +static SOURCE_DATABASE_URL: &str = + "postgres://postgres:notsecurepassword@localhost:5432/zksync_server_localhost_era"; +static DESTINATION_DATABASE_URL: &str = + "postgres://postgres:notsecurepassword@localhost:5432/zksync_server_localhost_imex_destination"; + +type FactoryDeps = HashMap>; + +#[tokio::main] +async fn main() { + println!("Connecting to source database..."); + let mut conn_source = PgConnection::connect(SOURCE_DATABASE_URL).await.unwrap(); + println!("Connected to source database."); + + println!("Reading initial writes..."); + #[derive(FromRow)] + struct InitialWriteRow { + hashed_key: [u8; 32], + l1_batch_number: i32, + index: i32, + } + let initial_writes = sqlx::query_as::<_, InitialWriteRow>( + "select hashed_key, l1_batch_number, index from initial_writes;", + ) + .fetch_all(&mut conn_source) + .await + .unwrap(); + println!( + "Loaded {} initial writes from source database.", + initial_writes.len(), + ); + + println!("Reading storage logs..."); + let storage_logs = sqlx::query(r#" + select distinct iw.hashed_key, + address, + key, + last_value(value) over (partition by iw.hashed_key order by miniblock_number) as value + from initial_writes iw + join storage_logs sl on sl.hashed_key = iw.hashed_key; + "#) + .fetch_all(&mut conn_source) + .await + .unwrap() + .into_iter() + .map(|r| StorageLog { + kind: StorageLogKind::InitialWrite, + key: StorageKey::new( + AccountTreeId::from_fixed_bytes(r.get("address")), + H256(r.get("key")), + ), + value: H256(r.get("value")), + }) + .collect::>(); + println!( + "Loaded {} storage logs from source database.", + storage_logs.len(), + ); + + println!("Loading factory deps from source database..."); + let factory_deps: FactoryDeps = + sqlx::query("select bytecode_hash, bytecode from factory_deps fd;") + .fetch_all(&mut conn_source) + .await + .unwrap() + .into_iter() + .map(|r| (H256(r.get("bytecode_hash")), r.get("bytecode"))) + .collect(); + println!( + "Loaded {} factory deps from source database.", + factory_deps.len(), + ); + + conn_source.close().await.unwrap(); + + println!("Connecting to destination database..."); + let mut conn_destination = PgConnection::connect(DESTINATION_DATABASE_URL) + .await + .unwrap(); + println!("Connected to destination database."); + + // insert initial writes + const IW_BINDINGS_PER_BATCH: usize = 3; + let iw_batches = initial_writes.chunks(BIND_LIMIT / IW_BINDINGS_PER_BATCH); + println!( + "Copying initial writes to destination in {} batches...", + iw_batches.len(), + ); + + let mut total_iw_insertions = 0; + for (i, batch) in iw_batches.enumerate() { + let mut q = QueryBuilder::new( + r#" + insert into initial_writes(hashed_key, l1_batch_number, index) + "#, + ); + + q.push_values(batch, |mut args_list, value| { + args_list + .push_bind(value.hashed_key) + .push_bind(value.l1_batch_number) // TODO: should the batch number reset to zero or should it be copied over? + .push_bind(value.index); + }); + + q.build().execute(&mut conn_destination).await.unwrap(); + total_iw_insertions += batch.len(); + println!("Inserted batch {i} to destination database, for total of {total_iw_insertions} initial write records inserted."); + } + println!("Finished inserting initial writes to destination database."); + + // insert storage logs + const SL_BINDINGS_PER_BATCH: usize = 7; + let sl_batches = storage_logs.chunks(BIND_LIMIT / SL_BINDINGS_PER_BATCH); + println!( + "Copying storage logs to destination in {} batches...", + sl_batches.len(), + ); + + let mut total_sl_insertions = 0; + for (i, batch) in sl_batches.enumerate() { + let mut q = QueryBuilder::new( + r#" + insert into storage_logs(hashed_key, address, key, value, operation_number, tx_hash, miniblock_number) + "#, + ); + + q.push_values(batch, |mut args_list, value| { + args_list + .push_bind(value.key.hashed_key().0) + .push_bind(value.key.address().0) + .push_bind(value.key.key().0) + .push_bind(value.value.0) + .push_bind(todo!("operation number") as i32) + .push_bind(todo!("transaction hash") as i32) + .push_bind(todo!("miniblock number") as i32); + }); + + q.build().execute(&mut conn_destination).await.unwrap(); + total_sl_insertions += batch.len(); + println!("Inserted batch {i} to destination database, for total of {total_sl_insertions} storage log records inserted."); + } + println!("Finished inserting storage logs to destination database."); +} From a3591d48610f9dca72ef5b8257d8d4d57ef7c0d2 Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 12 Nov 2024 17:27:39 +0900 Subject: [PATCH 2/7] chore: fmt + zeroes per misha --- core/bin/data_imex/src/main.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/core/bin/data_imex/src/main.rs b/core/bin/data_imex/src/main.rs index f9ff964f669b..c43ec8ba8c84 100644 --- a/core/bin/data_imex/src/main.rs +++ b/core/bin/data_imex/src/main.rs @@ -1,8 +1,6 @@ use std::collections::HashMap; -use sqlx::prelude::*; -use sqlx::PgConnection; -use sqlx::QueryBuilder; +use sqlx::{prelude::*, PgConnection, QueryBuilder}; use zksync_types::{AccountTreeId, StorageKey, StorageLog, StorageLogKind, H256}; const BIND_LIMIT: usize = 65535; // postgres limit = 65535 @@ -106,7 +104,7 @@ async fn main() { q.push_values(batch, |mut args_list, value| { args_list .push_bind(value.hashed_key) - .push_bind(value.l1_batch_number) // TODO: should the batch number reset to zero or should it be copied over? + .push_bind(0i32) // TODO: should the batch number reset to zero or should it be copied over? .push_bind(value.index); }); @@ -138,9 +136,9 @@ async fn main() { .push_bind(value.key.address().0) .push_bind(value.key.key().0) .push_bind(value.value.0) - .push_bind(todo!("operation number") as i32) - .push_bind(todo!("transaction hash") as i32) - .push_bind(todo!("miniblock number") as i32); + .push_bind(0i32) + .push_bind(0i32) + .push_bind(0i32); }); q.build().execute(&mut conn_destination).await.unwrap(); From c37c196a29376a479cc3a07e0a72e396e7aeb900 Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 12 Nov 2024 21:01:39 +0900 Subject: [PATCH 3/7] feat: factory deps in data_imex script --- core/bin/data_imex/src/main.rs | 65 ++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/core/bin/data_imex/src/main.rs b/core/bin/data_imex/src/main.rs index c43ec8ba8c84..3038be1595c9 100644 --- a/core/bin/data_imex/src/main.rs +++ b/core/bin/data_imex/src/main.rs @@ -10,10 +10,10 @@ static SOURCE_DATABASE_URL: &str = static DESTINATION_DATABASE_URL: &str = "postgres://postgres:notsecurepassword@localhost:5432/zksync_server_localhost_imex_destination"; -type FactoryDeps = HashMap>; - #[tokio::main] async fn main() { + // TODO: paginate, bc probably cannot store 25gb in memory + println!("Connecting to source database..."); let mut conn_source = PgConnection::connect(SOURCE_DATABASE_URL).await.unwrap(); println!("Connected to source database."); @@ -22,8 +22,8 @@ async fn main() { #[derive(FromRow)] struct InitialWriteRow { hashed_key: [u8; 32], - l1_batch_number: i32, - index: i32, + l1_batch_number: i64, + index: i64, } let initial_writes = sqlx::query_as::<_, InitialWriteRow>( "select hashed_key, l1_batch_number, index from initial_writes;", @@ -64,14 +64,16 @@ async fn main() { ); println!("Loading factory deps from source database..."); - let factory_deps: FactoryDeps = - sqlx::query("select bytecode_hash, bytecode from factory_deps fd;") + #[derive(FromRow)] + struct FactoryDepRow { + bytecode_hash: [u8; 32], + bytecode: Vec, + } + let factory_deps: Vec = + sqlx::query_as("select bytecode_hash, bytecode from factory_deps;") .fetch_all(&mut conn_source) .await - .unwrap() - .into_iter() - .map(|r| (H256(r.get("bytecode_hash")), r.get("bytecode"))) - .collect(); + .unwrap(); println!( "Loaded {} factory deps from source database.", factory_deps.len(), @@ -97,7 +99,7 @@ async fn main() { for (i, batch) in iw_batches.enumerate() { let mut q = QueryBuilder::new( r#" - insert into initial_writes(hashed_key, l1_batch_number, index) + insert into initial_writes(hashed_key, l1_batch_number, index, created_at, updated_at) "#, ); @@ -105,7 +107,9 @@ async fn main() { args_list .push_bind(value.hashed_key) .push_bind(0i32) // TODO: should the batch number reset to zero or should it be copied over? - .push_bind(value.index); + .push_bind(value.index) + .push("current_timestamp") + .push("current_timestamp"); }); q.build().execute(&mut conn_destination).await.unwrap(); @@ -126,7 +130,7 @@ async fn main() { for (i, batch) in sl_batches.enumerate() { let mut q = QueryBuilder::new( r#" - insert into storage_logs(hashed_key, address, key, value, operation_number, tx_hash, miniblock_number) + insert into storage_logs(hashed_key, address, key, value, operation_number, tx_hash, miniblock_number, created_at, updated_at) "#, ); @@ -137,8 +141,10 @@ async fn main() { .push_bind(value.key.key().0) .push_bind(value.value.0) .push_bind(0i32) + .push_bind([0u8; 32]) .push_bind(0i32) - .push_bind(0i32); + .push("current_timestamp") + .push("current_timestamp"); }); q.build().execute(&mut conn_destination).await.unwrap(); @@ -146,4 +152,35 @@ async fn main() { println!("Inserted batch {i} to destination database, for total of {total_sl_insertions} storage log records inserted."); } println!("Finished inserting storage logs to destination database."); + + // factory deps + const FD_BINDINGS_PER_BATCH: usize = 7; + let fd_batches = factory_deps.chunks(BIND_LIMIT / FD_BINDINGS_PER_BATCH); + println!( + "Copying factory deps to destination in {} batches...", + fd_batches.len(), + ); + + let mut total_fd_insertions = 0; + for (i, batch) in fd_batches.enumerate() { + let mut q = QueryBuilder::new( + r#" + insert into factory_deps(bytecode_hash, bytecode, miniblock_number, created_at, updated_at) + "#, + ); + + q.push_values(batch, |mut args_list, value| { + args_list + .push_bind(value.bytecode_hash) + .push_bind(&value.bytecode) + .push_bind(0i32) + .push("current_timestamp") + .push("current_timestamp"); + }); + + q.build().execute(&mut conn_destination).await.unwrap(); + total_fd_insertions += batch.len(); + println!("Inserted batch {i} to destination database, for total of {total_fd_insertions} factory deps records inserted."); + } + println!("Finished inserting factory deps to destination database."); } From decfc29ee3ac1f7674c9576e47ce22315c9c6f0a Mon Sep 17 00:00:00 2001 From: Jacob Date: Wed, 13 Nov 2024 13:26:17 +0900 Subject: [PATCH 4/7] fix: warnings --- core/bin/data_imex/src/main.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/bin/data_imex/src/main.rs b/core/bin/data_imex/src/main.rs index 3038be1595c9..5aeea783dc41 100644 --- a/core/bin/data_imex/src/main.rs +++ b/core/bin/data_imex/src/main.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use sqlx::{prelude::*, PgConnection, QueryBuilder}; use zksync_types::{AccountTreeId, StorageKey, StorageLog, StorageLogKind, H256}; @@ -22,7 +20,7 @@ async fn main() { #[derive(FromRow)] struct InitialWriteRow { hashed_key: [u8; 32], - l1_batch_number: i64, + // l1_batch_number: i64, index: i64, } let initial_writes = sqlx::query_as::<_, InitialWriteRow>( From 9d58b54edfee9e0a0b825c27798d1b9e7a824613 Mon Sep 17 00:00:00 2001 From: Jacob Date: Wed, 13 Nov 2024 15:10:15 +0900 Subject: [PATCH 5/7] feat: produce export bin file --- Cargo.lock | 31 ++-- Cargo.toml | 3 +- core/bin/custom_genesis_export/.gitignore | 1 + core/bin/custom_genesis_export/Cargo.toml | 22 +++ core/bin/custom_genesis_export/src/main.rs | 144 ++++++++++++++++ core/bin/data_imex/Cargo.toml | 27 --- core/bin/data_imex/src/main.rs | 184 --------------------- 7 files changed, 180 insertions(+), 232 deletions(-) create mode 100644 core/bin/custom_genesis_export/.gitignore create mode 100644 core/bin/custom_genesis_export/Cargo.toml create mode 100644 core/bin/custom_genesis_export/src/main.rs delete mode 100644 core/bin/data_imex/Cargo.toml delete mode 100644 core/bin/data_imex/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index e08b12038084..b6e968565646 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2215,6 +2215,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "custom_genesis_export" +version = "0.1.0" +dependencies = [ + "bincode", + "clap 4.5.20", + "futures 0.3.31", + "sqlx", + "tokio", +] + [[package]] name = "darling" version = "0.13.4" @@ -2373,26 +2384,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "data_imex" -version = "0.1.0" -dependencies = [ - "anyhow", - "bincode", - "clap 4.5.20", - "sqlx", - "tokio", - "tracing", - "zksync_config", - "zksync_contract_verifier_lib", - "zksync_core_leftovers", - "zksync_dal", - "zksync_queued_job_processor", - "zksync_types", - "zksync_utils", - "zksync_vlog", -] - [[package]] name = "debugid" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 89ab3d885f1a..7f08eab249a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ # Binaries "core/bin/block_reverter", "core/bin/contract-verifier", + "core/bin/custom_genesis_export", "core/bin/external_node", "core/bin/merkle_tree_consistency_checker", "core/bin/snapshots_creator", @@ -78,7 +79,7 @@ members = [ # Test infrastructure "core/tests/test_account", "core/tests/loadnext", - "core/tests/vm-benchmark", "core/bin/data_imex", + "core/tests/vm-benchmark", ] resolver = "2" diff --git a/core/bin/custom_genesis_export/.gitignore b/core/bin/custom_genesis_export/.gitignore new file mode 100644 index 000000000000..a8a0dcec4472 --- /dev/null +++ b/core/bin/custom_genesis_export/.gitignore @@ -0,0 +1 @@ +*.bin diff --git a/core/bin/custom_genesis_export/Cargo.toml b/core/bin/custom_genesis_export/Cargo.toml new file mode 100644 index 000000000000..a28fe9c13c30 --- /dev/null +++ b/core/bin/custom_genesis_export/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "custom_genesis_export" +version.workspace = true +edition.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +bincode.workspace = true +clap = { workspace = true, features = ["derive"] } +futures.workspace = true +sqlx = { workspace = true, features = [ + "runtime-tokio", + "tls-native-tls", + "macros", + "postgres", +] } +tokio = { workspace = true, features = ["full"] } diff --git a/core/bin/custom_genesis_export/src/main.rs b/core/bin/custom_genesis_export/src/main.rs new file mode 100644 index 000000000000..f41b3c578375 --- /dev/null +++ b/core/bin/custom_genesis_export/src/main.rs @@ -0,0 +1,144 @@ +use std::{ + fs::File, + io::{BufWriter, Write}, + path::PathBuf, +}; + +use clap::Parser; +use futures::TryStreamExt; +use sqlx::{prelude::*, Connection, PgConnection}; + +#[derive(Debug, Parser)] +#[command(name = "Custom genesis export tool", author = "Matter Labs")] +struct Args { + /// PostgreSQL connection string for the database to export. + #[arg(short, long)] + database_url: Option, + + /// Output file path. + #[arg(short, long, default_value = "gexport.bin")] + output: PathBuf, +} + +#[tokio::main] +async fn main() { + // TODO: paginate, bc probably cannot store 25gb in memory + let args = Args::parse(); + + let mut out = BufWriter::new(File::create_new(&args.output).unwrap()); + + println!( + "Export file: {}", + args.output.canonicalize().unwrap().display(), + ); + + println!("Connecting to source database..."); + let mut conn_source = + PgConnection::connect(&args.database_url.or_else(|| std::env::var("DATABASE_URL").ok()).expect("Specify the database connection string in either a CLI argument or in the DATABASE_URL environment variable.")) + .await + .unwrap(); + println!("Connected to source database."); + + println!("Reading initial writes..."); + #[derive(FromRow)] + struct InitialWriteRow { + hashed_key: [u8; 32], + index: i64, + } + let count_initial_writes: i64 = sqlx::query("select count(*) from initial_writes;") + .fetch_one(&mut conn_source) + .await + .unwrap() + .get(0); + let mut initial_writes = + sqlx::query_as::<_, InitialWriteRow>("select hashed_key, index from initial_writes;") + .fetch(&mut conn_source); + + // write count of initial writes + out.write_all(&i64::to_le_bytes(count_initial_writes)) + .unwrap(); + let mut actual_initial_writes_count = 0; + while let Some(r) = initial_writes.try_next().await.unwrap() { + out.write_all(&r.hashed_key).unwrap(); + out.write_all(&r.index.to_le_bytes()).unwrap(); + actual_initial_writes_count += 1; + } + if actual_initial_writes_count != count_initial_writes { + panic!("Database reported {count_initial_writes} initial writes; only received {actual_initial_writes_count} for export."); + } + drop(initial_writes); + + println!("Exported {count_initial_writes} initial writes."); + + println!("Reading storage logs..."); + #[derive(FromRow)] + struct StorageLogRow { + address: [u8; 20], + key: [u8; 32], + value: [u8; 32], + } + let count_storage_logs: i64 = + sqlx::query("select count(distinct hashed_key) from storage_logs;") + .fetch_one(&mut conn_source) + .await + .unwrap() + .get(0); + let mut storage_logs = sqlx::query_as::<_, StorageLogRow>( + r#" + select address, key, value + from storage_logs sl + where miniblock_number = ( + select max(miniblock_number) + from storage_logs + where hashed_key = sl.hashed_key + );"#, + ) + .fetch(&mut conn_source); + + let mut actual_storage_logs_count = 0; + while let Some(r) = storage_logs.try_next().await.unwrap() { + out.write_all(&r.address).unwrap(); + out.write_all(&r.key).unwrap(); + out.write_all(&r.value).unwrap(); + actual_storage_logs_count += 1; + } + if actual_storage_logs_count != count_storage_logs { + panic!("Retrieved {actual_storage_logs_count} storage logs from the database; expected {count_storage_logs}."); + } + drop(storage_logs); + + println!("Exported {count_storage_logs} storage logs from source database."); + + println!("Loading factory deps from source database..."); + #[derive(FromRow)] + struct FactoryDepRow { + bytecode_hash: [u8; 32], + bytecode: Vec, + } + let count_factory_deps: i64 = sqlx::query("select count(*) from factory_deps;") + .fetch_one(&mut conn_source) + .await + .unwrap() + .get(0); + let mut factory_deps = + sqlx::query_as::<_, FactoryDepRow>("select bytecode_hash, bytecode from factory_deps;") + .fetch(&mut conn_source); + + let mut actual_factory_deps_count = 0; + while let Some(r) = factory_deps.try_next().await.unwrap() { + out.write_all(&r.bytecode_hash).unwrap(); + out.write_all(&bincode::serialize(&r.bytecode).unwrap()) + .unwrap(); + actual_factory_deps_count += 1; + } + if actual_factory_deps_count != count_factory_deps { + panic!("Retrieved {actual_factory_deps_count} factory deps from the database; expected {count_factory_deps}."); + } + drop(factory_deps); + + println!("Exported {count_factory_deps} factory deps from source database."); + + conn_source.close().await.unwrap(); + + println!("Done."); +} diff --git a/core/bin/data_imex/Cargo.toml b/core/bin/data_imex/Cargo.toml deleted file mode 100644 index cf8cf907460e..000000000000 --- a/core/bin/data_imex/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "data_imex" -version.workspace = true -edition.workspace = true -authors.workspace = true -homepage.workspace = true -repository.workspace = true -license.workspace = true -keywords.workspace = true -categories.workspace = true - -[dependencies] -zksync_dal.workspace = true -zksync_config = { workspace = true, features = ["observability_ext"] } -zksync_contract_verifier_lib.workspace = true -zksync_queued_job_processor.workspace = true -zksync_types.workspace = true -zksync_utils.workspace = true -zksync_vlog.workspace = true -zksync_core_leftovers.workspace = true - -anyhow.workspace = true -bincode.workspace = true -clap = { workspace = true, features = ["derive"] } -sqlx.workspace = true -tokio = { workspace = true, features = ["full"] } -tracing.workspace = true diff --git a/core/bin/data_imex/src/main.rs b/core/bin/data_imex/src/main.rs deleted file mode 100644 index 5aeea783dc41..000000000000 --- a/core/bin/data_imex/src/main.rs +++ /dev/null @@ -1,184 +0,0 @@ -use sqlx::{prelude::*, PgConnection, QueryBuilder}; -use zksync_types::{AccountTreeId, StorageKey, StorageLog, StorageLogKind, H256}; - -const BIND_LIMIT: usize = 65535; // postgres limit = 65535 - -static SOURCE_DATABASE_URL: &str = - "postgres://postgres:notsecurepassword@localhost:5432/zksync_server_localhost_era"; -static DESTINATION_DATABASE_URL: &str = - "postgres://postgres:notsecurepassword@localhost:5432/zksync_server_localhost_imex_destination"; - -#[tokio::main] -async fn main() { - // TODO: paginate, bc probably cannot store 25gb in memory - - println!("Connecting to source database..."); - let mut conn_source = PgConnection::connect(SOURCE_DATABASE_URL).await.unwrap(); - println!("Connected to source database."); - - println!("Reading initial writes..."); - #[derive(FromRow)] - struct InitialWriteRow { - hashed_key: [u8; 32], - // l1_batch_number: i64, - index: i64, - } - let initial_writes = sqlx::query_as::<_, InitialWriteRow>( - "select hashed_key, l1_batch_number, index from initial_writes;", - ) - .fetch_all(&mut conn_source) - .await - .unwrap(); - println!( - "Loaded {} initial writes from source database.", - initial_writes.len(), - ); - - println!("Reading storage logs..."); - let storage_logs = sqlx::query(r#" - select distinct iw.hashed_key, - address, - key, - last_value(value) over (partition by iw.hashed_key order by miniblock_number) as value - from initial_writes iw - join storage_logs sl on sl.hashed_key = iw.hashed_key; - "#) - .fetch_all(&mut conn_source) - .await - .unwrap() - .into_iter() - .map(|r| StorageLog { - kind: StorageLogKind::InitialWrite, - key: StorageKey::new( - AccountTreeId::from_fixed_bytes(r.get("address")), - H256(r.get("key")), - ), - value: H256(r.get("value")), - }) - .collect::>(); - println!( - "Loaded {} storage logs from source database.", - storage_logs.len(), - ); - - println!("Loading factory deps from source database..."); - #[derive(FromRow)] - struct FactoryDepRow { - bytecode_hash: [u8; 32], - bytecode: Vec, - } - let factory_deps: Vec = - sqlx::query_as("select bytecode_hash, bytecode from factory_deps;") - .fetch_all(&mut conn_source) - .await - .unwrap(); - println!( - "Loaded {} factory deps from source database.", - factory_deps.len(), - ); - - conn_source.close().await.unwrap(); - - println!("Connecting to destination database..."); - let mut conn_destination = PgConnection::connect(DESTINATION_DATABASE_URL) - .await - .unwrap(); - println!("Connected to destination database."); - - // insert initial writes - const IW_BINDINGS_PER_BATCH: usize = 3; - let iw_batches = initial_writes.chunks(BIND_LIMIT / IW_BINDINGS_PER_BATCH); - println!( - "Copying initial writes to destination in {} batches...", - iw_batches.len(), - ); - - let mut total_iw_insertions = 0; - for (i, batch) in iw_batches.enumerate() { - let mut q = QueryBuilder::new( - r#" - insert into initial_writes(hashed_key, l1_batch_number, index, created_at, updated_at) - "#, - ); - - q.push_values(batch, |mut args_list, value| { - args_list - .push_bind(value.hashed_key) - .push_bind(0i32) // TODO: should the batch number reset to zero or should it be copied over? - .push_bind(value.index) - .push("current_timestamp") - .push("current_timestamp"); - }); - - q.build().execute(&mut conn_destination).await.unwrap(); - total_iw_insertions += batch.len(); - println!("Inserted batch {i} to destination database, for total of {total_iw_insertions} initial write records inserted."); - } - println!("Finished inserting initial writes to destination database."); - - // insert storage logs - const SL_BINDINGS_PER_BATCH: usize = 7; - let sl_batches = storage_logs.chunks(BIND_LIMIT / SL_BINDINGS_PER_BATCH); - println!( - "Copying storage logs to destination in {} batches...", - sl_batches.len(), - ); - - let mut total_sl_insertions = 0; - for (i, batch) in sl_batches.enumerate() { - let mut q = QueryBuilder::new( - r#" - insert into storage_logs(hashed_key, address, key, value, operation_number, tx_hash, miniblock_number, created_at, updated_at) - "#, - ); - - q.push_values(batch, |mut args_list, value| { - args_list - .push_bind(value.key.hashed_key().0) - .push_bind(value.key.address().0) - .push_bind(value.key.key().0) - .push_bind(value.value.0) - .push_bind(0i32) - .push_bind([0u8; 32]) - .push_bind(0i32) - .push("current_timestamp") - .push("current_timestamp"); - }); - - q.build().execute(&mut conn_destination).await.unwrap(); - total_sl_insertions += batch.len(); - println!("Inserted batch {i} to destination database, for total of {total_sl_insertions} storage log records inserted."); - } - println!("Finished inserting storage logs to destination database."); - - // factory deps - const FD_BINDINGS_PER_BATCH: usize = 7; - let fd_batches = factory_deps.chunks(BIND_LIMIT / FD_BINDINGS_PER_BATCH); - println!( - "Copying factory deps to destination in {} batches...", - fd_batches.len(), - ); - - let mut total_fd_insertions = 0; - for (i, batch) in fd_batches.enumerate() { - let mut q = QueryBuilder::new( - r#" - insert into factory_deps(bytecode_hash, bytecode, miniblock_number, created_at, updated_at) - "#, - ); - - q.push_values(batch, |mut args_list, value| { - args_list - .push_bind(value.bytecode_hash) - .push_bind(&value.bytecode) - .push_bind(0i32) - .push("current_timestamp") - .push("current_timestamp"); - }); - - q.build().execute(&mut conn_destination).await.unwrap(); - total_fd_insertions += batch.len(); - println!("Inserted batch {i} to destination database, for total of {total_fd_insertions} factory deps records inserted."); - } - println!("Finished inserting factory deps to destination database."); -} From 8bc82728da71265dca41ac9bb3502dbc2d5e8cf4 Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 15 Nov 2024 14:47:47 +0900 Subject: [PATCH 6/7] feat: custom genesis file reader --- core/bin/custom_genesis_export/src/main.rs | 3 +- core/node/genesis/src/export.rs | 179 ++++++++++++++++++ core/node/genesis/src/lib.rs | 88 +++++++-- core/node/genesis/src/utils.rs | 39 +++- .../src/commands/chain/args/genesis.rs | 2 + .../src/commands/chain/args/init/mod.rs | 1 + .../src/commands/ecosystem/args/init.rs | 1 + 7 files changed, 294 insertions(+), 19 deletions(-) create mode 100644 core/node/genesis/src/export.rs diff --git a/core/bin/custom_genesis_export/src/main.rs b/core/bin/custom_genesis_export/src/main.rs index f41b3c578375..86239731c2ad 100644 --- a/core/bin/custom_genesis_export/src/main.rs +++ b/core/bin/custom_genesis_export/src/main.rs @@ -127,8 +127,9 @@ async fn main() { let mut actual_factory_deps_count = 0; while let Some(r) = factory_deps.try_next().await.unwrap() { out.write_all(&r.bytecode_hash).unwrap(); - out.write_all(&bincode::serialize(&r.bytecode).unwrap()) + out.write_all(&(r.bytecode.len() as u64).to_le_bytes()) .unwrap(); + out.write_all(&r.bytecode).unwrap(); actual_factory_deps_count += 1; } if actual_factory_deps_count != count_factory_deps { diff --git a/core/node/genesis/src/export.rs b/core/node/genesis/src/export.rs new file mode 100644 index 000000000000..6723936c46bf --- /dev/null +++ b/core/node/genesis/src/export.rs @@ -0,0 +1,179 @@ +use std::{ + fs::File, + io::{BufRead, BufReader, Seek, SeekFrom}, + marker::PhantomData, + os::unix::fs::FileExt, +}; + +use zksync_types::{Address, H160, H256}; + +#[derive(Debug, Clone)] +pub struct InitialWriteExport { + pub hashed_key: H256, + pub index: usize, +} + +#[derive(Debug, Clone)] +pub struct StorageLogExport { + pub address: Address, + pub key: H256, + pub value: H256, +} + +#[derive(Debug, Clone)] +pub struct FactoryDepExport { + pub bytecode_hash: H256, + pub bytecode: Vec, +} + +#[derive(Debug)] +pub struct GenesisExportReader { + file: File, + initial_writes_offset: usize, + initial_writes_count: usize, + storage_logs_offset: usize, + storage_logs_count: usize, + factory_deps_offset: usize, + factory_deps_count: usize, +} + +fn randreadn(file: &File, at: usize) -> [u8; N] { + let mut buf = [0u8; N]; + file.read_exact_at(&mut buf, at as u64).unwrap(); + buf +} + +impl GenesisExportReader { + const INITIAL_WRITE_EXPORT_SIZE: usize = 32 + 8; + const STORAGE_LOG_EXPORT_SIZE: usize = 20 + 32 + 32; + + pub fn new(file: File) -> Self { + let initial_writes_count = usize::from_le_bytes(randreadn(&file, 0)); + let initial_writes_offset: usize = 8; + let storage_logs_count_offset = + initial_writes_offset + Self::INITIAL_WRITE_EXPORT_SIZE * initial_writes_count; + let storage_logs_count = usize::from_le_bytes(randreadn(&file, storage_logs_count_offset)); + let storage_logs_offset = storage_logs_count_offset + 8; + let factory_deps_count_offset = + storage_logs_offset + Self::STORAGE_LOG_EXPORT_SIZE * storage_logs_count; + let factory_deps_count = usize::from_le_bytes(randreadn(&file, factory_deps_count_offset)); + let factory_deps_offset = factory_deps_count_offset + 8; + + Self { + file, + initial_writes_count, + initial_writes_offset, + storage_logs_count, + storage_logs_offset, + factory_deps_count, + factory_deps_offset, + } + } + + pub fn initial_writes(&self) -> ExportItemReader { + let mut buf_reader = BufReader::new(&self.file); + buf_reader + .seek(SeekFrom::Start(self.initial_writes_offset as u64)) + .unwrap(); + + ExportItemReader::new(buf_reader, self.initial_writes_count) + } + + pub fn storage_logs(&self) -> ExportItemReader { + let mut buf_reader = BufReader::new(&self.file); + buf_reader + .seek(SeekFrom::Start(self.storage_logs_offset as u64)) + .unwrap(); + + ExportItemReader::new(buf_reader, self.storage_logs_count) + } + + pub fn factory_deps(&self) -> ExportItemReader { + let mut buf_reader = BufReader::new(&self.file); + buf_reader + .seek(SeekFrom::Start(self.factory_deps_offset as u64)) + .unwrap(); + + ExportItemReader::new(buf_reader, self.factory_deps_count) + } +} + +fn readn(reader: &mut impl BufRead) -> [u8; N] { + let mut buf = [0u8; N]; + reader.read_exact(&mut buf).unwrap(); + buf +} + +trait ExportItem { + fn read(reader: &mut impl BufRead) -> Self; +} + +#[derive(Debug)] +pub struct ExportItemReader<'a, T> { + _item: PhantomData, + reader: BufReader<&'a File>, + count: usize, + next_index: usize, +} + +impl<'a, T> ExportItemReader<'a, T> { + pub fn new(reader: BufReader<&'a File>, count: usize) -> Self { + Self { + _item: PhantomData, + next_index: 0, + count, + reader, + } + } +} + +impl<'a, T: ExportItem> Iterator for ExportItemReader<'a, T> { + type Item = T; + + fn next(&mut self) -> Option { + if self.next_index >= self.count { + return None; + } + + self.next_index += 1; + + Some(Self::Item::read(&mut self.reader)) + } +} + +impl ExportItem for InitialWriteExport { + fn read(reader: &mut impl BufRead) -> Self { + let hashed_key = H256(readn(reader)); + let index = i64::from_le_bytes(readn(reader)) as usize; + + Self { hashed_key, index } + } +} + +impl ExportItem for StorageLogExport { + fn read(reader: &mut impl BufRead) -> Self { + let address = H160(readn(reader)); + let key = H256(readn(reader)); + let value = H256(readn(reader)); + + Self { + address, + key, + value, + } + } +} + +impl ExportItem for FactoryDepExport { + fn read(reader: &mut impl BufRead) -> Self { + let bytecode_hash = H256(readn(reader)); + let bytecode_len = u64::from_le_bytes(readn(reader)) as usize; + let mut bytecode = vec![0u8; bytecode_len]; + reader.read_exact(&mut bytecode).unwrap(); + + Self { + bytecode_hash, + bytecode, + } + } +} diff --git a/core/node/genesis/src/lib.rs b/core/node/genesis/src/lib.rs index 82732342b407..a5abb182ae47 100644 --- a/core/node/genesis/src/lib.rs +++ b/core/node/genesis/src/lib.rs @@ -2,9 +2,10 @@ //! It initializes the Merkle tree with the basic setup (such as fields of special service accounts), //! setups the required databases, and outputs the data required to initialize a smart contract. -use std::fmt::Formatter; +use std::{collections::HashMap, fmt::Formatter, fs::File, io::BufReader}; use anyhow::Context as _; +use export::GenesisExportReader; use zksync_config::GenesisConfig; use zksync_contracts::{ hyperchain_contract, verifier_contract, BaseSystemContracts, BaseSystemContractsHashes, @@ -24,7 +25,7 @@ use zksync_types::{ system_contracts::get_system_smart_contracts, web3::{BlockNumber, FilterBuilder}, AccountTreeId, Address, Bloom, L1BatchNumber, L1ChainId, L2BlockNumber, L2ChainId, - ProtocolVersion, ProtocolVersionId, StorageKey, H256, U256, + ProtocolVersion, ProtocolVersionId, StorageKey, StorageLog, H256, U256, }; use zksync_utils::{bytecode::hash_bytecode, u256_to_h256}; @@ -34,6 +35,7 @@ use crate::utils::{ save_genesis_l1_batch_metadata, }; +mod export; #[cfg(test)] mod tests; mod utils; @@ -74,10 +76,11 @@ pub enum GenesisError { MalformedConfig(&'static str), } -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct GenesisParams { base_system_contracts: BaseSystemContracts, system_contracts: Vec, + genesis_export_reader: Option, config: GenesisConfig, } @@ -92,6 +95,33 @@ impl GenesisParams { &self.config } + pub fn storage_logs(&self) -> Vec { + let mut storage_logs = get_storage_logs(&self.system_contracts); + if let Some(ref e) = self.genesis_export_reader { + // TODO: This loads all of the exported storage logs into memory at once. + storage_logs.extend(e.storage_logs().map(|s| { + StorageLog::new_write_log( + StorageKey::new(AccountTreeId::new(s.address), s.key), + s.value, + ) + })); + } + storage_logs + } + + pub fn factory_deps(&self) -> HashMap> { + let s = self + .system_contracts + .iter() + .map(|c| (hash_bytecode(&c.bytecode), c.bytecode.clone())); // TODO: Optimize. The call to `get_storage_logs` in the `storage_logs` implementation calls `hash_bytecode` on all of the system contracts already; there should be some way to avoid duplicating all of that work. + if let Some(ref e) = self.genesis_export_reader { + s.chain(e.factory_deps().map(|f| (f.bytecode_hash, f.bytecode))) + .collect() + } else { + s.collect() + } + } + pub fn from_genesis_config( config: GenesisConfig, base_system_contracts: BaseSystemContracts, @@ -117,9 +147,15 @@ impl GenesisParams { if config.protocol_version.is_none() { return Err(GenesisError::MalformedConfig("protocol_version")); } + let genesis_export_reader = std::env::var("CUSTOM_GENESIS").ok().map(|path| { + GenesisExportReader::new( + File::open(path).expect("custom genesis file could not be opened"), + ) + }); Ok(GenesisParams { base_system_contracts, system_contracts, + genesis_export_reader, config, }) } @@ -137,6 +173,7 @@ impl GenesisParams { Self { base_system_contracts: BaseSystemContracts::load_from_disk(), system_contracts: get_system_smart_contracts(false), + genesis_export_reader: None, config: mock_genesis_config(), } } @@ -197,18 +234,18 @@ pub async fn insert_genesis_batch( snark_wrapper_vk_hash: genesis_params.config.snark_wrapper_vk_hash, }; - create_genesis_l1_batch( + create_genesis_l1_batch_from_storage_logs_and_factory_deps( &mut transaction, genesis_params.protocol_version(), genesis_params.base_system_contracts(), - genesis_params.system_contracts(), + &genesis_params.storage_logs(), + genesis_params.factory_deps(), verifier_config, ) .await?; tracing::info!("chain_schema_genesis is complete"); - let deduped_log_queries = - get_deduped_log_queries(&get_storage_logs(genesis_params.system_contracts())); + let deduped_log_queries = get_deduped_log_queries(&genesis_params.storage_logs()); let (deduplicated_writes, _): (Vec<_>, Vec<_>) = deduped_log_queries .into_iter() @@ -370,12 +407,12 @@ pub async fn ensure_genesis_state( Ok(root_hash) } -#[allow(clippy::too_many_arguments)] -pub async fn create_genesis_l1_batch( +pub(crate) async fn create_genesis_l1_batch_from_storage_logs_and_factory_deps( storage: &mut Connection<'_, Core>, protocol_version: ProtocolSemanticVersion, base_system_contracts: &BaseSystemContracts, - system_contracts: &[DeployedContract], + storage_logs: &[StorageLog], + factory_deps: HashMap>, l1_verifier_config: L1VerifierConfig, ) -> Result<(), GenesisError> { let version = ProtocolVersion { @@ -442,6 +479,22 @@ pub async fn create_genesis_l1_batch( .mark_l2_blocks_as_executed_in_l1_batch(L1BatchNumber(0)) .await?; + insert_base_system_contracts_to_factory_deps(&mut transaction, base_system_contracts).await?; + insert_system_contracts(&mut transaction, factory_deps, storage_logs).await?; + add_eth_token(&mut transaction).await?; + + transaction.commit().await?; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub async fn create_genesis_l1_batch( + storage: &mut Connection<'_, Core>, + protocol_version: ProtocolSemanticVersion, + base_system_contracts: &BaseSystemContracts, + system_contracts: &[DeployedContract], + l1_verifier_config: L1VerifierConfig, +) -> Result<(), GenesisError> { let storage_logs = get_storage_logs(system_contracts); let factory_deps = system_contracts @@ -449,12 +502,15 @@ pub async fn create_genesis_l1_batch( .map(|c| (hash_bytecode(&c.bytecode), c.bytecode.clone())) .collect(); - insert_base_system_contracts_to_factory_deps(&mut transaction, base_system_contracts).await?; - insert_system_contracts(&mut transaction, factory_deps, &storage_logs).await?; - add_eth_token(&mut transaction).await?; - - transaction.commit().await?; - Ok(()) + create_genesis_l1_batch_from_storage_logs_and_factory_deps( + storage, + protocol_version, + base_system_contracts, + &storage_logs, + factory_deps, + l1_verifier_config, + ) + .await } // Save chain id transaction into the database diff --git a/core/node/genesis/src/utils.rs b/core/node/genesis/src/utils.rs index 6042513537cd..1c1672826d90 100644 --- a/core/node/genesis/src/utils.rs +++ b/core/node/genesis/src/utils.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, io::Read}; use itertools::Itertools; use zksync_contracts::BaseSystemContracts; @@ -14,7 +14,7 @@ use zksync_types::{ get_code_key, get_known_code_key, get_system_context_init_logs, tokens::{TokenInfo, TokenMetadata}, zk_evm_types::{LogQuery, Timestamp}, - AccountTreeId, L1BatchNumber, L2BlockNumber, L2ChainId, StorageKey, StorageLog, H256, + AccountTreeId, L1BatchNumber, L2BlockNumber, L2ChainId, StorageKey, StorageLog, H160, H256, }; use zksync_utils::{be_words_to_bytes, bytecode::hash_bytecode, h256_to_u256, u256_to_h256}; @@ -217,3 +217,38 @@ pub(super) async fn insert_system_contracts( transaction.commit().await?; Ok(()) } + +fn readn(read: &mut dyn Read) -> [u8; N] { + let mut buf = [0u8; N]; + read.read_exact(&mut buf).unwrap(); + buf +} + +pub(crate) fn read_export_file(read: &mut dyn Read) -> () { + let count_initial_writes = i64::from_le_bytes(readn(read)) as usize; + let mut initial_writes = Vec::with_capacity(count_initial_writes); + for _ in 0..count_initial_writes { + let hashed_key = H256(readn(read)); + let index = i64::from_le_bytes(readn(read)) as usize; + initial_writes.push((hashed_key, index)); + } + + let count_storage_logs = i64::from_le_bytes(readn(read)) as usize; + let mut storage_logs = Vec::with_capacity(count_storage_logs); + for _ in 0..count_storage_logs { + let address = H160(readn(read)); + let key = H256(readn(read)); + let value = H256(readn(read)); + storage_logs.push((address, key, value)); + } + + let count_factory_deps = i64::from_le_bytes(readn(read)) as usize; + let mut factory_deps = HashMap::with_capacity(count_factory_deps); + for _ in 0..count_factory_deps { + let bytecode_hash = H256(readn(read)); + let bytecode_len = u64::from_le_bytes(readn(read)) as usize; + let mut bytecode = vec![0u8; bytecode_len]; + read.read_exact(&mut bytecode).unwrap(); + factory_deps.insert(bytecode_hash, bytecode); + } +} diff --git a/zkstack_cli/crates/zkstack/src/commands/chain/args/genesis.rs b/zkstack_cli/crates/zkstack/src/commands/chain/args/genesis.rs index f990cbfd77da..6733b9061274 100644 --- a/zkstack_cli/crates/zkstack/src/commands/chain/args/genesis.rs +++ b/zkstack_cli/crates/zkstack/src/commands/chain/args/genesis.rs @@ -20,6 +20,8 @@ pub struct GenesisArgs { pub server_db_url: Option, #[clap(long, help = MSG_SERVER_DB_NAME_HELP)] pub server_db_name: Option, + // #[clap(long, help = "Run with custom genesis file")] + // pub custom_genesis: Option, #[clap(long, short, help = MSG_USE_DEFAULT_DATABASES_HELP)] pub dev: bool, #[clap(long, short, action)] diff --git a/zkstack_cli/crates/zkstack/src/commands/chain/args/init/mod.rs b/zkstack_cli/crates/zkstack/src/commands/chain/args/init/mod.rs index a5c7a6890ca1..33f48a1f76eb 100644 --- a/zkstack_cli/crates/zkstack/src/commands/chain/args/init/mod.rs +++ b/zkstack_cli/crates/zkstack/src/commands/chain/args/init/mod.rs @@ -44,6 +44,7 @@ impl InitArgs { GenesisArgs { server_db_url: self.server_db_url.clone(), server_db_name: self.server_db_name.clone(), + // custom_genesis: None, dev: self.dev, dont_drop: self.dont_drop, } diff --git a/zkstack_cli/crates/zkstack/src/commands/ecosystem/args/init.rs b/zkstack_cli/crates/zkstack/src/commands/ecosystem/args/init.rs index 09115fd49ba7..ababb10bcdd6 100644 --- a/zkstack_cli/crates/zkstack/src/commands/ecosystem/args/init.rs +++ b/zkstack_cli/crates/zkstack/src/commands/ecosystem/args/init.rs @@ -108,6 +108,7 @@ impl EcosystemInitArgs { GenesisArgs { server_db_url: self.server_db_url.clone(), server_db_name: self.server_db_name.clone(), + // custom_genesis: None, dev: self.dev, dont_drop: self.dont_drop, } From 282c94a27e3d2847190edd93764af99249121297 Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 15 Nov 2024 16:06:10 +0900 Subject: [PATCH 7/7] fix: missing length in output file --- core/bin/custom_genesis_export/src/main.rs | 6 ++++ core/node/genesis/src/export.rs | 34 ++++++++++++++++++++++ core/node/genesis/src/lib.rs | 13 ++++++--- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/core/bin/custom_genesis_export/src/main.rs b/core/bin/custom_genesis_export/src/main.rs index 86239731c2ad..db15ab8cdef5 100644 --- a/core/bin/custom_genesis_export/src/main.rs +++ b/core/bin/custom_genesis_export/src/main.rs @@ -83,6 +83,9 @@ async fn main() { .await .unwrap() .get(0); + out.write_all(&i64::to_le_bytes(count_storage_logs)) + .unwrap(); + let mut storage_logs = sqlx::query_as::<_, StorageLogRow>( r#" select address, key, value @@ -120,6 +123,9 @@ async fn main() { .await .unwrap() .get(0); + out.write_all(&i64::to_le_bytes(count_factory_deps)) + .unwrap(); + let mut factory_deps = sqlx::query_as::<_, FactoryDepRow>("select bytecode_hash, bytecode from factory_deps;") .fetch(&mut conn_source); diff --git a/core/node/genesis/src/export.rs b/core/node/genesis/src/export.rs index 6723936c46bf..5e51585f0cea 100644 --- a/core/node/genesis/src/export.rs +++ b/core/node/genesis/src/export.rs @@ -59,6 +59,8 @@ impl GenesisExportReader { let factory_deps_count = usize::from_le_bytes(randreadn(&file, factory_deps_count_offset)); let factory_deps_offset = factory_deps_count_offset + 8; + eprintln!("Genesis export reader: {initial_writes_count}, {storage_logs_count}, {factory_deps_count}"); + Self { file, initial_writes_count, @@ -177,3 +179,35 @@ impl ExportItem for FactoryDepExport { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_genesis_reader() { + let path = "/Users/jacob/Projects/zksync-era/core/bin/custom_genesis_export/g2.bin"; + let reader = GenesisExportReader::new(File::open(path).unwrap()); + + let mut count_iw = 0; + for _ in reader.initial_writes() { + count_iw += 1; + } + println!("Initial writes: {count_iw}"); + assert_eq!(count_iw, 53); + + let mut count_sl = 0; + for _ in reader.storage_logs() { + count_sl += 1; + } + println!("Storage logs: {count_sl}"); + assert_eq!(count_sl, 53); + + let mut count_fd = 0; + for _ in reader.factory_deps() { + count_fd += 1; + } + println!("Factory dependencies: {count_fd}"); + assert_eq!(count_fd, 26); + } +} diff --git a/core/node/genesis/src/lib.rs b/core/node/genesis/src/lib.rs index a5abb182ae47..ff6228a07732 100644 --- a/core/node/genesis/src/lib.rs +++ b/core/node/genesis/src/lib.rs @@ -98,6 +98,7 @@ impl GenesisParams { pub fn storage_logs(&self) -> Vec { let mut storage_logs = get_storage_logs(&self.system_contracts); if let Some(ref e) = self.genesis_export_reader { + let len = storage_logs.len(); // TODO: This loads all of the exported storage logs into memory at once. storage_logs.extend(e.storage_logs().map(|s| { StorageLog::new_write_log( @@ -105,6 +106,8 @@ impl GenesisParams { s.value, ) })); + + eprintln!("Extended storage logs by {}", storage_logs.len() - len); } storage_logs } @@ -147,11 +150,13 @@ impl GenesisParams { if config.protocol_version.is_none() { return Err(GenesisError::MalformedConfig("protocol_version")); } - let genesis_export_reader = std::env::var("CUSTOM_GENESIS").ok().map(|path| { - GenesisExportReader::new( + eprintln!("About to load custom genesis if specified..."); + let path = "/Users/jacob/Projects/zksync-era/core/bin/custom_genesis_export/g3.bin"; + let genesis_export_reader = //std::env::var("CUSTOM_GENESIS").ok().map(|path| { + Some(GenesisExportReader::new( File::open(path).expect("custom genesis file could not be opened"), - ) - }); + )); + //}); Ok(GenesisParams { base_system_contracts, system_contracts,