From d142483507a512898fe59be2d12afad2d8989263 Mon Sep 17 00:00:00 2001 From: Rob Date: Wed, 4 Dec 2024 17:24:28 -0500 Subject: [PATCH 01/16] remove the orchestrator and examples --- .github/workflows/test-sequencer.yml | 1 - Cargo.lock | 104 +- Cargo.toml | 2 - crates/examples/Cargo.toml | 115 +- crates/examples/combined/all.rs | 164 --- crates/examples/combined/multi-validator.rs | 45 - crates/examples/combined/orchestrator.rs | 32 - crates/examples/combined/types.rs | 32 - crates/examples/combined/validator.rs | 39 - crates/examples/infra/mod.rs | 1129 ----------------- crates/examples/libp2p/all.rs | 64 - crates/examples/libp2p/multi-validator.rs | 45 - crates/examples/libp2p/types.rs | 32 - crates/examples/libp2p/validator.rs | 39 - crates/examples/orchestrator.rs | 31 - crates/examples/push-cdn/README.md | 68 - crates/examples/push-cdn/all.rs | 165 --- crates/examples/push-cdn/broker.rs | 124 -- crates/examples/push-cdn/marshal.rs | 90 -- crates/examples/push-cdn/multi-validator.rs | 45 - crates/examples/push-cdn/types.rs | 31 - crates/examples/push-cdn/validator.rs | 32 - crates/examples/push-cdn/whitelist-adapter.rs | 86 -- crates/examples/src/lib.rs | 1 + crates/hotshot/Cargo.toml | 2 - crates/hotshot/src/traits.rs | 3 +- .../src/traits/networking/combined_network.rs | 3 - .../src/traits/networking/libp2p_network.rs | 374 ++---- .../src/traits/networking/memory_network.rs | 1 - .../src/traits/networking/push_cdn_network.rs | 1 - crates/libp2p-networking/Cargo.toml | 2 - crates/libp2p-networking/src/lib.rs | 2 +- .../src/network/behaviours/dht/mod.rs | 28 +- crates/libp2p-networking/src/network/def.rs | 9 +- crates/libp2p-networking/src/network/mod.rs | 23 +- crates/libp2p-networking/src/network/node.rs | 251 ++-- .../src/network/node/config.rs | 99 +- .../src/network/node/handle.rs | 148 +-- crates/orchestrator/Cargo.toml | 25 - crates/orchestrator/README.md | 5 - crates/orchestrator/api.toml | 104 -- crates/orchestrator/run-config.toml | 68 - crates/orchestrator/src/client.rs | 522 -------- crates/orchestrator/src/lib.rs | 881 ------------- crates/orchestrator/staging-config.toml | 76 -- crates/testing/src/block_builder/random.rs | 2 +- crates/testing/src/test_builder.rs | 9 - crates/testing/tests/tests_1/block_builder.rs | 2 +- crates/testing/tests/tests_1/test_success.rs | 1 - .../tests/tests_1/test_with_failures_2.rs | 2 - .../tests_3/test_with_failures_half_f.rs | 2 - .../tests/tests_4/test_with_failures_f.rs | 1 - .../testing/tests/tests_5/combined_network.rs | 1 - .../tests/tests_5/test_with_failures.rs | 1 - crates/testing/tests/tests_5/timeout.rs | 1 - crates/types/Cargo.toml | 5 - crates/types/src/builder.rs | 20 + crates/types/src/constants.rs | 3 - crates/types/src/hotshot_config_file.rs | 4 - crates/types/src/lib.rs | 5 +- crates/types/src/network.rs | 384 +----- crates/types/src/traits/network.rs | 1 - .../types/src/traits/node_implementation.rs | 3 - 63 files changed, 412 insertions(+), 5178 deletions(-) delete mode 100644 crates/examples/combined/all.rs delete mode 100644 crates/examples/combined/multi-validator.rs delete mode 100644 crates/examples/combined/orchestrator.rs delete mode 100644 crates/examples/combined/types.rs delete mode 100644 crates/examples/combined/validator.rs delete mode 100755 crates/examples/infra/mod.rs delete mode 100644 crates/examples/libp2p/all.rs delete mode 100644 crates/examples/libp2p/multi-validator.rs delete mode 100644 crates/examples/libp2p/types.rs delete mode 100644 crates/examples/libp2p/validator.rs delete mode 100644 crates/examples/orchestrator.rs delete mode 100644 crates/examples/push-cdn/README.md delete mode 100644 crates/examples/push-cdn/all.rs delete mode 100644 crates/examples/push-cdn/broker.rs delete mode 100644 crates/examples/push-cdn/marshal.rs delete mode 100644 crates/examples/push-cdn/multi-validator.rs delete mode 100644 crates/examples/push-cdn/types.rs delete mode 100644 crates/examples/push-cdn/validator.rs delete mode 100644 crates/examples/push-cdn/whitelist-adapter.rs create mode 100644 crates/examples/src/lib.rs delete mode 100644 crates/orchestrator/Cargo.toml delete mode 100644 crates/orchestrator/README.md delete mode 100644 crates/orchestrator/api.toml delete mode 100644 crates/orchestrator/run-config.toml delete mode 100644 crates/orchestrator/src/client.rs delete mode 100644 crates/orchestrator/src/lib.rs delete mode 100644 crates/orchestrator/staging-config.toml create mode 100644 crates/types/src/builder.rs diff --git a/.github/workflows/test-sequencer.yml b/.github/workflows/test-sequencer.yml index 2850764bd8..53518911f2 100644 --- a/.github/workflows/test-sequencer.yml +++ b/.github/workflows/test-sequencer.yml @@ -53,7 +53,6 @@ jobs: hotshot-signature-key = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot-signature-key" } hotshot-stake-table = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot-stake-table" } hotshot-state-prover = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot-state-prover" } - hotshot-orchestrator = { path = "${GITHUB_WORKSPACE}/hotshot/crates/orchestrator" } hotshot-web-server = { path = "${GITHUB_WORKSPACE}/hotshot/crates/web_server" } hotshot-task-impls = { path = "${GITHUB_WORKSPACE}/hotshot/crates/task-impls" } hotshot-testing = { path = "${GITHUB_WORKSPACE}/hotshot/crates/testing" } diff --git a/Cargo.lock b/Cargo.lock index acc94c379c..fb905da911 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1009,12 +1009,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bimap" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" - [[package]] name = "bincode" version = "1.3.3" @@ -1745,27 +1739,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "csv" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" -dependencies = [ - "memchr", -] - [[package]] name = "ctr" version = "0.6.0" @@ -2242,6 +2215,10 @@ dependencies = [ "pin-project-lite 0.2.15", ] +[[package]] +name = "examples" +version = "0.5.79" + [[package]] name = "fastrand" version = "1.9.0" @@ -2873,7 +2850,6 @@ dependencies = [ "async-broadcast", "async-lock 3.4.0", "async-trait", - "bimap", "bincode", "blake3", "cdn-broker", @@ -2897,7 +2873,6 @@ dependencies = [ "primitive-types", "rand 0.8.5", "serde", - "sha2 0.10.8", "time 0.3.37", "tokio", "tracing", @@ -2949,37 +2924,6 @@ dependencies = [ "vbs", ] -[[package]] -name = "hotshot-examples" -version = "0.5.79" -dependencies = [ - "anyhow", - "async-trait", - "cdn-broker", - "cdn-marshal", - "chrono", - "clap", - "futures", - "hotshot", - "hotshot-example-types", - "hotshot-orchestrator", - "hotshot-testing", - "hotshot-types", - "libp2p-networking", - "local-ip-address", - "portpicker", - "rand 0.8.5", - "serde", - "sha2 0.10.8", - "surf-disco", - "time 0.3.37", - "tokio", - "toml", - "tracing", - "tracing-subscriber 0.3.19", - "url", -] - [[package]] name = "hotshot-fakeapi" version = "0.5.79" @@ -3007,28 +2951,6 @@ dependencies = [ "syn 2.0.90", ] -[[package]] -name = "hotshot-orchestrator" -version = "0.5.79" -dependencies = [ - "anyhow", - "async-lock 3.4.0", - "blake3", - "clap", - "csv", - "futures", - "hotshot-types", - "libp2p-identity", - "multiaddr", - "serde", - "surf-disco", - "tide-disco", - "tokio", - "toml", - "tracing", - "vbs", -] - [[package]] name = "hotshot-stake-table" version = "0.5.79" @@ -3151,7 +3073,6 @@ dependencies = [ "bincode", "bitvec", "blake3", - "clap", "committable", "derive_more 1.0.0", "digest 0.10.7", @@ -3164,17 +3085,13 @@ dependencies = [ "jf-utils", "jf-vid", "lazy_static", - "libp2p-identity", "memoize", "mnemonic", - "multiaddr", "primitive-types", "rand 0.8.5", "rand_chacha 0.3.1", "serde", - "serde-inline-default", "serde_bytes", - "serde_json", "sha2 0.10.8", "tagged-base64", "thiserror 2.0.4", @@ -4392,8 +4309,6 @@ dependencies = [ "blake3", "cbor4ii", "delegate", - "derive_builder", - "derive_more 1.0.0", "futures", "hotshot-example-types", "hotshot-types", @@ -6442,17 +6357,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-inline-default" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fb1bedd774187d304179493b0d3c41fbe97b04b14305363f68d2bdf5e47cb9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.90", -] - [[package]] name = "serde_bytes" version = "0.11.15" diff --git a/Cargo.toml b/Cargo.toml index 62b93b4d14..e5b0bbca1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,6 @@ members = [ "crates/hotshot-stake-table", "crates/libp2p-networking", "crates/macros", - "crates/orchestrator", "crates/task", "crates/task-impls", "crates/testing", @@ -120,7 +119,6 @@ cdn-client = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.5.6 cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.5.6" } cdn-marshal = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.5.6" } cdn-proto = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.5.6" } - ### Profiles ### ### Note: these only apply to example executables or tests built from within this crate. They have diff --git a/crates/examples/Cargo.toml b/crates/examples/Cargo.toml index 09e13b8e7d..849ee8305f 100644 --- a/crates/examples/Cargo.toml +++ b/crates/examples/Cargo.toml @@ -1,113 +1,14 @@ [package] -authors = { workspace = true } -description = "HotShot Examples and binaries" -edition = { workspace = true } -name = "hotshot-examples" -readme = "README.md" -version = { workspace = true } -rust-version = { workspace = true } - -[features] -default = ["docs", "doc-images", "hotshot-testing"] -gpu-vid = ["hotshot-example-types/gpu-vid"] - -# Build the extended documentation -docs = [] -doc-images = [] -hotshot-testing = ["hotshot/hotshot-testing"] -fixed-leader-election = [] - -# Common -[[example]] -name = "orchestrator" -path = "orchestrator.rs" - -# Libp2p -[[example]] -name = "validator-libp2p" -path = "libp2p/validator.rs" - -[[example]] -name = "multi-validator-libp2p" -path = "libp2p/multi-validator.rs" - -[[example]] -name = "all-libp2p" -path = "libp2p/all.rs" - -# Combined -[[example]] -name = "all-combined" -path = "combined/all.rs" - -[[example]] -name = "multi-validator-combined" -path = "combined/multi-validator.rs" - -[[example]] -name = "validator-combined" -path = "combined/validator.rs" - -[[example]] -name = "orchestrator-combined" -path = "combined/orchestrator.rs" - -# Push CDN -[[example]] -name = "all-push-cdn" -path = "push-cdn/all.rs" - -[[example]] -name = "validator-push-cdn" -path = "push-cdn/validator.rs" - -[[example]] -name = "multi-validator-push-cdn" -path = "push-cdn/multi-validator.rs" - -[[example]] -name = "cdn-broker" -path = "push-cdn/broker.rs" - -[[example]] -name = "cdn-marshal" -path = "push-cdn/marshal.rs" - -[[example]] -name = "whitelist-push-cdn" -path = "push-cdn/whitelist-adapter.rs" +name = "examples" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true [dependencies] -async-trait = { workspace = true } - -cdn-broker = { workspace = true, features = ["global-permits"] } -cdn-marshal = { workspace = true } -chrono = { workspace = true } -clap = { workspace = true, optional = true } -futures = { workspace = true } -hotshot = { path = "../hotshot" } -hotshot-example-types = { path = "../example-types" } -hotshot-orchestrator = { version = "0.5.36", path = "../orchestrator", default-features = false } -hotshot-testing = { path = "../testing" } -hotshot-types = { path = "../types" } -libp2p-networking = { workspace = true } -local-ip-address = "0.6" -portpicker = { workspace = true } -rand = { workspace = true } -serde = { workspace = true, features = ["rc"] } -sha2 = { workspace = true } -surf-disco = { workspace = true } -time = { workspace = true } -tokio = { workspace = true } - -tracing = { workspace = true } -url = { workspace = true } - -[dev-dependencies] -anyhow = { workspace = true } -clap = { workspace = true } -toml = { workspace = true } -tracing-subscriber = "0.3" [lints] workspace = true diff --git a/crates/examples/combined/all.rs b/crates/examples/combined/all.rs deleted file mode 100644 index 89864af1d1..0000000000 --- a/crates/examples/combined/all.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! An example program using both the web server and libp2p -/// types used for this example -pub mod types; - -use std::path::Path; - -use cdn_broker::{reexports::def::hook::NoMessageHook, Broker}; -use cdn_marshal::Marshal; -use hotshot::{ - helpers::initialize_logging, - traits::implementations::{KeyPair, TestingDef, WrappedSignatureKey}, - types::SignatureKey, -}; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use hotshot_types::traits::node_implementation::NodeType; -use infra::{gen_local_address, BUILDER_BASE_PORT, VALIDATOR_BASE_PORT}; -use rand::{rngs::StdRng, RngCore, SeedableRng}; -use tokio::spawn; -use tracing::{error, instrument}; - -use crate::{ - infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}, - types::{Network, NodeImpl, ThisRun}, -}; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let (config, orchestrator_url) = read_orchestrator_init_config::(); - - // The configuration we are using for testing is 2 brokers & 1 marshal - // A keypair shared between brokers - let (broker_public_key, broker_private_key) = - ::SignatureKey::generated_from_seed_indexed([0u8; 32], 1337); - - // Get the OS temporary directory - let temp_dir = std::env::temp_dir(); - - // Create an SQLite file inside of the temporary directory - let discovery_endpoint = temp_dir - .join(Path::new(&format!( - "test-{}.sqlite", - StdRng::from_entropy().next_u64() - ))) - .to_string_lossy() - .into_owned(); - - // 2 brokers - for _ in 0..2 { - // Get the ports to bind to - let private_port = portpicker::pick_unused_port().expect("could not find an open port"); - let public_port = portpicker::pick_unused_port().expect("could not find an open port"); - - // Extrapolate addresses - let private_address = format!("127.0.0.1:{private_port}"); - let public_address = format!("127.0.0.1:{public_port}"); - - let config: cdn_broker::Config::SignatureKey>> = - cdn_broker::Config { - discovery_endpoint: discovery_endpoint.clone(), - public_advertise_endpoint: public_address.clone(), - public_bind_endpoint: public_address, - private_advertise_endpoint: private_address.clone(), - private_bind_endpoint: private_address, - - keypair: KeyPair { - public_key: WrappedSignatureKey(broker_public_key), - private_key: broker_private_key.clone(), - }, - - user_message_hook: NoMessageHook, - broker_message_hook: NoMessageHook, - - metrics_bind_endpoint: None, - ca_cert_path: None, - ca_key_path: None, - global_memory_pool_size: Some(1024 * 1024 * 1024), - }; - - // Create and spawn the broker - spawn(async move { - let broker: Broker::SignatureKey>> = - Broker::new(config).await.expect("broker failed to start"); - - // Error if we stopped unexpectedly - if let Err(err) = broker.start().await { - error!("broker stopped: {err}"); - } - }); - } - - // Get the port to use for the marshal - let marshal_endpoint = config - .cdn_marshal_address - .clone() - .expect("CDN marshal address must be specified"); - - // Configure the marshal - let marshal_config = cdn_marshal::Config { - bind_endpoint: marshal_endpoint.clone(), - discovery_endpoint, - metrics_bind_endpoint: None, - ca_cert_path: None, - ca_key_path: None, - global_memory_pool_size: Some(1024 * 1024 * 1024), - }; - - // Spawn the marshal - spawn(async move { - let marshal: Marshal::SignatureKey>> = - Marshal::new(marshal_config) - .await - .expect("failed to spawn marshal"); - - // Error if we stopped unexpectedly - if let Err(err) = marshal.start().await { - error!("broker stopped: {err}"); - } - }); - - // orchestrator - spawn(run_orchestrator::(OrchestratorArgs { - url: orchestrator_url.clone(), - config: config.clone(), - })); - - // nodes - let mut nodes = Vec::new(); - for i in 0..config.config.num_nodes_with_stake.into() { - // Calculate our libp2p advertise address, which we will later derive the - // bind address from for example purposes. - let advertise_address = gen_local_address::(i); - let orchestrator_url = orchestrator_url.clone(); - let builder_address = gen_local_address::(i); - - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs { - url: orchestrator_url, - advertise_address: Some(advertise_address.to_string()), - builder_address: Some(builder_address), - network_config_file: None, - }, - ) - .await; - }); - nodes.push(node); - } - futures::future::join_all(nodes).await; -} diff --git a/crates/examples/combined/multi-validator.rs b/crates/examples/combined/multi-validator.rs deleted file mode 100644 index b721cb5c4f..0000000000 --- a/crates/examples/combined/multi-validator.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A multi-validator using both the web server libp2p -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::{MultiValidatorArgs, ValidatorArgs}; -use tokio::spawn; -use tracing::instrument; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let args = MultiValidatorArgs::parse(); - tracing::debug!("connecting to orchestrator at {:?}", args.url); - let mut nodes = Vec::new(); - for node_index in 0..args.num_nodes { - let args = args.clone(); - - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs::from_multi_args(args, node_index), - ) - .await; - }); - nodes.push(node); - } - let _result = futures::future::join_all(nodes).await; -} diff --git a/crates/examples/combined/orchestrator.rs b/crates/examples/combined/orchestrator.rs deleted file mode 100644 index c3d399f489..0000000000 --- a/crates/examples/combined/orchestrator.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! Orchestrator using the web server -/// types used for this example -pub mod types; - -use hotshot::helpers::initialize_logging; -use hotshot_example_types::state_types::TestTypes; -use tracing::instrument; - -use crate::infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}; -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let (config, orchestrator_url) = read_orchestrator_init_config::(); - run_orchestrator::(OrchestratorArgs:: { - url: orchestrator_url.clone(), - config: config.clone(), - }) - .await; -} diff --git a/crates/examples/combined/types.rs b/crates/examples/combined/types.rs deleted file mode 100644 index 1209891b71..0000000000 --- a/crates/examples/combined/types.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -use std::fmt::Debug; - -use hotshot::traits::implementations::CombinedNetworks; -use hotshot_example_types::{ - auction_results_provider_types::TestAuctionResultsProvider, state_types::TestTypes, - storage_types::TestStorage, -}; -use hotshot_types::traits::node_implementation::NodeImplementation; -use serde::{Deserialize, Serialize}; - -use crate::infra::CombinedDaRun; - -/// dummy struct so we can choose types -#[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] -pub struct NodeImpl {} - -/// Convenience type alias -pub type Network = CombinedNetworks; - -impl NodeImplementation for NodeImpl { - type Network = Network; - type Storage = TestStorage; - type AuctionResultsProvider = TestAuctionResultsProvider; -} -/// convenience type alias -pub type ThisRun = CombinedDaRun; diff --git a/crates/examples/combined/validator.rs b/crates/examples/combined/validator.rs deleted file mode 100644 index fd6ff83957..0000000000 --- a/crates/examples/combined/validator.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A validator using both the web server and libp2p - -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use local_ip_address::local_ip; -use tracing::{debug, instrument}; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let mut args = ValidatorArgs::parse(); - - // If we did not set the advertise address, use our local IP and port 8000 - let local_ip = local_ip().expect("failed to get local IP"); - args.advertise_address = Some(args.advertise_address.unwrap_or(format!("{local_ip}:8000"))); - - debug!("connecting to orchestrator at {:?}", args.url); - infra::main_entry_point::(args).await; -} diff --git a/crates/examples/infra/mod.rs b/crates/examples/infra/mod.rs deleted file mode 100755 index 2462215dd9..0000000000 --- a/crates/examples/infra/mod.rs +++ /dev/null @@ -1,1129 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -#![allow(clippy::panic)] -use std::{ - collections::HashMap, - fmt::Debug, - fs, - net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}, - num::NonZeroUsize, - sync::Arc, - time::Instant, -}; - -use async_trait::async_trait; -use cdn_broker::reexports::crypto::signature::KeyPair; -use chrono::Utc; -use clap::{value_parser, Arg, Command, Parser}; -use futures::StreamExt; -use hotshot::{ - traits::{ - implementations::{ - derive_libp2p_multiaddr, derive_libp2p_peer_id, CdnMetricsValue, CdnTopic, - CombinedNetworks, Libp2pMetricsValue, Libp2pNetwork, PushCdnNetwork, - WrappedSignatureKey, - }, - BlockPayload, NodeImplementation, - }, - types::SystemContextHandle, - MarketplaceConfig, SystemContext, -}; -use hotshot_example_types::{ - auction_results_provider_types::TestAuctionResultsProvider, - block_types::{TestBlockHeader, TestBlockPayload, TestTransaction}, - node_types::{Libp2pImpl, PushCdnImpl}, - state_types::TestInstanceState, - storage_types::TestStorage, -}; -use hotshot_orchestrator::{ - self, - client::{get_complete_config, BenchResults, OrchestratorClient, ValidatorArgs}, -}; -use hotshot_testing::block_builder::{ - BuilderTask, RandomBuilderImplementation, SimpleBuilderImplementation, - TestBuilderImplementation, -}; -use hotshot_types::{ - consensus::ConsensusMetricsValue, - data::{Leaf, TestableLeaf}, - event::{Event, EventType}, - network::{BuilderType, NetworkConfig, NetworkConfigFile, NetworkConfigSource}, - traits::{ - block_contents::{BlockHeader, TestableBlock}, - election::Membership, - network::ConnectedNetwork, - node_implementation::{ConsensusTime, NodeType, Versions}, - states::TestableState, - }, - HotShotConfig, PeerConfig, ValidatorConfig, -}; -use libp2p_networking::network::{GossipConfig, RequestResponseConfig}; -use rand::{rngs::StdRng, SeedableRng}; -use surf_disco::Url; -use tracing::{debug, error, info, warn}; - -#[derive(Debug, Clone)] -/// Arguments passed to the orchestrator -pub struct OrchestratorArgs { - /// The url the orchestrator runs on; this should be in the form of `http://localhost:5555` or `http://0.0.0.0:5555` - pub url: Url, - /// The configuration file to be used for this run - pub config: NetworkConfig, -} - -#[derive(Parser, Debug, Clone)] -#[command( - name = "Multi-machine consensus", - about = "Simulates consensus among multiple machines" -)] -/// The configuration file to be used for this run -pub struct ConfigArgs { - /// The configuration file to be used for this run - pub config_file: String, -} - -impl Default for ConfigArgs { - fn default() -> Self { - Self { - config_file: "./crates/orchestrator/run-config.toml".to_string(), - } - } -} - -/// Reads the orchestrator initialization config from the command line -/// # Panics -/// If unable to read the config file from the command line -#[allow(clippy::too_many_lines)] -pub fn read_orchestrator_init_config() -> (NetworkConfig, Url) -{ - // assign default setting - let mut orchestrator_url = Url::parse("http://localhost:4444").unwrap(); - let mut args = ConfigArgs::default(); - // start reading from the command line - let matches = Command::new("orchestrator") - .arg( - Arg::new("config_file") - .short('c') - .long("config_file") - .value_name("FILE") - .help("Sets a custom config file with default values, some might be changed if they are set manually in the command line") - .required(true), - ) - .arg( - Arg::new("total_nodes") - .short('n') - .long("total_nodes") - .value_name("NUM") - .help("Sets the total number of nodes") - .required(false), - ) - .arg( - Arg::new("da_committee_size") - .short('d') - .long("da_committee_size") - .value_name("NUM") - .help("Sets the size of the data availability committee") - .required(false), - ) - .arg( - Arg::new("transactions_per_round") - .short('t') - .long("transactions_per_round") - .value_name("NUM") - .help("Sets the number of transactions per round") - .required(false), - ) - .arg( - Arg::new("transaction_size") - .short('s') - .long("transaction_size") - .value_name("NUM") - .help("Sets the size of each transaction in bytes") - .required(false), - ) - .arg( - Arg::new("rounds") - .short('r') - .long("rounds") - .value_name("NUM") - .help("Sets the number of rounds to run") - .required(false), - ) - .arg( - Arg::new("commit_sha") - .short('o') - .long("commit_sha") - .value_name("SHA") - .help("Sets the commit sha to output in the results") - .required(false), - ) - .arg( - Arg::new("orchestrator_url") - .short('u') - .long("orchestrator_url") - .value_name("URL") - .help("Sets the url of the orchestrator") - .required(false), - ) - .arg( - Arg::new("fixed_leader_for_gpuvid") - .short('f') - .long("fixed_leader_for_gpuvid") - .value_name("BOOL") - .help("Sets the number of fixed leader for gpu vid, only be used when leaders running on gpu") - .required(false), - ) - .arg( - Arg::new("builder") - .short('b') - .long("builder") - .value_name("BUILDER_TYPE") - .value_parser(value_parser!(BuilderType)) - .help("Sets type of builder. `simple` or `random` to run corresponding integrated builder, `external` to use the one specified by `[config.builder_url]` in config") - .required(false), - ) - .arg( - Arg::new("cdn_marshal_address") - .short('m') - .long("cdn_marshal_address") - .value_name("URL") - .help("Sets the url for cdn_broker_marshal_endpoint") - .required(false), - ) - .get_matches(); - - if let Some(config_file_string) = matches.get_one::("config_file") { - args = ConfigArgs { - config_file: config_file_string.clone(), - }; - } else { - error!("No config file provided, we'll use the default one."); - } - let mut config: NetworkConfig = - load_config_from_file::(&args.config_file); - - if let Some(total_nodes_string) = matches.get_one::("total_nodes") { - config.config.num_nodes_with_stake = total_nodes_string.parse::().unwrap(); - config.config.known_nodes_with_stake = - vec![PeerConfig::default(); config.config.num_nodes_with_stake.get() as usize]; - error!( - "config.config.total_nodes: {:?}", - config.config.num_nodes_with_stake - ); - } - if let Some(da_committee_size_string) = matches.get_one::("da_committee_size") { - config.config.da_staked_committee_size = da_committee_size_string.parse::().unwrap(); - } - if let Some(fixed_leader_for_gpuvid_string) = - matches.get_one::("fixed_leader_for_gpuvid") - { - config.config.fixed_leader_for_gpuvid = - fixed_leader_for_gpuvid_string.parse::().unwrap(); - } - if let Some(transactions_per_round_string) = matches.get_one::("transactions_per_round") - { - config.transactions_per_round = transactions_per_round_string.parse::().unwrap(); - } - if let Some(transaction_size_string) = matches.get_one::("transaction_size") { - config.transaction_size = transaction_size_string.parse::().unwrap(); - } - if let Some(rounds_string) = matches.get_one::("rounds") { - config.rounds = rounds_string.parse::().unwrap(); - } - if let Some(commit_sha_string) = matches.get_one::("commit_sha") { - config.commit_sha = commit_sha_string.to_string(); - } - if let Some(orchestrator_url_string) = matches.get_one::("orchestrator_url") { - orchestrator_url = Url::parse(orchestrator_url_string).unwrap(); - } - if let Some(builder_type) = matches.get_one::("builder") { - config.builder = *builder_type; - } - if let Some(cdn_marshal_address_string) = matches.get_one::("cdn_marshal_address") { - config.cdn_marshal_address = Some(cdn_marshal_address_string.to_string()); - } - - (config, orchestrator_url) -} - -/// Reads a network configuration from a given filepath -/// # Panics -/// if unable to convert the config file into toml -/// # Note -/// This derived config is used for initialization of orchestrator, -/// therefore `known_nodes_with_stake` will be an initialized -/// vector full of the node's own config. -#[must_use] -pub fn load_config_from_file( - config_file: &str, -) -> NetworkConfig { - let config_file_as_string: String = fs::read_to_string(config_file) - .unwrap_or_else(|_| panic!("Could not read config file located at {config_file}")); - let config_toml: NetworkConfigFile = - toml::from_str::>(&config_file_as_string) - .expect("Unable to convert config file to TOML"); - - let mut config: NetworkConfig = config_toml.into(); - - // initialize it with size for better assignment of peers' config - config.config.known_nodes_with_stake = - vec![PeerConfig::default(); config.config.num_nodes_with_stake.get() as usize]; - - config -} - -/// Runs the orchestrator -pub async fn run_orchestrator( - OrchestratorArgs { url, config }: OrchestratorArgs, -) { - println!("Starting orchestrator",); - let _ = hotshot_orchestrator::run_orchestrator::(config, url).await; -} - -/// Helper function to calculate the number of transactions to send per node per round -#[allow(clippy::cast_possible_truncation)] -fn calculate_num_tx_per_round( - node_index: u64, - total_num_nodes: usize, - transactions_per_round: usize, -) -> usize { - transactions_per_round / total_num_nodes - + usize::from( - (total_num_nodes) - < (transactions_per_round % total_num_nodes) + 1 + (node_index as usize), - ) -} - -/// Helper function to generate transactions a given node should send -fn generate_transactions>( - node_index: u64, - rounds: usize, - transactions_to_send_per_round: usize, - transaction_size: usize, -) -> Vec -where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, -{ - let mut txn_rng = StdRng::seed_from_u64(node_index); - let mut transactions = Vec::new(); - - for _ in 0..rounds { - for _ in 0..transactions_to_send_per_round { - let txn = ::create_random_transaction( - None, - &mut txn_rng, - transaction_size as u64, - ); - - transactions.push(txn); - } - } - transactions -} - -/// Defines the behavior of a "run" of the network with a given configuration -#[async_trait] -pub trait RunDa< - TYPES: NodeType, - NETWORK: ConnectedNetwork, - NODE: NodeImplementation< - TYPES, - Network = NETWORK, - Storage = TestStorage, - AuctionResultsProvider = TestAuctionResultsProvider, - >, - V: Versions, -> where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - TYPES: NodeType, - Leaf: TestableLeaf, - Self: Sync, -{ - /// Initializes networking, returns self - async fn initialize_networking( - config: NetworkConfig, - validator_config: ValidatorConfig, - libp2p_advertise_address: Option, - ) -> Self; - - /// Initializes the genesis state and HotShot instance; does not start HotShot consensus - /// # Panics if it cannot generate a genesis block, fails to initialize HotShot, or cannot - /// get the anchored view - /// Note: sequencing leaf does not have state, so does not return state - async fn initialize_state_and_hotshot(&self) -> SystemContextHandle { - let initializer = - hotshot::HotShotInitializer::::from_genesis::(TestInstanceState::default()) - .await - .expect("Couldn't generate genesis block"); - - let config = self.config(); - let validator_config = self.validator_config(); - - // Get KeyPair for certificate Aggregation - let pk = validator_config.public_key.clone(); - let sk = validator_config.private_key.clone(); - - let network = self.network(); - - let all_nodes = if cfg!(feature = "fixed-leader-election") { - let mut vec = config.config.known_nodes_with_stake.clone(); - vec.truncate(config.config.fixed_leader_for_gpuvid); - vec - } else { - config.config.known_nodes_with_stake.clone() - }; - - let da_nodes = config.config.known_da_nodes.clone(); - - // Create the quorum membership from all nodes, specifying the committee - // as the known da nodes - let memberships = ::Membership::new(all_nodes, da_nodes); - - let marketplace_config = MarketplaceConfig { - auction_results_provider: TestAuctionResultsProvider::::default().into(), - // TODO: we need to pass a valid fallback builder url here somehow - fallback_builder_url: config.config.builder_urls.first().clone(), - }; - - SystemContext::init( - pk, - sk, - config.node_index, - config.config, - memberships, - Arc::from(network), - initializer, - ConsensusMetricsValue::default(), - TestStorage::::default(), - marketplace_config, - ) - .await - .expect("Could not init hotshot") - .0 - } - - /// Starts HotShot consensus, returns when consensus has finished - #[allow(clippy::too_many_lines)] - async fn run_hotshot( - &self, - context: SystemContextHandle, - transactions: &mut Vec, - transactions_to_send_per_round: u64, - transaction_size_in_bytes: u64, - ) -> BenchResults { - let NetworkConfig { - rounds, node_index, .. - } = self.config(); - - let mut total_transactions_committed = 0; - let mut total_transactions_sent = 0; - let mut minimum_latency = 1000; - let mut maximum_latency = 0; - let mut total_latency = 0; - let mut num_latency = 0; - - info!("Starting HotShot example!"); - let start = Instant::now(); - - let mut event_stream = context.event_stream(); - let mut anchor_view: TYPES::View = ::genesis(); - let mut num_successful_commits = 0; - - context.hotshot.start_consensus().await; - - loop { - match event_stream.next().await { - None => { - panic!("Error! Event stream completed before consensus ended."); - } - Some(Event { event, .. }) => { - match event { - EventType::Error { error } => { - error!("Error in consensus: {:?}", error); - // TODO what to do here - } - EventType::Decide { - leaf_chain, - qc: _, - block_size, - } => { - let current_timestamp = Utc::now().timestamp(); - // this might be a obob - if let Some(leaf_info) = leaf_chain.first() { - let leaf = &leaf_info.leaf; - info!("Decide event for leaf: {}", *leaf.view_number()); - - // iterate all the decided transactions to calculate latency - if let Some(block_payload) = &leaf.block_payload() { - for tx in - block_payload.transactions(leaf.block_header().metadata()) - { - let restored_timestamp_vec = - tx.bytes()[tx.bytes().len() - 8..].to_vec(); - let restored_timestamp = i64::from_be_bytes( - restored_timestamp_vec.as_slice().try_into().unwrap(), - ); - let cur_latency = current_timestamp - restored_timestamp; - total_latency += cur_latency; - num_latency += 1; - minimum_latency = - std::cmp::min(minimum_latency, cur_latency); - maximum_latency = - std::cmp::max(maximum_latency, cur_latency); - } - } - - let new_anchor = leaf.view_number(); - if new_anchor >= anchor_view { - anchor_view = leaf.view_number(); - } - - // send transactions - for _ in 0..transactions_to_send_per_round { - // append current timestamp to the tx to calc latency - let timestamp = Utc::now().timestamp(); - let mut tx = transactions.remove(0).into_bytes(); - let mut timestamp_vec = timestamp.to_be_bytes().to_vec(); - tx.append(&mut timestamp_vec); - - () = context - .submit_transaction(TestTransaction::new(tx)) - .await - .unwrap(); - total_transactions_sent += 1; - } - } - - if let Some(size) = block_size { - total_transactions_committed += size; - debug!("[{node_index}] got block with size: {:?}", size); - } - - num_successful_commits += leaf_chain.len(); - if num_successful_commits >= rounds { - break; - } - - if leaf_chain.len() > 1 { - warn!("Leaf chain is greater than 1 with len {}", leaf_chain.len()); - } - // when we make progress, submit new events - } - EventType::ReplicaViewTimeout { view_number } => { - warn!("Timed out as a replicas in view {:?}", view_number); - } - EventType::ViewTimeout { view_number } => { - warn!("Timed out in view {:?}", view_number); - } - _ => {} // mostly DA proposal - } - } - } - } - let consensus_lock = context.hotshot.consensus(); - let consensus = consensus_lock.read().await; - let num_eligible_leaders = context - .hotshot - .memberships - .committee_leaders(TYPES::View::genesis(), TYPES::Epoch::genesis()) - .len(); - let total_num_views = usize::try_from(consensus.locked_view().u64()).unwrap(); - // `failed_num_views` could include uncommitted views - let failed_num_views = total_num_views - num_successful_commits; - // When posting to the orchestrator, note that the total number of views also include un-finalized views. - println!("[{node_index}]: Total views: {total_num_views}, Failed views: {failed_num_views}, num_successful_commits: {num_successful_commits}"); - // Output run results - let total_time_elapsed = start.elapsed(); // in seconds - println!("[{node_index}]: {rounds} rounds completed in {total_time_elapsed:?} - Total transactions sent: {total_transactions_sent} - Total transactions committed: {total_transactions_committed} - Total commitments: {num_successful_commits}"); - if total_transactions_committed != 0 { - // prevent division by 0 - let total_time_elapsed_sec = std::cmp::max(total_time_elapsed.as_secs(), 1u64); - // extra 8 bytes for timestamp - let throughput_bytes_per_sec = total_transactions_committed - * (transaction_size_in_bytes + 8) - / total_time_elapsed_sec; - let avg_latency_in_sec = total_latency / num_latency; - println!("[{node_index}]: throughput: {throughput_bytes_per_sec} bytes/sec, avg_latency: {avg_latency_in_sec} sec."); - - BenchResults { - partial_results: "Unset".to_string(), - avg_latency_in_sec, - num_latency, - minimum_latency_in_sec: minimum_latency, - maximum_latency_in_sec: maximum_latency, - throughput_bytes_per_sec, - total_transactions_committed, - transaction_size_in_bytes: transaction_size_in_bytes + 8, // extra 8 bytes for timestamp - total_time_elapsed_in_sec: total_time_elapsed.as_secs(), - total_num_views, - failed_num_views, - committee_type: format!( - "{} with {num_eligible_leaders} eligible leaders", - std::any::type_name::() - ), - } - } else { - // all values with zero - BenchResults::default() - } - } - - /// Returns the underlying network for this run - fn network(&self) -> NETWORK; - - /// Returns the config for this run - fn config(&self) -> NetworkConfig; - - /// Returns the validator config with private signature keys for this run. - fn validator_config(&self) -> ValidatorConfig; -} - -// Push CDN - -/// Represents a Push CDN-based run -pub struct PushCdnDaRun { - /// The underlying configuration - config: NetworkConfig, - /// The private validator config - validator_config: ValidatorConfig, - /// The underlying network - network: PushCdnNetwork, -} - -#[async_trait] -impl< - TYPES: NodeType< - Transaction = TestTransaction, - BlockPayload = TestBlockPayload, - BlockHeader = TestBlockHeader, - InstanceState = TestInstanceState, - >, - NODE: NodeImplementation< - TYPES, - Network = PushCdnNetwork, - Storage = TestStorage, - AuctionResultsProvider = TestAuctionResultsProvider, - >, - V: Versions, - > RunDa, NODE, V> for PushCdnDaRun -where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - Leaf: TestableLeaf, - Self: Sync, -{ - async fn initialize_networking( - config: NetworkConfig, - validator_config: ValidatorConfig, - _libp2p_advertise_address: Option, - ) -> PushCdnDaRun { - // Convert to the Push-CDN-compatible type - let keypair = KeyPair { - public_key: WrappedSignatureKey(validator_config.public_key.clone()), - private_key: validator_config.private_key.clone(), - }; - - // See if we should be DA, subscribe to the DA topic if so - let mut topics = vec![CdnTopic::Global]; - if validator_config.is_da { - topics.push(CdnTopic::Da); - } - - // Create the network and await the initial connection - let network = PushCdnNetwork::new( - config - .cdn_marshal_address - .clone() - .expect("`cdn_marshal_address` needs to be supplied for a push CDN run"), - topics, - keypair, - CdnMetricsValue::default(), - ) - .expect("failed to create network"); - - // Wait for the network to be ready - network.wait_for_ready().await; - - PushCdnDaRun { - config, - validator_config, - network, - } - } - - fn network(&self) -> PushCdnNetwork { - self.network.clone() - } - - fn config(&self) -> NetworkConfig { - self.config.clone() - } - - fn validator_config(&self) -> ValidatorConfig { - self.validator_config.clone() - } -} - -// Libp2p - -/// Represents a libp2p-based run -pub struct Libp2pDaRun { - /// The underlying network configuration - config: NetworkConfig, - /// The private validator config - validator_config: ValidatorConfig, - /// The underlying network - network: Libp2pNetwork, -} - -#[async_trait] -impl< - TYPES: NodeType< - Transaction = TestTransaction, - BlockPayload = TestBlockPayload, - BlockHeader = TestBlockHeader, - InstanceState = TestInstanceState, - >, - NODE: NodeImplementation< - TYPES, - Network = Libp2pNetwork, - Storage = TestStorage, - AuctionResultsProvider = TestAuctionResultsProvider, - >, - V: Versions, - > RunDa, NODE, V> for Libp2pDaRun -where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - Leaf: TestableLeaf, - Self: Sync, -{ - async fn initialize_networking( - config: NetworkConfig, - validator_config: ValidatorConfig, - libp2p_advertise_address: Option, - ) -> Libp2pDaRun { - // Extrapolate keys for ease of use - let public_key = &validator_config.public_key; - let private_key = &validator_config.private_key; - - // In an example, we can calculate the libp2p bind address as a function - // of the advertise address. - let bind_address = if let Some(libp2p_advertise_address) = libp2p_advertise_address { - let libp2p_advertise_address: SocketAddrV4 = libp2p_advertise_address - .parse() - .expect("failed to parse advertise address"); - - // If we have supplied one, use it - SocketAddr::new( - IpAddr::V4(Ipv4Addr::UNSPECIFIED), - libp2p_advertise_address.port(), - ) - .to_string() - } else { - // If not, index a base port with our node index - SocketAddr::new( - IpAddr::V4(Ipv4Addr::UNSPECIFIED), - 8000 + (u16::try_from(config.node_index) - .expect("failed to create advertise address")), - ) - .to_string() - }; - - // Create the qurorum membership from the list of known nodes - let all_nodes = config.config.known_nodes_with_stake.clone(); - let da_nodes = config.config.known_da_nodes.clone(); - let quorum_membership = TYPES::Membership::new(all_nodes, da_nodes); - - // Derive the bind address - let bind_address = - derive_libp2p_multiaddr(&bind_address).expect("failed to derive bind address"); - - // Create the Libp2p network - let libp2p_network = Libp2pNetwork::from_config( - config.clone(), - quorum_membership, - GossipConfig::default(), - RequestResponseConfig::default(), - bind_address, - public_key, - private_key, - Libp2pMetricsValue::default(), - ) - .await - .expect("failed to create libp2p network"); - - // Wait for the network to be ready - libp2p_network.wait_for_ready().await; - - Libp2pDaRun { - config, - validator_config, - network: libp2p_network, - } - } - - fn network(&self) -> Libp2pNetwork { - self.network.clone() - } - - fn config(&self) -> NetworkConfig { - self.config.clone() - } - - fn validator_config(&self) -> ValidatorConfig { - self.validator_config.clone() - } -} - -// Combined network - -/// Represents a combined-network-based run -pub struct CombinedDaRun { - /// The underlying network configuration - config: NetworkConfig, - /// The private validator config - validator_config: ValidatorConfig, - /// The underlying network - network: CombinedNetworks, -} - -#[async_trait] -impl< - TYPES: NodeType< - Transaction = TestTransaction, - BlockPayload = TestBlockPayload, - BlockHeader = TestBlockHeader, - InstanceState = TestInstanceState, - >, - NODE: NodeImplementation< - TYPES, - Network = CombinedNetworks, - Storage = TestStorage, - AuctionResultsProvider = TestAuctionResultsProvider, - >, - V: Versions, - > RunDa, NODE, V> for CombinedDaRun -where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - Leaf: TestableLeaf, - Self: Sync, -{ - async fn initialize_networking( - config: NetworkConfig, - validator_config: ValidatorConfig, - libp2p_advertise_address: Option, - ) -> CombinedDaRun { - // Initialize our Libp2p network - let libp2p_network: Libp2pDaRun = as RunDa< - TYPES, - Libp2pNetwork, - Libp2pImpl, - V, - >>::initialize_networking( - config.clone(), - validator_config.clone(), - libp2p_advertise_address.clone(), - ) - .await; - - // Initialize our CDN network - let cdn_network: PushCdnDaRun = as RunDa< - TYPES, - PushCdnNetwork, - PushCdnImpl, - V, - >>::initialize_networking( - config.clone(), - validator_config.clone(), - libp2p_advertise_address, - ) - .await; - - // Create our combined network config - let delay_duration = config - .combined_network_config - .as_ref() - .map(|config| config.delay_duration); - - // Create our combined network - let network = - CombinedNetworks::new(cdn_network.network, libp2p_network.network, delay_duration); - - // Return the run configuration - CombinedDaRun { - config, - validator_config, - network, - } - } - - fn network(&self) -> CombinedNetworks { - self.network.clone() - } - - fn config(&self) -> NetworkConfig { - self.config.clone() - } - - fn validator_config(&self) -> ValidatorConfig { - self.validator_config.clone() - } -} - -/// Main entry point for validators -/// # Panics -/// if unable to get the local ip address -pub async fn main_entry_point< - TYPES: NodeType< - Transaction = TestTransaction, - BlockHeader = TestBlockHeader, - InstanceState = TestInstanceState, - >, - NETWORK: ConnectedNetwork, - NODE: NodeImplementation< - TYPES, - Network = NETWORK, - Storage = TestStorage, - AuctionResultsProvider = TestAuctionResultsProvider, - >, - V: Versions, - RUNDA: RunDa, ->( - args: ValidatorArgs, -) where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - Leaf: TestableLeaf, -{ - // Initialize logging - hotshot::helpers::initialize_logging(); - - info!("Starting validator"); - - let orchestrator_client: OrchestratorClient = OrchestratorClient::new(args.url.clone()); - - // We assume one node will not call this twice to generate two validator_config-s with same identity. - let validator_config = NetworkConfig::::generate_init_validator_config( - orchestrator_client - .get_node_index_for_init_validator_config() - .await, - // we assign nodes to the DA committee by default - true, - ); - - // Derives our Libp2p private key from our private key, and then returns the public key of that key - let libp2p_public_key = - derive_libp2p_peer_id::(&validator_config.private_key) - .expect("failed to derive Libp2p keypair"); - - // We need this to be able to register our node - let peer_config = - PeerConfig::::to_bytes(&validator_config.public_config()).clone(); - - // Derive the advertise multiaddress from the supplied string - let advertise_multiaddress = args.advertise_address.clone().map(|advertise_address| { - derive_libp2p_multiaddr(&advertise_address).expect("failed to derive Libp2p multiaddr") - }); - - // conditionally save/load config from file or orchestrator - // This is a function that will return correct complete config from orchestrator. - // It takes in a valid args.network_config_file when loading from file, or valid validator_config when loading from orchestrator, the invalid one will be ignored. - // It returns the complete config which also includes peer's public key and public config. - // This function will be taken solely by sequencer right after OrchestratorClient::new, - // which means the previous `generate_validator_config_when_init` will not be taken by sequencer, it's only for key pair generation for testing in hotshot. - - let (mut run_config, validator_config, source) = get_complete_config( - &orchestrator_client, - validator_config, - advertise_multiaddress, - Some(libp2p_public_key), - ) - .await - .expect("failed to get config"); - - let builder_task = initialize_builder( - &mut run_config, - &validator_config, - &args, - &orchestrator_client, - ) - .await; - - run_config.config.builder_urls = orchestrator_client - .get_builder_addresses() - .await - .try_into() - .expect("Orchestrator didn't provide any builder addresses"); - - debug!( - "Assigned urls from orchestrator: {}", - run_config - .config - .builder_urls - .iter() - .map(ToString::to_string) - .collect::>() - .join(",") - ); - - info!("Initializing networking"); - let run = - RUNDA::initialize_networking(run_config.clone(), validator_config, args.advertise_address) - .await; - let hotshot = run.initialize_state_and_hotshot().await; - - if let Some(task) = builder_task { - task.start(Box::new(hotshot.event_stream())); - } - - // pre-generate transactions - let NetworkConfig { - transaction_size, - rounds, - transactions_per_round, - node_index, - config: HotShotConfig { - num_nodes_with_stake, - .. - }, - .. - } = run_config; - - let transactions_to_send_per_round = calculate_num_tx_per_round( - node_index, - num_nodes_with_stake.get(), - transactions_per_round, - ); - let mut transactions: Vec = generate_transactions::( - node_index, - rounds, - transactions_to_send_per_round, - transaction_size, - ); - - if let NetworkConfigSource::Orchestrator = source { - info!("Waiting for the start command from orchestrator"); - orchestrator_client - .wait_for_all_nodes_ready(peer_config) - .await; - } - - info!("Starting HotShot"); - let bench_results = run - .run_hotshot( - hotshot, - &mut transactions, - transactions_to_send_per_round as u64, - (transaction_size + 8) as u64, // extra 8 bytes for transaction base, see `create_random_transaction`. - ) - .await; - orchestrator_client.post_bench_results(bench_results).await; -} - -/// Sets correct builder_url and registers a builder with orchestrator if this node is running one. -/// Returns a `BuilderTask` if this node is going to be running a builder. -async fn initialize_builder< - TYPES: NodeType< - Transaction = TestTransaction, - BlockHeader = TestBlockHeader, - InstanceState = TestInstanceState, - >, ->( - run_config: &mut NetworkConfig<::SignatureKey>, - validator_config: &ValidatorConfig<::SignatureKey>, - args: &ValidatorArgs, - orchestrator_client: &OrchestratorClient, -) -> Option>> -where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - Leaf: TestableLeaf, -{ - if !validator_config.is_da { - return None; - } - - let advertise_urls: Vec; - let bind_address: Url; - - match args.builder_address { - None => { - let port = portpicker::pick_unused_port().expect("Failed to pick an unused port"); - advertise_urls = local_ip_address::list_afinet_netifas() - .expect("Couldn't get list of local IP addresses") - .into_iter() - .map(|(_name, ip)| ip) - .filter(|ip| !ip.is_loopback()) - .map(|ip| match ip { - IpAddr::V4(addr) => Url::parse(&format!("http://{addr}:{port}")).unwrap(), - IpAddr::V6(addr) => Url::parse(&format!("http://[{addr}]:{port}")).unwrap(), - }) - .collect(); - bind_address = Url::parse(&format!("http://0.0.0.0:{port}")).unwrap(); - } - Some(ref addr) => { - bind_address = Url::parse(&format!("http://{addr}")).expect("Valid URL"); - advertise_urls = vec![bind_address.clone()]; - } - } - - match run_config.builder { - BuilderType::External => None, - BuilderType::Random => { - let builder_task = - >::start( - run_config.config.num_nodes_with_stake.into(), - bind_address, - run_config.random_builder.clone().unwrap_or_default(), - HashMap::new(), - ) - .await; - - orchestrator_client - .post_builder_addresses(advertise_urls) - .await; - - Some(builder_task) - } - BuilderType::Simple => { - let builder_task = - >::start( - run_config.config.num_nodes_with_stake.into(), - bind_address, - (), - HashMap::new(), - ) - .await; - - orchestrator_client - .post_builder_addresses(advertise_urls) - .await; - - Some(builder_task) - } - } -} - -/// Base port for validator -pub const VALIDATOR_BASE_PORT: u16 = 8000; -/// Base port for builder -pub const BUILDER_BASE_PORT: u16 = 9000; - -/// Generate a local address for node with index `node_index`, offsetting from port `BASE_PORT`. -/// # Panics -/// If `node_index` is too large to fit in a `u16` -#[must_use] -pub fn gen_local_address(node_index: usize) -> SocketAddr { - SocketAddr::new( - IpAddr::V4(Ipv4Addr::LOCALHOST), - BASE_PORT + (u16::try_from(node_index).expect("node index too large")), - ) -} diff --git a/crates/examples/libp2p/all.rs b/crates/examples/libp2p/all.rs deleted file mode 100644 index 4fd99cd0e8..0000000000 --- a/crates/examples/libp2p/all.rs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! An example program using libp2p -/// types used for this example -pub mod types; - -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use infra::{gen_local_address, BUILDER_BASE_PORT, VALIDATOR_BASE_PORT}; -use tokio::spawn; -use tracing::instrument; - -use crate::{ - infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}, - types::{Network, NodeImpl, ThisRun}, -}; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - // use configfile args - let (config, orchestrator_url) = read_orchestrator_init_config::(); - - // orchestrator - spawn(run_orchestrator::(OrchestratorArgs { - url: orchestrator_url.clone(), - config: config.clone(), - })); - - // nodes - let mut nodes = Vec::new(); - for i in 0..config.config.num_nodes_with_stake.into() { - // Calculate our libp2p advertise address, which we will later derive the - // bind address from for example purposes. - let advertise_address = gen_local_address::(i); - let builder_address = gen_local_address::(i); - let orchestrator_url = orchestrator_url.clone(); - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs { - url: orchestrator_url, - advertise_address: Some(advertise_address.to_string()), - builder_address: Some(builder_address), - network_config_file: None, - }, - ) - .await; - }); - nodes.push(node); - } - futures::future::join_all(nodes).await; -} diff --git a/crates/examples/libp2p/multi-validator.rs b/crates/examples/libp2p/multi-validator.rs deleted file mode 100644 index 0767245c3b..0000000000 --- a/crates/examples/libp2p/multi-validator.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A multi-validator using libp2p -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::{MultiValidatorArgs, ValidatorArgs}; -use tokio::spawn; -use tracing::instrument; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let args = MultiValidatorArgs::parse(); - tracing::debug!("connecting to orchestrator at {:?}", args.url); - let mut nodes = Vec::new(); - for node_index in 0..args.num_nodes { - let args = args.clone(); - - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs::from_multi_args(args, node_index), - ) - .await; - }); - nodes.push(node); - } - let _result = futures::future::join_all(nodes).await; -} diff --git a/crates/examples/libp2p/types.rs b/crates/examples/libp2p/types.rs deleted file mode 100644 index ed8fbcda6f..0000000000 --- a/crates/examples/libp2p/types.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -use std::fmt::Debug; - -use hotshot::traits::implementations::Libp2pNetwork; -use hotshot_example_types::{ - auction_results_provider_types::TestAuctionResultsProvider, state_types::TestTypes, - storage_types::TestStorage, -}; -use hotshot_types::traits::node_implementation::NodeImplementation; -use serde::{Deserialize, Serialize}; - -use crate::infra::Libp2pDaRun; - -/// dummy struct so we can choose types -#[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] -pub struct NodeImpl {} - -/// Convenience type alias -pub type Network = Libp2pNetwork; - -impl NodeImplementation for NodeImpl { - type Network = Network; - type Storage = TestStorage; - type AuctionResultsProvider = TestAuctionResultsProvider; -} -/// convenience type alias -pub type ThisRun = Libp2pDaRun; diff --git a/crates/examples/libp2p/validator.rs b/crates/examples/libp2p/validator.rs deleted file mode 100644 index c85e52688e..0000000000 --- a/crates/examples/libp2p/validator.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A validator using libp2p - -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use local_ip_address::local_ip; -use tracing::{debug, instrument}; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let mut args = ValidatorArgs::parse(); - - // If we did not set the advertise address, use our local IP and port 8000 - let local_ip = local_ip().expect("failed to get local IP"); - args.advertise_address = Some(args.advertise_address.unwrap_or(format!("{local_ip}:8000"))); - - debug!("connecting to orchestrator at {:?}", args.url); - infra::main_entry_point::(args).await; -} diff --git a/crates/examples/orchestrator.rs b/crates/examples/orchestrator.rs deleted file mode 100644 index 3bb419b980..0000000000 --- a/crates/examples/orchestrator.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A orchestrator - -use hotshot::helpers::initialize_logging; -use hotshot_example_types::state_types::TestTypes; -use tracing::instrument; - -use crate::infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}; - -/// general infra used for this example -#[path = "./infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let (config, orchestrator_url) = read_orchestrator_init_config::(); - run_orchestrator::(OrchestratorArgs:: { - url: orchestrator_url.clone(), - config: config.clone(), - }) - .await; -} diff --git a/crates/examples/push-cdn/README.md b/crates/examples/push-cdn/README.md deleted file mode 100644 index c460beb89a..0000000000 --- a/crates/examples/push-cdn/README.md +++ /dev/null @@ -1,68 +0,0 @@ -Steps ---------------- - -KeyDB is the ephemeral database, it's like Redis but with extra features. The only thing we run it with is with `--requirepass` to set a password. - -**Marshals:** -The marshal is the entry point of the push CDN, all users connect there first. It tells users which broker to connect to. - -- `-d` is the "discovery endpoint", which in this case is the URL of KeyDB. -- `-b` is the bind port. This is what you would set in run_config.toml for cdn_broker_marshal_endpoint -- `-m` is metrics stuff. You shouldn't have to use that - - -**Brokers:** -In a run with multiple machines, we want two brokers. With one machine, it's probably fine to do one broker. These are what route the messages. Here are the relevant command line arguments: - -- `-d` is the "discovery endpoint", which in this case is the URL of KeyDB. -- `--public-bind-endpoint`: the endpoint which we bind to locally for users to connect to (e.g. 0.0.0.0:1740) -- `--public-advertise-endpoint`: the endpoint which we advertise to users (e.g. my.public.ip:1740) -- `--private-bind-endpoint`: the endpoint which we bind to locally for brokers to connect to (e.g. 0.0.0.0:1741) -- `--private-advertise-endpoint`: the endpoint which we advertise to brokers (e.g. my.public.ip:1741) -- `-m` is metrics stuff. You shouldn't have to use that -For brokers, there is a magic value called `local_ip`. This resolves to the local IP address, which skips the need for talking to the AWS metadata server. For in-AWS uses, the following configuration is probably fine: -`cdn-broker --public-bind-endpoint 0.0.0.0:1740 --public-advertise-endpoint local_ip:1740 --private-bind-endpoint 0.0.0.0:1741 --private-advertise-endpoint local_ip:1741`. You won't need to put this port or values anywhere, as the marshal does everything for you. - -Examples: ---------------- - -**Run Locally** - -`just example all-push-cdn -- --config_file ./crates/orchestrator/run-config.toml` - -OR - -``` -docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb -just example cdn-marshal -- -d redis://localhost:6379 -b 9000 -just example cdn-broker -- -d redis://localhost:6379 --public-bind-endpoint 0.0.0.0:1740 --public-advertise-endpoint local_ip:1740 --private-bind-endpoint 0.0.0.0:1741 --private-advertise-endpoint local_ip:1741 -just example orchestrator -- --config_file ./crates/orchestrator/run-config.toml --orchestrator_url http://0.0.0.0:4444 -just example multi-validator-push-cdn -- 10 http://127.0.0.1:4444 -``` - -**Run with GPU-VID** -``` -docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb -just example cdn-marshal -- -d redis://localhost:6379 -b 9000 -just example cdn-broker -- -d redis://localhost:6379 --public-bind-endpoint 0.0.0.0:1740 --public-advertise-endpoint local_ip:1740 --private-bind-endpoint 0.0.0.0:1741 --private-advertise-endpoint local_ip:1741 -just example_fixed_leader orchestrator -- --config_file ./crates/orchestrator/run-config.toml --orchestrator_url http://0.0.0.0:4444 --fixed_leader_for_gpuvid 1 -just example_gpuvid_leader multi-validator-push-cdn -- 1 http://127.0.0.1:4444 -sleep 1m -just example_fixed_leader multi-validator-push-cdn -- 9 http://127.0.0.1:4444 -``` - -Where ones using `example_gpuvid_leader` could be the leader and should be running on a nvidia GPU, and other validators using `example_fixed_leader` will never be a leader. In practice, these url should be changed to the corresponding ip and port. - - -If you don't have a gpu but want to test out fixed leader, you can run: -``` -docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb -just example cdn-marshal -- -d redis://localhost:6379 -b 9000 -just example cdn-broker -- -d redis://localhost:6379 --public-bind-endpoint 0.0.0.0:1740 --public-advertise-endpoint local_ip:1740 --private-bind-endpoint 0.0.0.0:1741 --private-advertise-endpoint local_ip:1741 -just example_fixed_leader orchestrator -- --config_file ./crates/orchestrator/run-config.toml --orchestrator_url http://0.0.0.0:4444 --fixed_leader_for_gpuvid 1 -just example_fixed_leader multi-validator-push-cdn -- 1 http://127.0.0.1:4444 -sleep 1m -just example_fixed_leader multi-validator-push-cdn -- 9 http://127.0.0.1:4444 -``` - -Remember, you have to run leaders first, then other validators, so that leaders will have lower index. \ No newline at end of file diff --git a/crates/examples/push-cdn/all.rs b/crates/examples/push-cdn/all.rs deleted file mode 100644 index 12599c36a4..0000000000 --- a/crates/examples/push-cdn/all.rs +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A example program using the Push CDN -/// The types we're importing -pub mod types; - -use std::path::Path; - -use cdn_broker::{ - reexports::{crypto::signature::KeyPair, def::hook::NoMessageHook}, - Broker, -}; -use cdn_marshal::Marshal; -use hotshot::{ - helpers::initialize_logging, - traits::implementations::{TestingDef, WrappedSignatureKey}, - types::SignatureKey, -}; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use hotshot_types::traits::node_implementation::NodeType; -use infra::{gen_local_address, BUILDER_BASE_PORT}; -use rand::{rngs::StdRng, RngCore, SeedableRng}; -use tokio::spawn; - -use crate::{ - infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}, - types::{Network, NodeImpl, ThisRun}, -}; - -/// The infra implementation -#[path = "../infra/mod.rs"] -pub mod infra; - -use tracing::error; - -#[tokio::main] -async fn main() { - // Initialize logging - initialize_logging(); - - // use configfile args - let (config, orchestrator_url) = read_orchestrator_init_config::(); - - // Start the orhcestrator - spawn(run_orchestrator::(OrchestratorArgs { - url: orchestrator_url.clone(), - config: config.clone(), - })); - - // The configuration we are using for this example is 2 brokers & 1 marshal - - // A keypair shared between brokers - let (broker_public_key, broker_private_key) = - ::SignatureKey::generated_from_seed_indexed([0u8; 32], 1337); - - // Get the OS temporary directory - let temp_dir = std::env::temp_dir(); - - // Create an SQLite file inside of the temporary directory - let discovery_endpoint = temp_dir - .join(Path::new(&format!( - "test-{}.sqlite", - StdRng::from_entropy().next_u64() - ))) - .to_string_lossy() - .into_owned(); - - // 2 brokers - for _ in 0..2 { - // Get the ports to bind to - let private_port = portpicker::pick_unused_port().expect("could not find an open port"); - let public_port = portpicker::pick_unused_port().expect("could not find an open port"); - - // Extrapolate addresses - let private_address = format!("127.0.0.1:{private_port}"); - let public_address = format!("127.0.0.1:{public_port}"); - - let config: cdn_broker::Config::SignatureKey>> = - cdn_broker::Config { - discovery_endpoint: discovery_endpoint.clone(), - public_advertise_endpoint: public_address.clone(), - public_bind_endpoint: public_address, - private_advertise_endpoint: private_address.clone(), - private_bind_endpoint: private_address, - - keypair: KeyPair { - public_key: WrappedSignatureKey(broker_public_key), - private_key: broker_private_key.clone(), - }, - - user_message_hook: NoMessageHook, - broker_message_hook: NoMessageHook, - - metrics_bind_endpoint: None, - ca_cert_path: None, - ca_key_path: None, - global_memory_pool_size: Some(1024 * 1024 * 1024), - }; - - // Create and spawn the broker - spawn(async move { - let broker: Broker::SignatureKey>> = - Broker::new(config).await.expect("broker failed to start"); - - // Error if we stopped unexpectedly - if let Err(err) = broker.start().await { - error!("broker stopped: {err}"); - } - }); - } - - // Get the port to use for the marshal - let marshal_endpoint = config - .cdn_marshal_address - .clone() - .expect("CDN marshal address must be specified"); - - // Configure the marshal - let marshal_config = cdn_marshal::Config { - bind_endpoint: marshal_endpoint.clone(), - discovery_endpoint, - metrics_bind_endpoint: None, - ca_cert_path: None, - ca_key_path: None, - global_memory_pool_size: Some(1024 * 1024 * 1024), - }; - - // Spawn the marshal - spawn(async move { - let marshal: Marshal::SignatureKey>> = - Marshal::new(marshal_config) - .await - .expect("failed to spawn marshal"); - - // Error if we stopped unexpectedly - if let Err(err) = marshal.start().await { - error!("broker stopped: {err}"); - } - }); - - // Start the proper number of nodes - let mut nodes = Vec::new(); - for i in 0..(config.config.num_nodes_with_stake.get()) { - let orchestrator_url = orchestrator_url.clone(); - let builder_address = gen_local_address::(i); - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs { - url: orchestrator_url, - advertise_address: None, - builder_address: Some(builder_address), - network_config_file: None, - }, - ) - .await; - }); - nodes.push(node); - } - let _result = futures::future::join_all(nodes).await; -} diff --git a/crates/examples/push-cdn/broker.rs b/crates/examples/push-cdn/broker.rs deleted file mode 100644 index 8e03999c33..0000000000 --- a/crates/examples/push-cdn/broker.rs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! The following is the main `Broker` binary, which just instantiates and runs -//! a `Broker` object. -use anyhow::Result; -use cdn_broker::{reexports::def::hook::NoMessageHook, Broker, Config}; -use clap::Parser; -use hotshot::traits::implementations::{KeyPair, ProductionDef, WrappedSignatureKey}; -use hotshot_example_types::node_types::TestTypes; -use hotshot_types::traits::{node_implementation::NodeType, signature_key::SignatureKey}; -use sha2::Digest; -use tracing_subscriber::EnvFilter; - -#[derive(Parser, Debug)] -#[command(author, version, about, long_about = None)] -/// The main component of the push CDN. -struct Args { - /// The discovery client endpoint (including scheme) to connect to. - /// With the local discovery feature, this is a file path. - /// With the remote (redis) discovery feature, this is a redis URL (e.g. `redis://127.0.0.1:6789`). - #[arg(short, long)] - discovery_endpoint: String, - - /// The user-facing endpoint in `IP:port` form to bind to for connections from users - #[arg(long, default_value = "0.0.0.0:1738")] - public_bind_endpoint: String, - - /// The user-facing endpoint in `IP:port` form to advertise - #[arg(long, default_value = "local_ip:1738")] - public_advertise_endpoint: String, - - /// The broker-facing endpoint in `IP:port` form to bind to for connections from - /// other brokers - #[arg(long, default_value = "0.0.0.0:1739")] - private_bind_endpoint: String, - - /// The broker-facing endpoint in `IP:port` form to advertise - #[arg(long, default_value = "local_ip:1739")] - private_advertise_endpoint: String, - - /// The endpoint to bind to for externalizing metrics (in `IP:port` form). If not provided, - /// metrics are not exposed. - #[arg(short, long)] - metrics_bind_endpoint: Option, - - /// The path to the CA certificate - /// If not provided, a local, pinned CA is used - #[arg(long)] - ca_cert_path: Option, - - /// The path to the CA key - /// If not provided, a local, pinned CA is used - #[arg(long)] - ca_key_path: Option, - - /// The seed for broker key generation - #[arg(short, long, default_value_t = 0)] - key_seed: u64, - - /// The size of the global memory pool (in bytes). This is the maximum number of bytes that - /// can be allocated at once for all connections. A connection will block if it - /// tries to allocate more than this amount until some memory is freed. - /// Default is 1GB. - #[arg(long, default_value_t = 1_073_741_824)] - global_memory_pool_size: usize, -} - -#[tokio::main] -async fn main() -> Result<()> { - // Parse command line arguments - let args = Args::parse(); - - // Initialize tracing - if std::env::var("RUST_LOG_FORMAT") == Ok("json".to_string()) { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .json() - .init(); - } else { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .init(); - } - - // Generate the broker key from the supplied seed - let key_hash = sha2::Sha256::digest(args.key_seed.to_le_bytes()); - let (public_key, private_key) = - ::SignatureKey::generated_from_seed_indexed(key_hash.into(), 1337); - - // Create config - let broker_config: Config::SignatureKey>> = Config { - ca_cert_path: args.ca_cert_path, - ca_key_path: args.ca_key_path, - - discovery_endpoint: args.discovery_endpoint, - metrics_bind_endpoint: args.metrics_bind_endpoint, - keypair: KeyPair { - public_key: WrappedSignatureKey(public_key), - private_key, - }, - - user_message_hook: NoMessageHook, - broker_message_hook: NoMessageHook, - - public_bind_endpoint: args.public_bind_endpoint, - public_advertise_endpoint: args.public_advertise_endpoint, - private_bind_endpoint: args.private_bind_endpoint, - private_advertise_endpoint: args.private_advertise_endpoint, - global_memory_pool_size: Some(args.global_memory_pool_size), - }; - - // Create new `Broker` - // Uses TCP from broker connections and TCP+TLS for user connections. - let broker = Broker::new(broker_config).await?; - - // Start the main loop, consuming it - broker.start().await?; - - Ok(()) -} diff --git a/crates/examples/push-cdn/marshal.rs b/crates/examples/push-cdn/marshal.rs deleted file mode 100644 index 569cb0dc33..0000000000 --- a/crates/examples/push-cdn/marshal.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! The following is the main `Marshal` binary, which just instantiates and runs -//! a `Marshal` object. - -use anyhow::Result; -use cdn_marshal::{Config, Marshal}; -use clap::Parser; -use hotshot::traits::implementations::ProductionDef; -use hotshot_example_types::node_types::TestTypes; -use hotshot_types::traits::node_implementation::NodeType; -use tracing_subscriber::EnvFilter; - -// TODO: forall, add logging where we need it - -#[derive(Parser, Debug)] -#[command(author, version, about, long_about = None)] -/// The main component of the push CDN. -struct Args { - /// The discovery client endpoint (including scheme) to connect to - #[arg(short, long)] - discovery_endpoint: String, - - /// The port to bind to for connections (from users) - #[arg(short, long, default_value_t = 1737)] - bind_port: u16, - - /// The endpoint to bind to for externalizing metrics (in `IP:port` form). If not provided, - /// metrics are not exposed. - #[arg(short, long)] - metrics_bind_endpoint: Option, - - /// The path to the CA certificate - /// If not provided, a local, pinned CA is used - #[arg(long)] - ca_cert_path: Option, - - /// The path to the CA key - /// If not provided, a local, pinned CA is used - #[arg(long)] - ca_key_path: Option, - - /// The size of the global memory pool (in bytes). This is the maximum number of bytes that - /// can be allocated at once for all connections. A connection will block if it - /// tries to allocate more than this amount until some memory is freed. - /// Default is 1GB. - #[arg(long, default_value_t = 1_073_741_824)] - global_memory_pool_size: usize, -} - -#[tokio::main] -async fn main() -> Result<()> { - // Parse command-line arguments - let args = Args::parse(); - - // Initialize tracing - if std::env::var("RUST_LOG_FORMAT") == Ok("json".to_string()) { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .json() - .init(); - } else { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .init(); - } - - // Create a new `Config` - let config = Config { - discovery_endpoint: args.discovery_endpoint, - bind_endpoint: format!("0.0.0.0:{}", args.bind_port), - metrics_bind_endpoint: args.metrics_bind_endpoint, - ca_cert_path: args.ca_cert_path, - ca_key_path: args.ca_key_path, - global_memory_pool_size: Some(args.global_memory_pool_size), - }; - - // Create new `Marshal` from the config - let marshal = - Marshal::::SignatureKey>>::new(config).await?; - - // Start the main loop, consuming it - marshal.start().await?; - - Ok(()) -} diff --git a/crates/examples/push-cdn/multi-validator.rs b/crates/examples/push-cdn/multi-validator.rs deleted file mode 100644 index 54718468b3..0000000000 --- a/crates/examples/push-cdn/multi-validator.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A multi validator -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::{MultiValidatorArgs, ValidatorArgs}; -use tokio::spawn; -use tracing::instrument; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let args = MultiValidatorArgs::parse(); - tracing::debug!("connecting to orchestrator at {:?}", args.url); - let mut nodes = Vec::new(); - for node_index in 0..args.num_nodes { - let args = args.clone(); - - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs::from_multi_args(args, node_index), - ) - .await; - }); - nodes.push(node); - } - let _result = futures::future::join_all(nodes).await; -} diff --git a/crates/examples/push-cdn/types.rs b/crates/examples/push-cdn/types.rs deleted file mode 100644 index 8803c72152..0000000000 --- a/crates/examples/push-cdn/types.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -use hotshot::traits::{implementations::PushCdnNetwork, NodeImplementation}; -use hotshot_example_types::{ - auction_results_provider_types::TestAuctionResultsProvider, state_types::TestTypes, - storage_types::TestStorage, -}; -use hotshot_types::traits::node_implementation::NodeType; -use serde::{Deserialize, Serialize}; - -use crate::infra::PushCdnDaRun; - -#[derive(Clone, Deserialize, Serialize, Hash, PartialEq, Eq)] -/// Convenience type alias -pub struct NodeImpl {} - -/// Convenience type alias -pub type Network = PushCdnNetwork<::SignatureKey>; - -impl NodeImplementation for NodeImpl { - type Network = Network; - type Storage = TestStorage; - type AuctionResultsProvider = TestAuctionResultsProvider; -} - -/// Convenience type alias -pub type ThisRun = PushCdnDaRun; diff --git a/crates/examples/push-cdn/validator.rs b/crates/examples/push-cdn/validator.rs deleted file mode 100644 index 7b546dfabe..0000000000 --- a/crates/examples/push-cdn/validator.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A validator -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use tracing::{debug, instrument}; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let args = ValidatorArgs::parse(); - debug!("connecting to orchestrator at {:?}", args.url); - infra::main_entry_point::(args).await; -} diff --git a/crates/examples/push-cdn/whitelist-adapter.rs b/crates/examples/push-cdn/whitelist-adapter.rs deleted file mode 100644 index e855a41aba..0000000000 --- a/crates/examples/push-cdn/whitelist-adapter.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! The whitelist is an adaptor that is able to update the allowed public keys for -//! all brokers. Right now, we do this by asking the orchestrator for the list of -//! allowed public keys. In the future, we will pull the stake table from the L1. - -use std::{str::FromStr, sync::Arc}; - -use anyhow::{Context, Result}; -use cdn_broker::reexports::discovery::{DiscoveryClient, Embedded, Redis}; -use clap::Parser; -use hotshot_example_types::node_types::TestTypes; -use hotshot_orchestrator::client::OrchestratorClient; -use hotshot_types::{ - network::NetworkConfig, - traits::{node_implementation::NodeType, signature_key::SignatureKey}, -}; -use surf_disco::Url; - -#[derive(Parser, Debug)] -#[command(author, version, about, long_about = None)] -/// The main component of the push CDN. -struct Args { - /// The discovery client endpoint (including scheme) to connect to. - /// With the local discovery feature, this is a file path. - /// With the remote (redis) discovery feature, this is a redis URL (e.g. `redis://127.0.0.1:6789`). - #[arg(short, long)] - discovery_endpoint: String, - - /// The URL the orchestrator is running on. This should be something like `http://localhost:5555` - #[arg(short, long)] - orchestrator_url: String, - - /// Whether or not to use the local discovery client - #[arg(short, long)] - local_discovery: bool, -} - -#[tokio::main] -async fn main() -> Result<()> { - // Parse the command line arguments - let args = Args::parse(); - - // Initialize tracing - tracing_subscriber::fmt::init(); - - // Create a new `OrchestratorClient` from the supplied URL - let orchestrator_client = OrchestratorClient::new( - Url::from_str(&args.orchestrator_url).with_context(|| "Invalid URL")?, - ); - - // Attempt to get the config from the orchestrator. - // Loops internally until the config is received. - let config: NetworkConfig<::SignatureKey> = - orchestrator_client.get_config_after_collection().await; - - tracing::info!("Received config from orchestrator"); - - // Extrapolate the state_ver_keys from the config and convert them to a compatible format - let whitelist = config - .config - .known_nodes_with_stake - .iter() - .map(|k| Arc::from(k.stake_table_entry.stake_key.to_bytes())) - .collect(); - - if args.local_discovery { - ::new(args.discovery_endpoint, None) - .await? - .set_whitelist(whitelist) - .await?; - } else { - ::new(args.discovery_endpoint, None) - .await? - .set_whitelist(whitelist) - .await?; - } - - tracing::info!("Posted config to discovery endpoint"); - - Ok(()) -} diff --git a/crates/examples/src/lib.rs b/crates/examples/src/lib.rs new file mode 100644 index 0000000000..a7001964d2 --- /dev/null +++ b/crates/examples/src/lib.rs @@ -0,0 +1 @@ +//! This crate contains examples of HotShot usage diff --git a/crates/hotshot/Cargo.toml b/crates/hotshot/Cargo.toml index a74bc74746..75aa2384e3 100644 --- a/crates/hotshot/Cargo.toml +++ b/crates/hotshot/Cargo.toml @@ -23,7 +23,6 @@ anyhow = { workspace = true } async-broadcast = { workspace = true } async-lock = { workspace = true } async-trait = { workspace = true } -bimap = "0.6" bincode = { workspace = true } blake3 = { workspace = true } cdn-broker = { workspace = true, features = ["global-permits"] } @@ -47,7 +46,6 @@ portpicker = "0.1" primitive-types = { workspace = true } rand = { workspace = true } serde = { workspace = true, features = ["rc"] } -sha2 = { workspace = true } time = { workspace = true } tokio = { workspace = true } diff --git a/crates/hotshot/src/traits.rs b/crates/hotshot/src/traits.rs index 1ff090fc06..6022efca7e 100644 --- a/crates/hotshot/src/traits.rs +++ b/crates/hotshot/src/traits.rs @@ -10,7 +10,6 @@ mod networking; mod node_implementation; pub use hotshot_types::traits::{BlockPayload, ValidatedState}; -pub use libp2p_networking::network::NetworkNodeConfigBuilder; pub use networking::{NetworkError, NetworkReliability}; pub use node_implementation::{NodeImplementation, TestableNodeImplementation}; @@ -20,7 +19,7 @@ pub mod implementations { combined_network::{CombinedNetworks, UnderlyingCombinedNetworks}, libp2p_network::{ derive_libp2p_keypair, derive_libp2p_multiaddr, derive_libp2p_peer_id, GossipConfig, - Libp2pMetricsValue, Libp2pNetwork, PeerInfoVec, RequestResponseConfig, + Libp2pMetricsValue, Libp2pNetwork, RequestResponseConfig, }, memory_network::{MasterMap, MemoryNetwork}, push_cdn_network::{ diff --git a/crates/hotshot/src/traits/networking/combined_network.rs b/crates/hotshot/src/traits/networking/combined_network.rs index 6c891fa6f7..d7b3c0d6a6 100644 --- a/crates/hotshot/src/traits/networking/combined_network.rs +++ b/crates/hotshot/src/traits/networking/combined_network.rs @@ -255,7 +255,6 @@ pub struct UnderlyingCombinedNetworks( impl TestableNetworkingImplementation for CombinedNetworks { fn generator( expected_node_count: usize, - num_bootstrap: usize, network_id: usize, da_committee_size: usize, reliability_config: Option>, @@ -264,7 +263,6 @@ impl TestableNetworkingImplementation for CombinedNetwor let generators = ( as TestableNetworkingImplementation>::generator( expected_node_count, - num_bootstrap, network_id, da_committee_size, None, @@ -272,7 +270,6 @@ impl TestableNetworkingImplementation for CombinedNetwor ), as TestableNetworkingImplementation>::generator( expected_node_count, - num_bootstrap, network_id, da_committee_size, reliability_config, diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index 0f1fbfc858..ac40cabca5 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -10,11 +10,9 @@ #[cfg(feature = "hotshot-testing")] use std::str::FromStr; use std::{ - cmp::min, - collections::{BTreeSet, HashSet}, + collections::HashSet, fmt::Debug, net::{IpAddr, ToSocketAddrs}, - num::NonZeroUsize, sync::{ atomic::{AtomicBool, AtomicU64, Ordering}, Arc, @@ -23,9 +21,7 @@ use std::{ }; use anyhow::{anyhow, Context}; -use async_lock::RwLock; use async_trait::async_trait; -use bimap::BiHashMap; use futures::future::join_all; #[cfg(feature = "hotshot-testing")] use hotshot_types::traits::network::{ @@ -35,7 +31,7 @@ use hotshot_types::{ boxed_sync, constants::LOOK_AHEAD, data::ViewNumber, - network::NetworkConfig, + light_client::StateVerKey, traits::{ election::Membership, metrics::{Counter, Gauge, Metrics, NoMetrics}, @@ -43,7 +39,7 @@ use hotshot_types::{ node_implementation::{ConsensusTime, NodeType}, signature_key::{PrivateSignatureKey, SignatureKey}, }, - BoxSyncFuture, + BoxSyncFuture, PeerConfig, }; use libp2p_identity::{ ed25519::{self, SecretKey}, @@ -53,15 +49,14 @@ pub use libp2p_networking::network::{GossipConfig, RequestResponseConfig}; use libp2p_networking::{ network::{ behaviours::dht::record::{Namespace, RecordKey, RecordValue}, + node::config::{KademliaConfig, Libp2pConfig}, spawn_network_node, transport::construct_auth_message, NetworkEvent::{self, DirectRequest, DirectResponse, GossipMsg}, - NetworkNodeConfig, NetworkNodeConfigBuilder, NetworkNodeHandle, NetworkNodeReceiver, - DEFAULT_REPLICATION_FACTOR, + NetworkNodeHandle, NetworkNodeReceiver, }, reexport::Multiaddr, }; -use rand::{rngs::StdRng, seq::IteratorRandom, SeedableRng}; use serde::Serialize; use tokio::{ select, spawn, @@ -108,12 +103,8 @@ impl Default for Libp2pMetricsValue { } } -/// convenience alias for the type for bootstrap addresses -/// concurrency primitives are needed for having tests -pub type BootstrapAddrs = Arc>>; - -/// hardcoded topic of QC used -pub const QC_TOPIC: &str = "global"; +/// The topic that all nodes subscribe to +pub const GLOBAL_TOPIC: &str = "global"; /// Stubbed out Ack /// @@ -136,25 +127,20 @@ impl Debug for Libp2pNetwork { } } -/// Type alias for a shared collection of peerid, multiaddrs -pub type PeerInfoVec = Arc>>; - /// The underlying state of the libp2p network #[derive(Debug)] struct Libp2pNetworkInner { - /// this node's public key - pk: T::SignatureKey, - /// handle to control the network - handle: Arc>, + /// The HotShot public key + hotshot_public_key: T::SignatureKey, + + /// The handle that we use to send requests to Libp2p + handle: Arc>, /// Message Receiver receiver: Mutex>>, /// Sender for broadcast messages sender: Sender>, /// Sender for node lookup (relevant view number, key of node) (None for shutdown) node_lookup_send: Sender>, - /// this is really cheating to enable local tests - /// hashset of (bootstrap_addr, peer_id) - bootstrap_addrs: PeerInfoVec, /// whether or not the network is ready to send is_ready: Arc, /// max time before dropping message due to DHT error @@ -184,6 +170,25 @@ pub struct Libp2pNetwork { inner: Arc>, } +/// Generate the expected [testing] multiaddr from a node index +fn multiaddr_from_node_index(i: usize) -> Multiaddr { + // Generate the node's private key from the node ID + let peers_hotshot_private_key = + T::SignatureKey::generated_from_seed_indexed([0u8; 32], i as u64).1; + + // Derive the Libp2p keypair from the private key + let peers_libp2p_keypair = derive_libp2p_keypair::(&peers_hotshot_private_key) + .expect("Failed to derive libp2p keypair"); + + // Generate the multiaddress using the peer id and port + Multiaddr::from_str(&format!( + "/ip4/127.0.0.1/udp/{}/quic-v1/p2p/{}", + 48000 + i, + peers_libp2p_keypair.public().to_peer_id() + )) + .expect("Failed to create multiaddr") +} + #[cfg(feature = "hotshot-testing")] impl TestableNetworkingImplementation for Libp2pNetwork { /// Returns a boxed function `f(node_id, public_key) -> Libp2pNetwork` @@ -198,7 +203,6 @@ impl TestableNetworkingImplementation for Libp2pNetwork { #[allow(clippy::panic, clippy::too_many_lines)] fn generator( expected_node_count: usize, - num_bootstrap: usize, _network_id: usize, da_committee_size: usize, reliability_config: Option>, @@ -208,76 +212,100 @@ impl TestableNetworkingImplementation for Libp2pNetwork { da_committee_size <= expected_node_count, "DA committee size must be less than or equal to total # nodes" ); - let bootstrap_addrs: PeerInfoVec = Arc::default(); - let node_ids: Arc>> = Arc::default(); // NOTE uncomment this for easier debugging - // let start_port = 5000; Box::pin({ move |node_id| { - info!( - "GENERATOR: Node id {:?}, is bootstrap: {:?}", - node_id, - node_id < num_bootstrap as u64 - ); - - // pick a free, unused UDP port for testing - let port = portpicker::pick_unused_port().expect("Could not find an open port"); + // The port is 48000 + the node id. This way it's deterministic and easy to connect nodes together + let port = 48000 + node_id; - let addr = + // Create the bind address + let bind_address = Multiaddr::from_str(&format!("/ip4/127.0.0.1/udp/{port}/quic-v1")).unwrap(); - // We assign node's public key and stake value rather than read from config file since it's a test - let privkey = T::SignatureKey::generated_from_seed_indexed([0u8; 32], node_id).1; - let pubkey = T::SignatureKey::from_private(&privkey); + // Deterministically generate the private key from the node ID + let hotshot_private_key = + T::SignatureKey::generated_from_seed_indexed([0u8; 32], node_id).1; + let hotshot_public_key = T::SignatureKey::from_private(&hotshot_private_key); // Derive the Libp2p keypair from the private key - let libp2p_keypair = derive_libp2p_keypair::(&privkey) + let libp2p_keypair = derive_libp2p_keypair::(&hotshot_private_key) .expect("Failed to derive libp2p keypair"); // Sign the lookup record let lookup_record_value = RecordValue::new_signed( - &RecordKey::new(Namespace::Lookup, pubkey.to_bytes()), + &RecordKey::new(Namespace::Lookup, hotshot_public_key.to_bytes()), libp2p_keypair.public().to_peer_id().to_bytes(), - &privkey, + &hotshot_private_key, ) .expect("Failed to sign DHT lookup record"); - // We want at least 2/3 of the nodes to have any given record in the DHT - let replication_factor = - NonZeroUsize::new((2 * expected_node_count).div_ceil(3)).unwrap(); - - // Build the network node configuration - let config = NetworkNodeConfigBuilder::default() - .keypair(libp2p_keypair) - .replication_factor(replication_factor) - .bind_address(Some(addr)) - .to_connect_addrs(HashSet::default()) - .republication_interval(None) - .build() - .expect("Failed to build network node config"); - - let bootstrap_addrs_ref = Arc::clone(&bootstrap_addrs); - let node_ids_ref = Arc::clone(&node_ids); + // Configure Kademlia with our lookup record value + let kademlia_config = KademliaConfig:: { + // At least 2/3 of the nodes should have any given record + replication_factor: 2 * expected_node_count / 3, + record_ttl: None, + publication_interval: None, + file_path: format!("/tmp/libp2p_dht-{node_id}.bin"), + lookup_record_value, + }; + + // Use the default gossip configuration + let gossip_config = GossipConfig::default(); + + // Use the default request response configuration + let request_response_config = RequestResponseConfig::default(); + + // Create and sign an authentication message over our Libp2p `PeerID` + let auth_message = construct_auth_message( + &hotshot_public_key, + &libp2p_keypair.public().to_peer_id(), + &hotshot_private_key, + ) + .expect("Failed to construct authentication message"); + + // Create the list of known peers + let known_peers = (0..expected_node_count) + .map(|i| multiaddr_from_node_index::(i)) + .collect(); + + // Collect the `PeerConfig`s of all nodes + let mut all_nodes = Vec::new(); + for i in 0..expected_node_count { + // Calculate the staking key from the node ID + let stake_key = + T::SignatureKey::generated_from_seed_indexed([0u8; 32], i as u64).0; + + // Push the peer config to the list of all nodes + all_nodes.push(PeerConfig { + state_ver_key: StateVerKey::default(), + stake_table_entry: T::SignatureKey::stake_table_entry(&stake_key, 1), + }); + } + + // Create the quorum membership from the list we just made + let quorum_membership = T::Membership::new(all_nodes.clone(), all_nodes); + + // Create the Libp2p configuration + let config = Libp2pConfig:: { + keypair: libp2p_keypair, + bind_address, + known_peers, + quorum_membership: Some(quorum_membership), + auth_message: Some(auth_message), + kademlia_config, + gossip_config, + request_response_config, + }; + let reliability_config_dup = reliability_config.clone(); Box::pin(async move { - // If it's the second time we are starting this network, clear the bootstrap info - let mut write_ids = node_ids_ref.write().await; - if write_ids.contains(&node_id) { - write_ids.clear(); - bootstrap_addrs_ref.write().await.clear(); - } - write_ids.insert(node_id); - drop(write_ids); Arc::new( match Libp2pNetwork::new( - Libp2pMetricsValue::default(), config, - pubkey.clone(), - lookup_record_value, - bootstrap_addrs_ref, - usize::try_from(node_id).unwrap(), + &hotshot_public_key, + Libp2pMetricsValue::default(), #[cfg(feature = "hotshot-testing")] reliability_config_dup, ) @@ -379,8 +407,7 @@ pub fn derive_libp2p_multiaddr(addr: &String) -> anyhow::Result { } impl Libp2pNetwork { - /// Create and return a Libp2p network from a network config file - /// and various other configuration-specific values. + /// Create and return a Libp2p network from a Libp2p config /// /// # Errors /// If we are unable to parse a Multiaddress @@ -388,150 +415,23 @@ impl Libp2pNetwork { /// # Panics /// If we are unable to calculate the replication factor #[allow(clippy::too_many_arguments)] - pub async fn from_config( - mut config: NetworkConfig, - quorum_membership: T::Membership, - gossip_config: GossipConfig, - request_response_config: RequestResponseConfig, - bind_address: Multiaddr, - pub_key: &T::SignatureKey, - priv_key: &::PrivateKey, - metrics: Libp2pMetricsValue, - ) -> anyhow::Result { - // Try to take our Libp2p config from our broader network config - let libp2p_config = config - .libp2p_config - .take() - .ok_or(anyhow!("Libp2p config not supplied"))?; - - // Derive our Libp2p keypair from our supplied private key - let keypair = derive_libp2p_keypair::(priv_key)?; - - // Build our libp2p configuration - let mut config_builder = NetworkNodeConfigBuilder::default(); - - // Set the gossip configuration - config_builder.gossip_config(gossip_config.clone()); - config_builder.request_response_config(request_response_config); - - // Construct the auth message - let auth_message = - construct_auth_message(pub_key, &keypair.public().to_peer_id(), priv_key) - .with_context(|| "Failed to construct auth message")?; - - // Set the auth message and stake table - config_builder - .stake_table(Some(quorum_membership)) - .auth_message(Some(auth_message)); - - // The replication factor is the minimum of [the default and 2/3 the number of nodes] - let Some(default_replication_factor) = DEFAULT_REPLICATION_FACTOR else { - return Err(anyhow!("Default replication factor not supplied")); - }; - - let replication_factor = NonZeroUsize::new(min( - default_replication_factor.get(), - config.config.num_nodes_with_stake.get() * 2 / 3, - )) - .with_context(|| "Failed to calculate replication factor")?; - - // Sign our DHT lookup record - let lookup_record_value = RecordValue::new_signed( - &RecordKey::new(Namespace::Lookup, pub_key.to_bytes()), - // The value is our Libp2p Peer ID - keypair.public().to_peer_id().to_bytes(), - priv_key, - ) - .with_context(|| "Failed to sign DHT lookup record")?; - - config_builder - .keypair(keypair) - .replication_factor(replication_factor) - .bind_address(Some(bind_address.clone())); - - // Choose `mesh_n` random nodes to connect to for bootstrap - let bootstrap_nodes = libp2p_config - .bootstrap_nodes - .into_iter() - .choose_multiple(&mut StdRng::from_entropy(), gossip_config.mesh_n); - config_builder.to_connect_addrs(HashSet::from_iter(bootstrap_nodes.clone())); - - // Build the node's configuration - let node_config = config_builder.build()?; - - // Calculate all keys so we can keep track of direct message recipients - let mut all_keys = BTreeSet::new(); - - // Insert all known nodes into the set of all keys - for node in config.config.known_nodes_with_stake { - all_keys.insert(T::SignatureKey::public_key(&node.stake_table_entry)); - } - - Ok(Libp2pNetwork::new( - metrics, - node_config, - pub_key.clone(), - lookup_record_value, - Arc::new(RwLock::new(bootstrap_nodes)), - usize::try_from(config.node_index)?, - #[cfg(feature = "hotshot-testing")] - None, - ) - .await?) - } - - /// Returns whether or not the network is currently ready. - #[must_use] - pub fn is_ready(&self) -> bool { - self.inner.is_ready.load(Ordering::Relaxed) - } - - /// Returns only when the network is ready. - pub async fn wait_for_ready(&self) { - loop { - if self.is_ready() { - break; - } - sleep(Duration::from_secs(1)).await; - } - } - - /// Constructs new network for a node. Note that this network is unconnected. - /// One must call `connect` in order to connect. - /// * `config`: the configuration of the node - /// * `pk`: public key associated with the node - /// * `bootstrap_addrs`: rwlock containing the bootstrap addrs - /// # Errors - /// Returns error in the event that the underlying libp2p network - /// is unable to create a network. - /// - /// # Panics - /// - /// This will panic if there are less than 5 bootstrap nodes - #[allow(clippy::too_many_arguments)] pub async fn new( + config: Libp2pConfig, + hotshot_public_key: &T::SignatureKey, metrics: Libp2pMetricsValue, - config: NetworkNodeConfig, - pk: T::SignatureKey, - lookup_record_value: RecordValue, - bootstrap_addrs: BootstrapAddrs, - id: usize, #[cfg(feature = "hotshot-testing")] reliability_config: Option>, - ) -> Result, NetworkError> { - let (mut rx, network_handle) = spawn_network_node::(config.clone(), id) + ) -> anyhow::Result { + // Warn in case the node was accidentally compiled with the `hotshot-testing` feature + #[cfg(feature = "hotshot-testing")] + warn!("Compiled with `hotshot-testing` feature, Libp2p messages will be unreliable"); + + // Spawn the network node with a copy of our config + let (mut rx, network_handle) = spawn_network_node::(config.clone()) .await .map_err(|e| NetworkError::ConfigError(format!("failed to spawn network node: {e}")))?; - // Add our own address to the bootstrap addresses - let addr = network_handle.listen_addr(); - let pid = network_handle.peer_id(); - bootstrap_addrs.write().await.push((pid, addr)); - - let mut pubkey_pid_map = BiHashMap::new(); - pubkey_pid_map.insert(pk.clone(), network_handle.peer_id()); - - // Subscribe to the relevant topics - let subscribed_topics = HashSet::from_iter(vec![QC_TOPIC.to_string()]); + // Subscribe only to the global topic (DA messages are direct-broadcasted) + let subscribed_topics = HashSet::from_iter(vec![GLOBAL_TOPIC.to_string()]); // unbounded channels may not be the best choice (spammed?) // if bounded figure out a way to log dropped msgs @@ -545,11 +445,9 @@ impl Libp2pNetwork { handle: Arc::new(network_handle), receiver: Mutex::new(receiver), sender: sender.clone(), - pk, - bootstrap_addrs, + hotshot_public_key: hotshot_public_key.clone(), is_ready: Arc::new(AtomicBool::new(false)), - // This is optimal for 10-30 nodes. TODO: parameterize this for both tests and examples - dht_timeout: config.dht_timeout.unwrap_or(Duration::from_secs(120)), + dht_timeout: Duration::from_secs(60), is_bootstrapped: Arc::new(AtomicBool::new(false)), metrics, subscribed_topics, @@ -569,11 +467,27 @@ impl Libp2pNetwork { result.handle_event_generator(sender, rx); result.spawn_node_lookup(node_lookup_recv); - result.spawn_connect(id, lookup_record_value); + result.spawn_connect(config.kademlia_config.lookup_record_value); Ok(result) } + /// Returns whether or not the network is currently ready. + #[must_use] + pub fn is_ready(&self) -> bool { + self.inner.is_ready.load(Ordering::Relaxed) + } + + /// Returns only when the network is ready. + pub async fn wait_for_ready(&self) { + loop { + if self.is_ready() { + break; + } + sleep(Duration::from_secs(1)).await; + } + } + /// Spawns task for looking up nodes pre-emptively #[allow(clippy::cast_sign_loss, clippy::cast_precision_loss)] fn spawn_node_lookup( @@ -606,21 +520,15 @@ impl Libp2pNetwork { } /// Initiates connection to the outside world - fn spawn_connect(&mut self, id: usize, lookup_record_value: RecordValue) { - let pk = self.inner.pk.clone(); - let bootstrap_ref = Arc::clone(&self.inner.bootstrap_addrs); + fn spawn_connect(&mut self, lookup_record_value: RecordValue) { let handle = Arc::clone(&self.inner.handle); let is_bootstrapped = Arc::clone(&self.inner.is_bootstrapped); let inner = Arc::clone(&self.inner); + let hotshot_public_key = self.inner.hotshot_public_key.clone(); spawn({ let is_ready = Arc::clone(&self.inner.is_ready); async move { - let bs_addrs = bootstrap_ref.read().await.clone(); - - // Add known peers to the network - handle.add_known_peers(bs_addrs).unwrap(); - // Begin the bootstrap process handle.begin_bootstrap()?; while !is_bootstrapped.load(Ordering::Relaxed) { @@ -628,14 +536,14 @@ impl Libp2pNetwork { handle.begin_bootstrap()?; } - // Subscribe to the QC topic - handle.subscribe(QC_TOPIC.to_string()).await.unwrap(); + // Subscribe to the global topic + handle.subscribe(GLOBAL_TOPIC.to_string()).await.unwrap(); // Map our staking key to our Libp2p Peer ID so we can properly // route direct messages while handle .put_record( - RecordKey::new(Namespace::Lookup, pk.to_bytes()), + RecordKey::new(Namespace::Lookup, hotshot_public_key.to_bytes()), lookup_record_value.clone(), ) .await @@ -645,7 +553,7 @@ impl Libp2pNetwork { } // Wait for the network to connect to the required number of peers - if let Err(e) = handle.wait_to_connect(4, id).await { + if let Err(e) = handle.wait_to_connect(4).await { error!("Failed to connect to peers: {:?}", e); return Err::<(), NetworkError>(e); } @@ -878,8 +786,8 @@ impl ConnectedNetwork for Libp2pNetwork { return Err(NetworkError::NotReadyYet); }; - // short circuit if we're dming ourselves - if recipient == self.inner.pk { + // Short circuit if we're trying to message ourselves + if recipient == self.inner.hotshot_public_key { // panic if we already shut down? self.inner.sender.try_send(message).map_err(|_x| { self.inner.metrics.num_failed_messages.add(1); diff --git a/crates/hotshot/src/traits/networking/memory_network.rs b/crates/hotshot/src/traits/networking/memory_network.rs index 5925a85eff..7c7665b051 100644 --- a/crates/hotshot/src/traits/networking/memory_network.rs +++ b/crates/hotshot/src/traits/networking/memory_network.rs @@ -180,7 +180,6 @@ impl TestableNetworkingImplementation { fn generator( _expected_node_count: usize, - _num_bootstrap: usize, _network_id: usize, da_committee_size: usize, reliability_config: Option>, diff --git a/crates/hotshot/src/traits/networking/push_cdn_network.rs b/crates/hotshot/src/traits/networking/push_cdn_network.rs index 553b2545ef..9f7391ba9e 100644 --- a/crates/hotshot/src/traits/networking/push_cdn_network.rs +++ b/crates/hotshot/src/traits/networking/push_cdn_network.rs @@ -281,7 +281,6 @@ impl TestableNetworkingImplementation #[allow(clippy::too_many_lines)] fn generator( _expected_node_count: usize, - _num_bootstrap: usize, _network_id: usize, da_committee_size: usize, _reliability_config: Option>, diff --git a/crates/libp2p-networking/Cargo.toml b/crates/libp2p-networking/Cargo.toml index 98f1449508..b320e57b80 100644 --- a/crates/libp2p-networking/Cargo.toml +++ b/crates/libp2p-networking/Cargo.toml @@ -20,8 +20,6 @@ bincode = { workspace = true } blake3 = { workspace = true } cbor4ii = "0.3" delegate = "0.13" -derive_builder = "0.20" -derive_more = { workspace = true } futures = { workspace = true } hotshot-types = { path = "../types" } lazy_static = { workspace = true } diff --git a/crates/libp2p-networking/src/lib.rs b/crates/libp2p-networking/src/lib.rs index 6b71165223..31bfee245f 100644 --- a/crates/libp2p-networking/src/lib.rs +++ b/crates/libp2p-networking/src/lib.rs @@ -9,7 +9,7 @@ /// Network logic pub mod network; -/// symbols needed to implement a networking instance over libp2p-netorking +/// Re-exports from libp2p pub mod reexport { pub use libp2p::{request_response::ResponseChannel, Multiaddr}; pub use libp2p_identity::PeerId; diff --git a/crates/libp2p-networking/src/network/behaviours/dht/mod.rs b/crates/libp2p-networking/src/network/behaviours/dht/mod.rs index f785d1fa10..a963fc785b 100644 --- a/crates/libp2p-networking/src/network/behaviours/dht/mod.rs +++ b/crates/libp2p-networking/src/network/behaviours/dht/mod.rs @@ -27,7 +27,6 @@ use libp2p::kad::{ use libp2p::kad::{ store::RecordStore, Behaviour as KademliaBehaviour, BootstrapError, Event as KademliaEvent, }; -use libp2p_identity::PeerId; use store::{file_backed::FileBackedStore, validated::ValidatedStore}; use tokio::{spawn, sync::mpsc::UnboundedSender, time::sleep}; use tracing::{debug, error, warn}; @@ -68,10 +67,6 @@ pub struct DHTBehaviour { in_progress_put_record_queries: HashMap, /// State of bootstrapping pub bootstrap_state: Bootstrap, - /// the peer id (useful only for debugging right now) - pub peer_id: PeerId, - /// replication factor - pub replication_factor: NonZeroUsize, /// Sender to retry requests. retry_tx: Option>, /// Sender to the bootstrap task @@ -81,6 +76,13 @@ pub struct DHTBehaviour { phantom: PhantomData, } +/// The default implementation just calls the constructor +impl Default for DHTBehaviour { + fn default() -> Self { + Self::new() + } +} + /// State of bootstrapping #[derive(Debug, Clone)] pub struct Bootstrap { @@ -117,14 +119,13 @@ impl DHTBehaviour { } /// Create a new DHT behaviour #[must_use] - pub fn new(pid: PeerId, replication_factor: NonZeroUsize) -> Self { + pub fn new() -> Self { // needed because otherwise we stay in client mode when testing locally // and don't publish keys stuff // e.g. dht just doesn't work. We'd need to add mdns and that doesn't seem worth it since // we won't have a local network // Self { - peer_id: pid, in_progress_record_queries: HashMap::default(), in_progress_put_record_queries: HashMap::default(), outstanding_dht_query_keys: HashSet::default(), @@ -133,7 +134,6 @@ impl DHTBehaviour { backoff: ExponentialBackoff::new(2, Duration::from_secs(1)), }, in_progress_get_closest_peers: HashMap::default(), - replication_factor, retry_tx: None, bootstrap_tx: None, phantom: PhantomData, @@ -145,7 +145,7 @@ impl DHTBehaviour { &mut self, kadem: &mut KademliaBehaviour>>, ) { - let mut err = format!("KBUCKETS: PID: {:?}, ", self.peer_id); + let mut err = "KBUCKETS: ".to_string(); let v = kadem.kbuckets().collect::>(); for i in v { for j in i.iter() { @@ -159,11 +159,6 @@ impl DHTBehaviour { error!("{:?}", err); } - /// Get the replication factor for queries - #[must_use] - pub fn replication_factor(&self) -> NonZeroUsize { - self.replication_factor - } /// Publish a key/value to the kv store. /// Once replicated upon all nodes, the caller is notified over /// `chan` @@ -381,10 +376,7 @@ impl DHTBehaviour { query.progress = DHTProgress::NotStarted; query.backoff.start_next(false); - warn!( - "Put DHT: error performing put: {:?}. Retrying on pid {:?}.", - e, self.peer_id - ); + warn!("Put DHT: error performing put: {:?}. Retrying.", e); // push back onto the queue self.retry_put(query); } diff --git a/crates/libp2p-networking/src/network/def.rs b/crates/libp2p-networking/src/network/def.rs index a52fdae36c..4ed4ded3cd 100644 --- a/crates/libp2p-networking/src/network/def.rs +++ b/crates/libp2p-networking/src/network/def.rs @@ -15,7 +15,7 @@ use libp2p::{ }; use libp2p_identity::PeerId; use libp2p_swarm_derive::NetworkBehaviour; -use tracing::{debug, error}; +use tracing::error; use super::{ behaviours::dht::store::{file_backed::FileBackedStore, validated::ValidatedStore}, @@ -27,32 +27,27 @@ use super::{ /// - direct messaging /// - p2p broadcast /// - connection management -#[derive(NetworkBehaviour, derive_more::Debug)] +#[derive(NetworkBehaviour)] #[behaviour(to_swarm = "NetworkEventInternal")] pub struct NetworkDef { /// purpose: broadcasting messages to many peers /// NOTE gossipsub works ONLY for sharing messages right now /// in the future it may be able to do peer discovery and routing /// - #[debug(skip)] gossipsub: GossipBehaviour, /// The DHT store. We use a `FileBackedStore` to occasionally save the DHT to /// a file on disk and a `ValidatedStore` to validate the records stored. - #[debug(skip)] pub dht: libp2p::kad::Behaviour>>, /// purpose: identifying the addresses from an outside POV - #[debug(skip)] identify: IdentifyBehaviour, /// purpose: directly messaging peer - #[debug(skip)] pub direct_message: cbor::Behaviour, Vec>, /// Auto NAT behaviour to determine if we are publicly reachable and /// by which address - #[debug(skip)] pub autonat: libp2p::autonat::Behaviour, } diff --git a/crates/libp2p-networking/src/network/mod.rs b/crates/libp2p-networking/src/network/mod.rs index 21a2811bb8..b8fdd2ffdd 100644 --- a/crates/libp2p-networking/src/network/mod.rs +++ b/crates/libp2p-networking/src/network/mod.rs @@ -9,7 +9,7 @@ pub mod behaviours; /// defines the swarm and network definition (internal) mod def; /// functionality of a libp2p network node -mod node; +pub mod node; /// Alternative Libp2p transport implementations pub mod transport; @@ -21,7 +21,6 @@ use std::{collections::HashSet, fmt::Debug}; use futures::channel::oneshot::Sender; use hotshot_types::traits::{network::NetworkError, node_implementation::NodeType}; use libp2p::{ - build_multiaddr, core::{muxing::StreamMuxerBox, transport::Boxed}, dns::tokio::Transport as DnsTransport, gossipsub::Event as GossipEvent, @@ -29,7 +28,7 @@ use libp2p::{ identity::Keypair, quic, request_response::ResponseChannel, - Multiaddr, Transport, + Transport, }; use libp2p_identity::PeerId; use quic::tokio::Transport as QuicTransport; @@ -39,9 +38,8 @@ use transport::StakeTableAuthentication; pub use self::{ def::NetworkDef, node::{ - spawn_network_node, GossipConfig, NetworkNode, NetworkNodeConfig, NetworkNodeConfigBuilder, - NetworkNodeConfigBuilderError, NetworkNodeHandle, NetworkNodeReceiver, - RequestResponseConfig, DEFAULT_REPLICATION_FACTOR, + spawn_network_node, GossipConfig, NetworkNode, NetworkNodeHandle, NetworkNodeReceiver, + RequestResponseConfig, }, }; @@ -71,8 +69,6 @@ pub enum ClientRequest { DirectResponse(ResponseChannel>, Vec), /// prune a peer Prune(PeerId), - /// add vec of known peers or addresses - AddKnownPeers(Vec<(PeerId, Multiaddr)>), /// Ignore peers. Only here for debugging purposes. /// Allows us to have nodes that are never pruned IgnorePeers(Vec), @@ -138,13 +134,6 @@ pub enum NetworkEventInternal { AutonatEvent(libp2p::autonat::Event), } -/// Bind all interfaces on port `port` -/// NOTE we may want something more general in the fture. -#[must_use] -pub fn gen_multiaddr(port: u16) -> Multiaddr { - build_multiaddr!(Ip4([0, 0, 0, 0]), Udp(port), QuicV1) -} - /// `BoxedTransport` is a type alias for a boxed tuple containing a `PeerId` and a `StreamMuxerBox`. /// /// This type is used to represent a transport in the libp2p network framework. The `PeerId` is a unique identifier for each peer in the network, and the `StreamMuxerBox` is a type of multiplexer that can handle multiple substreams over a single connection. @@ -159,7 +148,7 @@ type BoxedTransport = Boxed<(PeerId, StreamMuxerBox)>; #[instrument(skip(identity))] pub async fn gen_transport( identity: Keypair, - stake_table: Option, + quorum_membership: Option, auth_message: Option>, ) -> Result { // Create the initial `Quic` transport @@ -171,7 +160,7 @@ pub async fn gen_transport( // Require authentication against the stake table let transport: StakeTableAuthentication<_, T, _> = - StakeTableAuthentication::new(transport, stake_table, auth_message); + StakeTableAuthentication::new(transport, quorum_membership, auth_message); // Support DNS resolution let transport = { diff --git a/crates/libp2p-networking/src/network/node.rs b/crates/libp2p-networking/src/network/node.rs index 1ea25a36d0..ac223e2a51 100644 --- a/crates/libp2p-networking/src/network/node.rs +++ b/crates/libp2p-networking/src/network/node.rs @@ -5,23 +5,22 @@ // along with the HotShot repository. If not, see . /// configuration for the libp2p network (e.g. how it should be built) -mod config; +pub mod config; /// libp2p network handle /// allows for control over the libp2p network mod handle; use std::{ - collections::{HashMap, HashSet}, - iter, + collections::HashSet, num::{NonZeroU32, NonZeroUsize}, - time::Duration, + time::{Duration, Instant}, }; +use anyhow::{anyhow, Context}; +use config::{KademliaConfig, Libp2pConfig}; use futures::{channel::mpsc, SinkExt, StreamExt}; -use hotshot_types::{ - constants::KAD_DEFAULT_REPUB_INTERVAL_SEC, traits::node_implementation::NodeType, -}; +use hotshot_types::traits::node_implementation::NodeType; use libp2p::{ autonat, core::transport::ListenerId, @@ -33,8 +32,8 @@ use libp2p::{ Behaviour as IdentifyBehaviour, Config as IdentifyConfig, Event as IdentifyEvent, Info as IdentifyInfo, }, - identity::Keypair, kad::{store::MemoryStore, Behaviour, Config, Mode, Record}, + multiaddr::Protocol, request_response::{ Behaviour as RequestResponse, Config as Libp2pRequestResponseConfig, ProtocolSupport, }, @@ -42,18 +41,15 @@ use libp2p::{ Multiaddr, StreamProtocol, Swarm, SwarmBuilder, }; use libp2p_identity::PeerId; -use rand::{prelude::SliceRandom, thread_rng}; use tokio::{ select, spawn, sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, + time::timeout, }; -use tracing::{debug, error, info, info_span, instrument, warn, Instrument}; +use tracing::{debug, error, info, info_span, instrument, trace, warn, Instrument}; pub use self::{ - config::{ - GossipConfig, NetworkNodeConfig, NetworkNodeConfigBuilder, NetworkNodeConfigBuilderError, - RequestResponseConfig, DEFAULT_REPLICATION_FACTOR, - }, + config::{GossipConfig, RequestResponseConfig}, handle::{spawn_network_node, NetworkNodeHandle, NetworkNodeReceiver}, }; use super::{ @@ -81,21 +77,20 @@ pub const ESTABLISHED_LIMIT: NonZeroU32 = pub const ESTABLISHED_LIMIT_UNWR: u32 = 10; /// Network definition -#[derive(derive_more::Debug)] pub struct NetworkNode { - /// peer id of network node - peer_id: PeerId, - /// the swarm of networkbehaviours - #[debug(skip)] + /// The swarm of network behaviours swarm: Swarm>, - /// the listener id we are listening on, if it exists + /// The listener id we are listening on, if it is initialized listener_id: Option, - /// Handler for direct messages - direct_message_state: DMBehaviour, - /// Handler for DHT Events + /// The handler for direct messages + direct_message_handler: DMBehaviour, + /// The handler for DHT events dht_handler: DHTBehaviour, - /// Channel to resend requests, set to Some when we call `spawn_listeners` + /// Channel to resend requests (set when we call `spawn_listeners`) resend_tx: Option>, + + /// The Kademlia config + kademlia_config: KademliaConfig, } impl NetworkNode { @@ -109,69 +104,61 @@ impl NetworkNode { self.swarm.connected_peers().copied().collect() } - /// starts the swarm listening on `listen_addr` - /// and optionally dials into peer `known_peer` - /// returns the address the swarm is listening upon + /// Bind the swarm to a given address #[instrument(skip(self))] - pub async fn start_listen( - &mut self, - listen_addr: Multiaddr, - ) -> Result { - self.listener_id = Some(self.swarm.listen_on(listen_addr).map_err(|err| { - NetworkError::ListenError(format!("failed to listen for Libp2p: {err}")) - })?); - let addr = loop { - if let Some(SwarmEvent::NewListenAddr { address, .. }) = self.swarm.next().await { - break address; + pub async fn bind_to(&mut self, bind_address: &Multiaddr) -> anyhow::Result<()> { + // Debug log the address we are binding to + debug!("Libp2p binding to {:?}", bind_address); + + // Start listening on the given address + self.listener_id = Some( + self.swarm + .listen_on(bind_address.clone()) + .with_context(|| "failed to bind to address")?, + ); + + // Wait for the listener to be bound with a 60s timeout + let start_time = Instant::now(); + loop { + match timeout(Duration::from_secs(60), self.swarm.next()).await { + // If we successfully get a NewListenAddr event, break + Ok(Some(SwarmEvent::NewListenAddr { .. })) => break, + // If we timeout, return an error + Err(_) => { + return Err(anyhow!("Timed out binding to address")); + } + // If we get any other event, continue waiting + _ => {} } - }; - info!("Libp2p listening on {:?}", addr); - Ok(addr) - } - /// initialize the DHT with known peers - /// add the peers to kademlia and then - /// the `spawn_listeners` function - /// will start connecting to peers - #[instrument(skip(self))] - pub fn add_known_peers(&mut self, known_peers: &[(PeerId, Multiaddr)]) { - debug!("Adding {} known peers", known_peers.len()); - let behaviour = self.swarm.behaviour_mut(); - let mut bs_nodes = HashMap::>::new(); - let mut shuffled = known_peers.iter().collect::>(); - shuffled.shuffle(&mut thread_rng()); - for (peer_id, addr) in shuffled { - if *peer_id != self.peer_id { - behaviour.dht.add_address(peer_id, addr.clone()); - behaviour.autonat.add_server(*peer_id, Some(addr.clone())); - bs_nodes.insert(*peer_id, iter::once(addr.clone()).collect()); + // If we've been waiting for more than 60 seconds, return an error + if start_time.elapsed() > Duration::from_secs(60) { + return Err(anyhow!("Timed out binding to address")); } } + + // Log the address we are listening on + info!("Libp2p listening on {:?}", bind_address); + + Ok(()) } - /// Creates a new `Network` with the given settings. + /// Creates a new `NetworkNode` with the given Libp2p configuration /// - /// Currently: - /// * Generates a random key pair and associated [`PeerId`] - /// * Launches a hopefully production ready transport: - /// QUIC v1 (RFC 9000) + DNS - /// * Generates a connection to the "broadcast" topic - /// * Creates a swarm to manage peers and events - #[instrument] - pub async fn new(config: NetworkNodeConfig) -> Result { - // Generate a random `KeyPair` if one is not specified - let keypair = config - .keypair - .clone() - .unwrap_or_else(Keypair::generate_ed25519); - + /// # Errors + /// If the network node cannot be created + /// + /// # Panics + /// If `5 == 0` + #[allow(clippy::too_many_lines)] + pub async fn new(config: &Libp2pConfig) -> anyhow::Result { // Get the `PeerId` from the `KeyPair` - let peer_id = PeerId::from(keypair.public()); + let peer_id = PeerId::from(config.keypair.public()); // Generate the transport from the keypair, stake table, and auth message let transport: BoxedTransport = gen_transport::( - keypair.clone(), - config.stake_table.clone(), + config.keypair.clone(), + config.quorum_membership.clone(), config.auth_message.clone(), ) .await?; @@ -215,62 +202,41 @@ impl NetworkNode { NetworkError::ConfigError(format!("error building gossipsub config: {err:?}")) })?; - // - Build a gossipsub network behavior + // Create the Gossipsub behaviour let gossipsub: Gossipsub = Gossipsub::new( - MessageAuthenticity::Signed(keypair.clone()), + MessageAuthenticity::Signed(config.keypair.clone()), gossipsub_config, ) .map_err(|err| { NetworkError::ConfigError(format!("error building gossipsub behaviour: {err:?}")) })?; - // Build a identify network behavior needed for own - // node connection information - // E.g. this will answer the question: how are other nodes - // seeing the peer from behind a NAT + // Configure and create the Identify behaviour let identify_cfg = - IdentifyConfig::new("HotShot/identify/1.0".to_string(), keypair.public()); + IdentifyConfig::new("HotShot/identify/1.0".to_string(), config.keypair.public()); let identify = IdentifyBehaviour::new(identify_cfg); - // - Build DHT needed for peer discovery + // Configure the Kademlia behaviour let mut kconfig = Config::default(); - // 8 hours by default - let record_republication_interval = config - .republication_interval - .unwrap_or(Duration::from_secs(KAD_DEFAULT_REPUB_INTERVAL_SEC)); - let ttl = Some(config.ttl.unwrap_or(16 * record_republication_interval)); kconfig .set_parallelism(NonZeroUsize::new(5).unwrap()) - .set_provider_publication_interval(Some(record_republication_interval)) - .set_publication_interval(Some(record_republication_interval)) - .set_record_ttl(ttl); - - // allowing panic here because something is very wrong if this fales - #[allow(clippy::panic)] - if let Some(factor) = config.replication_factor { - kconfig.set_replication_factor(factor); - } else { - panic!("Replication factor not set"); - } + .set_provider_publication_interval(config.kademlia_config.publication_interval) + .set_publication_interval(config.kademlia_config.publication_interval) + .set_record_ttl(config.kademlia_config.record_ttl); - // Extract the DHT file path from the config, defaulting to `libp2p_dht.json` - let dht_file_path = config - .dht_file_path - .clone() - .unwrap_or_else(|| "libp2p_dht.bin".into()); - - // Create the DHT behaviour + // Create the Kademlia behaviour let mut kadem = Behaviour::with_config( peer_id, FileBackedStore::new( ValidatedStore::new(MemoryStore::new(peer_id)), - dht_file_path, + config.kademlia_config.file_path.clone(), 10, ), kconfig, ); kadem.set_mode(Some(Mode::Server)); + // Use the default request response config let rrconfig = Libp2pRequestResponseConfig::default(); // Create a new `cbor` codec with the given request and response sizes @@ -279,6 +245,7 @@ impl NetworkNode { config.request_response_config.response_size_maximum, ); + // Create the direct message behaviour with our configured codec let direct_message: super::cbor::Behaviour, Vec> = RequestResponse::with_codec( cbor, @@ -303,8 +270,8 @@ impl NetworkNode { autonat::Behaviour::new(peer_id, autonat_config), ); - // build swarm - let swarm = SwarmBuilder::with_existing_identity(keypair.clone()); + // Build the swarm + let swarm = SwarmBuilder::with_existing_identity(config.keypair.clone()); let swarm = swarm.with_tokio(); swarm @@ -314,24 +281,41 @@ impl NetworkNode { .unwrap() .build() }; - for (peer, addr) in &config.to_connect_addrs { - if peer != swarm.local_peer_id() { - swarm.behaviour_mut().add_address(peer, addr.clone()); + + // Add the known peers to Libp2p + let mut num_known_peers_added = 0; + for mut known_peer in config.known_peers.clone() { + // Lob off the last protocol from the multiaddr + if let Some(protocol) = known_peer.pop() { + // Make sure it is the P2P protocol (which includes the peer id) + if let Protocol::P2p(peer_id) = protocol { + // Add the address to Libp2p behaviors + swarm.behaviour_mut().add_address(&peer_id, known_peer); + num_known_peers_added += 1; + } else { + warn!("Known peer {:?} has no P2P address", known_peer); + } + } else { + warn!("Known peer {:?} has no address", known_peer); } } + // If we hadn't found any suitable known peers, return an error + if num_known_peers_added == 0 { + return Err(anyhow!("No suitable known peers known")); + } + + // Create our own DHT handler + let dht_handler = DHTBehaviour::new(); + + // Create and return the new network node Ok(Self { - peer_id, swarm, listener_id: None, - direct_message_state: DMBehaviour::default(), - dht_handler: DHTBehaviour::new( - peer_id, - config - .replication_factor - .unwrap_or(NonZeroUsize::new(4).unwrap()), - ), + direct_message_handler: DMBehaviour::default(), + dht_handler, resend_tx: None, + kademlia_config: config.kademlia_config.clone(), }) } @@ -344,7 +328,7 @@ impl NetworkNode { match self.swarm.behaviour_mut().dht.put_record( record, libp2p::kad::Quorum::N( - NonZeroUsize::try_from(self.dht_handler.replication_factor().get() / 2) + NonZeroUsize::try_from(self.kademlia_config.replication_factor / 2) .expect("replication factor should be bigger than 0"), ), ) { @@ -352,7 +336,7 @@ impl NetworkNode { // failed try again later query.progress = DHTProgress::NotStarted; query.backoff.start_next(false); - error!("Error publishing to DHT: {e:?} for peer {:?}", self.peer_id); + warn!("Error publishing to DHT: {e:?}"); } Ok(qid) => { debug!("Published record to DHT with qid {:?}", qid); @@ -475,14 +459,11 @@ impl NetworkNode { backoff: ExponentialBackoff::default(), retry_count, }; - self.direct_message_state.add_direct_request(req, id); + self.direct_message_handler.add_direct_request(req, id); } ClientRequest::DirectResponse(chan, msg) => { behaviour.add_direct_response(chan, msg); } - ClientRequest::AddKnownPeers(peers) => { - self.add_known_peers(&peers); - } ClientRequest::Prune(pid) => { if self.swarm.disconnect_peer_id(pid).is_err() { warn!("Could not disconnect from {:?}", pid); @@ -634,7 +615,7 @@ impl NetworkNode { } }, NetworkEventInternal::DMEvent(e) => self - .direct_message_state + .direct_message_handler .handle_dm_event(e, self.resend_tx.clone()), NetworkEventInternal::AutonatEvent(e) => { match e { @@ -711,8 +692,10 @@ impl NetworkNode { /// Spawn a task to listen for requests on the returned channel /// as well as any events produced by libp2p - #[instrument] - pub async fn spawn_listeners( + /// + /// # Errors + /// If the listeners cannot be spawned + pub fn spawn_listeners( mut self, ) -> Result< ( @@ -733,14 +716,13 @@ impl NetworkNode { loop { select! { event = self.swarm.next() => { - debug!("peerid {:?}\t\thandling maybe event {:?}", self.peer_id, event); if let Some(event) = event { - debug!("peerid {:?}\t\thandling event {:?}", self.peer_id, event); + trace!("Libp2p handling event {:?}", event); self.handle_swarm_events(event, &r_input).await?; } }, msg = s_output.recv() => { - debug!("peerid {:?}\t\thandling msg {:?}", self.peer_id, msg); + trace!("Libp2p handling client request {:?}", msg); let shutdown = self.handle_client_requests(msg).await?; if shutdown { let _ = bootstrap_tx.send(InputEvent::ShutdownBootstrap).await; @@ -755,9 +737,4 @@ impl NetworkNode { ); Ok((s_input, r_output)) } - - /// Get a reference to the network node's peer id. - pub fn peer_id(&self) -> PeerId { - self.peer_id - } } diff --git a/crates/libp2p-networking/src/network/node/config.rs b/crates/libp2p-networking/src/network/node/config.rs index 1f5422e321..a2f0f1a300 100644 --- a/crates/libp2p-networking/src/network/node/config.rs +++ b/crates/libp2p-networking/src/network/node/config.rs @@ -4,66 +4,15 @@ // You should have received a copy of the MIT License // along with the HotShot repository. If not, see . -use std::{collections::HashSet, num::NonZeroUsize, time::Duration}; +use std::time::Duration; use hotshot_types::traits::node_implementation::NodeType; -use libp2p::{identity::Keypair, Multiaddr}; -use libp2p_identity::PeerId; +use libp2p::Multiaddr; +use libp2p_identity::Keypair; -use super::MAX_GOSSIP_MSG_SIZE; - -/// The default Kademlia replication factor -pub const DEFAULT_REPLICATION_FACTOR: Option = NonZeroUsize::new(10); - -/// describe the configuration of the network -#[derive(Clone, Default, derive_builder::Builder, derive_more::Debug)] -pub struct NetworkNodeConfig { - /// The keypair for the node - #[builder(setter(into, strip_option), default)] - #[debug(skip)] - pub keypair: Option, - /// The address to bind to - #[builder(default)] - pub bind_address: Option, - /// Replication factor for entries in the DHT - #[builder(setter(into, strip_option), default = "DEFAULT_REPLICATION_FACTOR")] - pub replication_factor: Option, - - #[builder(default)] - /// Configuration for `GossipSub` - pub gossip_config: GossipConfig, - - #[builder(default)] - /// Configuration for `RequestResponse` - pub request_response_config: RequestResponseConfig, +use crate::network::behaviours::dht::record::RecordValue; - /// list of addresses to connect to at initialization - pub to_connect_addrs: HashSet<(PeerId, Multiaddr)>, - /// republication interval in DHT, must be much less than `ttl` - #[builder(default)] - pub republication_interval: Option, - /// expiratiry for records in DHT - #[builder(default)] - pub ttl: Option, - - /// The stake table. Used for authenticating other nodes. If not supplied - /// we will not check other nodes against the stake table - #[builder(default)] - pub stake_table: Option, - - /// The path to the file to save the DHT to - #[builder(default)] - pub dht_file_path: Option, - - /// The signed authentication message sent to the remote peer - /// If not supplied we will not send an authentication message during the handshake - #[builder(default)] - pub auth_message: Option>, - - #[builder(default)] - /// The timeout for DHT lookups. - pub dht_timeout: Option, -} +use super::MAX_GOSSIP_MSG_SIZE; /// Configuration for Libp2p's Gossipsub #[derive(Clone, Debug)] @@ -177,3 +126,41 @@ impl Default for RequestResponseConfig { } } } + +/// Configuration for Libp2p's Kademlia +#[derive(Clone, Debug)] +pub struct KademliaConfig { + /// The replication factor + pub replication_factor: usize, + /// The record ttl + pub record_ttl: Option, + /// The publication interval + pub publication_interval: Option, + /// The file path for the [file-backed] record store + pub file_path: String, + /// The lookup record value (a signed peer ID so it can be verified by any node) + pub lookup_record_value: RecordValue, +} + +/// Configuration for Libp2p +#[derive(Clone)] +pub struct Libp2pConfig { + /// The Libp2p keypair + pub keypair: Keypair, + /// The address to bind Libp2p to + pub bind_address: Multiaddr, + /// Addresses of known peers to add to Libp2p on startup + pub known_peers: Vec, + + /// The quorum membership + pub quorum_membership: Option, + /// The (signed) authentication message + pub auth_message: Option>, + + /// The gossip config + pub gossip_config: GossipConfig, + /// The request response config + pub request_response_config: RequestResponseConfig, + /// The kademlia config + pub kademlia_config: KademliaConfig, +} diff --git a/crates/libp2p-networking/src/network/node/handle.rs b/crates/libp2p-networking/src/network/node/handle.rs index b7a6832286..18fd00dd0d 100644 --- a/crates/libp2p-networking/src/network/node/handle.rs +++ b/crates/libp2p-networking/src/network/node/handle.rs @@ -4,50 +4,46 @@ // You should have received a copy of the MIT License // along with the HotShot repository. If not, see . -use std::{collections::HashSet, fmt::Debug, time::Duration}; +use std::{collections::HashSet, fmt::Debug, marker::PhantomData, time::Duration}; -use hotshot_types::traits::{network::NetworkError, node_implementation::NodeType}; -use libp2p::{request_response::ResponseChannel, Multiaddr}; +use anyhow::Context; +use hotshot_types::traits::{ + network::NetworkError, node_implementation::NodeType, signature_key::SignatureKey, +}; +use libp2p::request_response::ResponseChannel; use libp2p_identity::PeerId; use tokio::{ sync::mpsc::{Receiver, UnboundedReceiver, UnboundedSender}, time::{sleep, timeout}, }; -use tracing::{debug, info, instrument}; +use tracing::{info, instrument}; use crate::network::{ behaviours::dht::record::{Namespace, RecordKey, RecordValue}, - gen_multiaddr, ClientRequest, NetworkEvent, NetworkNode, NetworkNodeConfig, + ClientRequest, NetworkEvent, NetworkNode, }; +use super::config::Libp2pConfig; + /// A handle containing: /// - A reference to the state /// - Controls for the swarm #[derive(Debug, Clone)] -pub struct NetworkNodeHandle { - /// network configuration - network_config: NetworkNodeConfig, - - /// send an action to the networkbehaviour - send_network: UnboundedSender, - - /// the local address we're listening on - listen_addr: Multiaddr, +pub struct NetworkNodeHandle { + /// For sending requests to the network behaviour + request_sender: UnboundedSender, - /// the peer id of the networkbehaviour - peer_id: PeerId, - - /// human readable id - id: usize, + /// Phantom data + pd: PhantomData, } /// internal network node receiver #[derive(Debug)] pub struct NetworkNodeReceiver { - /// the receiver - receiver: UnboundedReceiver, + /// The receiver for requests from the application + request_receiver: UnboundedReceiver, - ///kill switch + /// The kill switch for the receiver recv_kill: Option>, } @@ -56,7 +52,7 @@ impl NetworkNodeReceiver { /// # Errors /// Errors if the receiver channel is closed pub async fn recv(&mut self) -> Result { - self.receiver + self.request_receiver .recv() .await .ok_or(NetworkError::ChannelReceiveError( @@ -78,42 +74,40 @@ impl NetworkNodeReceiver { /// # Errors /// Errors if spawning the task fails pub async fn spawn_network_node( - config: NetworkNodeConfig, - id: usize, -) -> Result<(NetworkNodeReceiver, NetworkNodeHandle), NetworkError> { - let mut network = NetworkNode::new(config.clone()) + config: Libp2pConfig, +) -> anyhow::Result<(NetworkNodeReceiver, NetworkNodeHandle)> { + // Create the network node (what we send requests to and from in the application) + let mut network = NetworkNode::new(&config) + .await + .with_context(|| "failed to create network node")?; + + // Bind the swarm to the given address + network + .bind_to(&config.bind_address) .await - .map_err(|e| NetworkError::ConfigError(format!("failed to create network node: {e}")))?; - // randomly assigned port - let listen_addr = config - .bind_address - .clone() - .unwrap_or_else(|| gen_multiaddr(0)); - let peer_id = network.peer_id(); - let listen_addr = network.start_listen(listen_addr).await.map_err(|e| { - NetworkError::ListenError(format!("failed to start listening on Libp2p: {e}")) - })?; - // pin here to force the future onto the heap since it can be large - // in the case of flume - let (send_chan, recv_chan) = Box::pin(network.spawn_listeners()).await.map_err(|err| { - NetworkError::ListenError(format!("failed to spawn listeners for Libp2p: {err}")) - })?; + .with_context(|| format!("failed to bind to {:?}", config.bind_address))?; + + // Spawn the listeners and get the request sender and receiver + let (request_sender, request_receiver) = network + .spawn_listeners() + .with_context(|| "failed to spawn listeners")?; + + // Create the receiver let receiver = NetworkNodeReceiver { - receiver: recv_chan, + request_receiver, recv_kill: None, }; - let handle = NetworkNodeHandle:: { - network_config: config, - send_network: send_chan, - listen_addr, - peer_id, - id, + // Create the handle (what the application uses to interact with the network) + let handle = NetworkNodeHandle { + request_sender, + pd: PhantomData, }; + Ok((receiver, handle)) } -impl NetworkNodeHandle { +impl NetworkNodeHandle { /// Cleanly shuts down a swarm node /// This is done by sending a message to /// the swarm itself to spin down @@ -131,12 +125,6 @@ impl NetworkNodeHandle { self.send_request(req) } - /// Get a reference to the network node handle's listen addr. - #[must_use] - pub fn listen_addr(&self) -> Multiaddr { - self.listen_addr.clone() - } - /// Print out the routing table used by kademlia /// NOTE: only for debugging purposes currently /// # Errors @@ -152,11 +140,7 @@ impl NetworkNodeHandle { /// /// # Errors /// If the channel closes before the result can be sent back - pub async fn wait_to_connect( - &self, - num_required_peers: usize, - node_id: usize, - ) -> Result<(), NetworkError> { + pub async fn wait_to_connect(&self, num_required_peers: usize) -> Result<(), NetworkError> { // Wait for the required number of peers to connect loop { // Get the number of currently connected peers @@ -167,8 +151,8 @@ impl NetworkNodeHandle { // Log the number of connected peers info!( - "Node {} connected to {}/{} peers", - node_id, num_connected, num_required_peers + "Libp2p connected to {}/{} peers", + num_connected, num_required_peers ); // Sleep for a second before checking again @@ -215,7 +199,7 @@ impl NetworkNodeHandle { pub async fn put_record( &self, key: RecordKey, - value: RecordValue, + value: RecordValue, ) -> Result<(), NetworkError> { // Serialize the key let key = key.to_bytes(); @@ -261,7 +245,7 @@ impl NetworkNodeHandle { let result = r.await.map_err(|_| NetworkError::RequestCancelled)?; // Deserialize the record's value - let record: RecordValue = bincode::deserialize(&result) + let record: RecordValue = bincode::deserialize(&result) .map_err(|e| NetworkError::FailedToDeserialize(e.to_string()))?; Ok(record.value().to_vec()) @@ -290,7 +274,7 @@ impl NetworkNodeHandle { pub async fn put_record_timeout( &self, key: RecordKey, - value: RecordValue, + value: RecordValue, timeout_duration: Duration, ) -> Result<(), NetworkError> { timeout(timeout_duration, self.put_record(key, value)) @@ -396,24 +380,12 @@ impl NetworkNodeHandle { self.send_request(req) } - /// Tell libp2p about known network nodes - /// # Errors - /// - Will return [`NetworkError::ChannelSendError`] when underlying `NetworkNode` has been killed - pub fn add_known_peers( - &self, - known_peers: Vec<(PeerId, Multiaddr)>, - ) -> Result<(), NetworkError> { - debug!("Adding {} known peers", known_peers.len()); - let req = ClientRequest::AddKnownPeers(known_peers); - self.send_request(req) - } - /// Send a client request to the network /// /// # Errors /// - Will return [`NetworkError::ChannelSendError`] when underlying `NetworkNode` has been killed fn send_request(&self, req: ClientRequest) -> Result<(), NetworkError> { - self.send_network + self.request_sender .send(req) .map_err(|err| NetworkError::ChannelSendError(err.to_string())) } @@ -445,22 +417,4 @@ impl NetworkNodeHandle { self.send_request(req)?; Ok(r.await.unwrap()) } - - /// Get a reference to the network node handle's id. - #[must_use] - pub fn id(&self) -> usize { - self.id - } - - /// Get a reference to the network node handle's peer id. - #[must_use] - pub fn peer_id(&self) -> PeerId { - self.peer_id - } - - /// Return a reference to the network config - #[must_use] - pub fn config(&self) -> &NetworkNodeConfig { - &self.network_config - } } diff --git a/crates/orchestrator/Cargo.toml b/crates/orchestrator/Cargo.toml deleted file mode 100644 index a80dcba85a..0000000000 --- a/crates/orchestrator/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "hotshot-orchestrator" -version = { workspace = true } -edition = { workspace = true } - -[dependencies] -anyhow = { workspace = true } -async-lock = { workspace = true } -blake3 = { workspace = true } -clap = { workspace = true } -csv = "1" -futures = { workspace = true } -hotshot-types = { path = "../types" } -libp2p-identity = { workspace = true } -multiaddr = { workspace = true } -serde = { workspace = true } -surf-disco = { workspace = true } -tide-disco = { workspace = true } -tokio = { workspace = true } -toml = { workspace = true } -tracing = { workspace = true } -vbs = { workspace = true } - -[lints] -workspace = true diff --git a/crates/orchestrator/README.md b/crates/orchestrator/README.md deleted file mode 100644 index 1bf5ade9c2..0000000000 --- a/crates/orchestrator/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Orchestrator - -This crate implements an orchestrator that coordinates starting the network with a particular configuration. It is useful for testing and benchmarking. Like the web server, the orchestrator is built using [Tide Disco](https://github.com/EspressoSystems/tide-disco). - -To run the orchestrator: `just example orchestrator http://0.0.0.0:3333 ./crates/orchestrator/run-config.toml` \ No newline at end of file diff --git a/crates/orchestrator/api.toml b/crates/orchestrator/api.toml deleted file mode 100644 index 25fcb2b7c2..0000000000 --- a/crates/orchestrator/api.toml +++ /dev/null @@ -1,104 +0,0 @@ -[meta] -NAME = "orchestrator" -DESCRIPTION = "Orchestrator for HotShot" -FORMAT_VERSION = "0.1.0" - -# POST node's identity -[route.post_identity] -PATH = ["identity"] -METHOD = "POST" -DOC = """ -POST a node's identity (IP address) to the orchestrator. Returns the node's node_index. -""" - -# POST retrieve the network configuration -[route.post_getconfig] -PATH = ["config/:node_index"] -METHOD = "POST" -":node_index" = "Integer" -DOC = """ -Get networking configuration needed for nodes to initialize HotShot and themselves. See `config.rs` for more information. -This must be a POST request so we can update the OrchestratorState in the server accordingly. Must use the node_index previously -received from the 'identity' endpoint -""" - -# POST the latest temporary node index only for generating validator's key pair -[route.get_tmp_node_index] -PATH = ["get_tmp_node_index"] -METHOD = "POST" -DOC = """ -Get the latest temporary node index only for generating validator's key pair for testing in hotshot, later the generated key pairs might be bound with other node_index. -""" - -# POST the node's node index for pubkey and is_da collection -[route.post_pubkey] -PATH = ["pubkey/:is_da"] -METHOD = "POST" -":is_da" = "Boolean" -DOC = """ -Post a node's node_index so that its public key could be posted and collected by the orchestrator. -Supply whether or not we are DA. -""" - -# GET whether or not the config with all peers' public keys / configs are ready -[route.peer_pubconfig_ready] -PATH = ["peer_pub_ready"] -DOC = """ -Get whether the node can collect the final config which includs all peer's public config/info like public keys, returns a boolean. -""" - -# POST the updated config with all peers' public keys / configs -[route.post_config_after_peer_collected] -PATH = ["post_config_after_peer_collected"] -METHOD = "POST" -DOC = """ -Get the updated config with all peers' public keys / configs, returns a NetworkConfig. -""" - -# POST whether the node is ready to begin the run -# TODO ED Use the node index parameter -[route.post_ready] -PATH = ["ready"] -METHOD = "POST" -":node_index" = "Integer" -DOC = """ -Post whether the node with node_index is ready to start the run -""" - -# GET whether or not to start the run -[route.get_start] -PATH = ["start"] -DOC = """ -Get whether the node should start the run, returns a boolean -""" - -# POST the run results -[route.post_results] -PATH = ["results"] -METHOD = "POST" -DOC = """ -Post run results. -""" - -# POST to manually start the run -[route.post_manual_start] -PATH = ["manual_start"] -METHOD = "POST" -DOC = """ -Post whether the orchestrator should start the run immediately, with the nodes that have already registered. -""" - -# GET builder URLs -[route.get_builders] -PATH = ["builders"] -DOC = """ -Get list of builder URLs -""" - -# POST builder URL -[route.post_builder] -PATH = ["builder"] -METHOD = "POST" -DOC = """ -Register a builder URL to orchestrator's pool of builder URLs -""" diff --git a/crates/orchestrator/run-config.toml b/crates/orchestrator/run-config.toml deleted file mode 100644 index 1e5a8700d7..0000000000 --- a/crates/orchestrator/run-config.toml +++ /dev/null @@ -1,68 +0,0 @@ -rounds = 100 -indexed_da = true -transactions_per_round = 10 -transaction_size = 1000 -node_index = 0 -seed = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -cdn_marshal_address = "127.0.0.1:8999" -public_keys = [ - { stake_table_key = "BLS_VER_KEY~bQszS-QKYvUij2g20VqS8asttGSb95NrTu2PUj0uMh1CBUxNy1FqyPDjZqB29M7ZbjWqj79QkEOWkpga84AmDYUeTuWmy-0P1AdKHD3ehc-dKvei78BDj5USwXPJiDUlCxvYs_9rWYhagaq-5_LXENr78xel17spftNd5MA1Mw5U", state_ver_key = "SCHNORR_VER_KEY~lJqDaVZyM0hWP2Br52IX5FeE-dCAIC-dPX7bL5-qUx-vjbunwe-ENOeZxj6FuOyvDCFzoGeP7yZ0fM995qF-CRE", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~IBRoz_Q1EXvcm1pNZcmVlyYZU8hZ7qmy337ePAjEMhz8Hl2q8vWPFOd3BaLwgRS1UzAPW3z4E-XIgRDGcRBTAMZX9b_0lKYjlyTlNF2EZfNnKmvv-xJ0yurkfjiveeYEsD2l5d8q_rJJbH1iZdXy-yPEbwI0SIvQfwdlcaKw9po4", state_ver_key = "SCHNORR_VER_KEY~tyuplKrHzvjODsjPKMHVFYfoMcgklQsMye-2aSCktBcbW_CIzLOq3wZXRIPBbw3FiV6_QoUXYAlpZ5up0zG_ANY", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~4zQnaCOFJ7m95OjxeNls0QOOwWbz4rfxaL3NwmN2zSdnf8t5Nw_dfmMHq05ee8jCegw6Bn5T8inmrnGGAsQJMMWLv77nd7FJziz2ViAbXg-XGGF7o4HyzELCmypDOIYF3X2UWferFE_n72ZX0iQkUhOvYZZ7cfXToXxRTtb_mwRR", state_ver_key = "SCHNORR_VER_KEY~qQAC373HPv4s0mTTpdmSaynfUXC4SfPCuGD2fbeigSpexFB2ycCeXV9UAjuR86CC9udPhopgMsFLyD29VO2iJSg", stake = 1, da = true}, - - { stake_table_key = "BLS_VER_KEY~rO2PIjyY30HGfapFcloFe3mNDKMIFi6JlOLkH5ZWBSYoRm5fE2-Rm6Lp3EvmAcB5r7KFJ0c1Uor308x78r04EY_sfjcsDCWt7RSJdL4cJoD_4fSTCv_bisO8k98hs_8BtqQt8BHlPeJohpUXvcfnK8suXJETiJ6Er97pfxRbzgAL", state_ver_key = "SCHNORR_VER_KEY~le6RHdTasbBsTcbMqArt0XWFwfIJTY7RbUwaCvdxswL8LpXpO3eb86iyYUr63dtv4GGa5fIJaRH97nCd1lV9H8g", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~r6b-Cwzp-b3czlt0MHmYPJIow5kMsXbrNmZsLSYg9RV49oCCO4WEeCRFR02x9bqLCa_sgNFMrIeNdEa11qNiBAohApYFIvrSa-zP5QGj3xbZaMOCrshxYit6E2TR-XsWvv6gjOrypmugjyTAth-iqQzTboSfmO9DD1-gjJIdCaD7", state_ver_key = "SCHNORR_VER_KEY~LfL6fFJQ8UZWR1Jro6LHtKm_y5-VQZBapO0XhcB8ABAmsVght9B8k7NntrgniffAMD8_OJ6Zjg8XUklhbb42CIw", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~kEUEUJFBtCXl68fM_2roQw856wQlu1ZoDmPn8uu4bQgeZwyb5oz5_kMl-oAJ_OtbYV1serjWE--eXB_qYIpQLZka42-cML6WjCQjNl1hGSejtoBDkExNeUNcweFQBbEsaDiIy3-sgHTrfYpFd1icKeAVihLRn5_RtSU_RUu1TQqR", state_ver_key = "SCHNORR_VER_KEY~qKOggsQMNIIvmiIPM3smiGk40kYGXCVupsmgIrf3RgMmua683F3vUwzWcx0s7mxdzLXJwPAB06LD96cxip7JLJM", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~96hAcdFZxQT8CEHcyV8j2ILJRsXagquENPkc9AwLSx3u6AE_uMupIKGbNJRiM99oFneK2vI5g1u61HidWeuTLRPM2537xAXeaO8e-wJYx4FaPKw_xTcLPrIm0OZT7SsLAMwFuqfMbDdKM71-RyrLwhff5517xXBKEk5Tg9iT9Qrr", state_ver_key = "SCHNORR_VER_KEY~y0nltwKyKSpwO3ki9Czu5asjAt5g1Ya3XmAywcerOSUg__FuZOcYq6tKxMsnsjE7ylpWLZv8R5W4-6WkP0DWI94", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~-pVi7j6TEBeG7ABata4uWWDRM2SrY8wWotWsGnTpIhnOVYJI_lNWyig6VJUuFmBsMS8rLMU7nDxDm8SbObxyA-SLFcr_jCkZqsbx8GcVQrnBAfjNRWuPZP0xcTDMu2IkQqtc3L0OpzbMEgGRGE8Wj09pNqouzl-xhPoYjTmD06Bw", state_ver_key = "SCHNORR_VER_KEY~6rPZ_plXxp8Uoh-E8VPb37csDRLND66zAorA3crYOhf9ARJapk8151RRVXWHe5Q2uF_RmQQmOCAov6tIpJ4yHz0", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~IUPSdnsNUHgNx_74ZhBPrICcDZ9Bp_DAt-6kFz8vSwJES2Vy1Ws8NJ1mxb9XGE1u13sw0FRe8kn5Ib3p2stbEtR_1Qgbuif6aoLrGaSUzy0MvwrO58u9kHZk3rXIuSAN7n4ok3-KKk2CmnBfx7fchFoqT56FXCd1EJ7XRrYj8wTh", state_ver_key = "SCHNORR_VER_KEY~qLqeTM1ZT1ecLEpzHwmlr-GeMUOest-kAm5nKOnKnB-W_TRj4IL77lnmamYvUdXR_ddQp24wQh2IlOIdp5jKEgw", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~PAAQNgOYfj3GiVX7LxSlkXfOCDSnNKZDqPVYQ_jBMxKzOCn0PXbqQ62kKPenWOmCxiCE7X158s-VenBna6MjHJgf61eBAO-3-OyTP5NWVx49RTgHhQf2iMTKk2iqK2gjnjZimBU135YU4lQFtrG-ZgRezwqkC5vy8V-q46fschIG", state_ver_key = "SCHNORR_VER_KEY~APKGX39-mOmApq6jMdIEiuuyddJ_k8xFeIwU1Zs2zShH1rI--eZR180Us5vqNWmK3zAidScvVcW4bAsOMHB3LPg", stake = 1, da = true } -] - -[config] -num_nodes_with_stake = 10 -start_threshold = [8, 10] -staked_da_nodes = 10 -fixed_leader_for_gpuvid = 1 -next_view_timeout = 30000 -num_bootstrap = 5 -epoch_height = 0 - -[random_builder] -txn_in_block = 100 -blocks_per_second = 1 -txn_size = { start = 20, end = 100 } - -[combined_network_config.delay_duration] -secs = 1 -nanos = 0 - -[config.view_sync_timeout] -secs = 2 -nanos = 0 - -[config.data_request_delay] -secs = 0 -nanos = 200_000_000 - -[config.builder_timeout] -secs = 10 -nanos = 0 - -[config.upgrade] -start_proposing_view = 1 -stop_proposing_view = 0 -start_voting_view = 1 -stop_voting_view = 0 -start_proposing_time = 1 -stop_proposing_time = 0 -start_voting_time = 1 -stop_voting_time = 0 diff --git a/crates/orchestrator/src/client.rs b/crates/orchestrator/src/client.rs deleted file mode 100644 index de167ff505..0000000000 --- a/crates/orchestrator/src/client.rs +++ /dev/null @@ -1,522 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -use std::{net::SocketAddr, time::Duration}; - -use clap::Parser; -use futures::{Future, FutureExt}; -use hotshot_types::{ - network::{NetworkConfig, NetworkConfigSource}, - traits::signature_key::SignatureKey, - PeerConfig, ValidatorConfig, -}; -use libp2p_identity::PeerId; -use multiaddr::Multiaddr; -use surf_disco::{error::ClientError, Client}; -use tide_disco::Url; -use tokio::time::sleep; -use tracing::{info, instrument}; -use vbs::BinarySerializer; - -use crate::OrchestratorVersion; - -/// Holds the client connection to the orchestrator -pub struct OrchestratorClient { - /// the client - pub client: surf_disco::Client, -} - -/// Struct describing a benchmark result -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default, PartialEq)] -pub struct BenchResults { - /// Whether it's partial collected results - pub partial_results: String, - /// The average latency of the transactions - pub avg_latency_in_sec: i64, - /// The number of transactions that were latency measured - pub num_latency: i64, - /// The minimum latency of the transactions - pub minimum_latency_in_sec: i64, - /// The maximum latency of the transactions - pub maximum_latency_in_sec: i64, - /// The throughput of the consensus protocol = number of transactions committed per second * transaction size in bytes - pub throughput_bytes_per_sec: u64, - /// The number of transactions committed during benchmarking - pub total_transactions_committed: u64, - /// The size of each transaction in bytes - pub transaction_size_in_bytes: u64, - /// The total time elapsed for benchmarking - pub total_time_elapsed_in_sec: u64, - /// The total number of views during benchmarking - pub total_num_views: usize, - /// The number of failed views during benchmarking - pub failed_num_views: usize, - /// The membership committee type used - pub committee_type: String, -} - -impl BenchResults { - /// printout the results of one example run - pub fn printout(&self) { - println!("====================="); - println!("{0} Benchmark results:", self.partial_results); - println!("Committee type: {}", self.committee_type); - println!( - "Average latency: {} seconds, Minimum latency: {} seconds, Maximum latency: {} seconds", - self.avg_latency_in_sec, self.minimum_latency_in_sec, self.maximum_latency_in_sec - ); - println!("Throughput: {} bytes/sec", self.throughput_bytes_per_sec); - println!( - "Total transactions committed: {}", - self.total_transactions_committed - ); - println!( - "Total number of views: {}, Failed number of views: {}", - self.total_num_views, self.failed_num_views - ); - println!("====================="); - } -} - -/// Struct describing a benchmark result needed for download, also include the config -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default, PartialEq)] -pub struct BenchResultsDownloadConfig { - // Config starting here - /// The commit this benchmark was run on - pub commit_sha: String, - /// Total number of nodes - pub total_nodes: usize, - /// The size of the da committee - pub da_committee_size: usize, - /// The number of fixed_leader_for_gpuvid when we enable the feature [fixed-leader-election] - pub fixed_leader_for_gpuvid: usize, - /// Number of transactions submitted per round - pub transactions_per_round: usize, - /// The size of each transaction in bytes - pub transaction_size: u64, - /// The number of rounds - pub rounds: usize, - - // Results starting here - /// Whether the results are partially collected - /// "One" when the results are collected for one node - /// "Half" when the results are collective for half running nodes if not all nodes terminate successfully - /// "Full" if the results are successfully collected from all nodes - pub partial_results: String, - /// The average latency of the transactions - pub avg_latency_in_sec: i64, - /// The minimum latency of the transactions - pub minimum_latency_in_sec: i64, - /// The maximum latency of the transactions - pub maximum_latency_in_sec: i64, - /// The throughput of the consensus protocol = number of transactions committed per second * transaction size in bytes - pub throughput_bytes_per_sec: u64, - /// The number of transactions committed during benchmarking - pub total_transactions_committed: u64, - /// The total time elapsed for benchmarking - pub total_time_elapsed_in_sec: u64, - /// The total number of views during benchmarking - pub total_num_views: usize, - /// The number of failed views during benchmarking - pub failed_num_views: usize, - /// The membership committee type used - pub committee_type: String, -} - -// VALIDATOR - -#[derive(Parser, Debug, Clone)] -#[command( - name = "Multi-machine consensus", - about = "Simulates consensus among multiple machines" -)] -/// Arguments passed to the validator -pub struct ValidatorArgs { - /// The address the orchestrator runs on - pub url: Url, - /// The optional advertise address to use for Libp2p - pub advertise_address: Option, - /// Optional address to run builder on. Address must be accessible by other nodes - pub builder_address: Option, - /// An optional network config file to save to/load from - /// Allows for rejoining the network on a complete state loss - #[arg(short, long)] - pub network_config_file: Option, -} - -/// arguments to run multiple validators -#[derive(Parser, Debug, Clone)] -pub struct MultiValidatorArgs { - /// Number of validators to run - pub num_nodes: u16, - /// The address the orchestrator runs on - pub url: Url, - /// The optional advertise address to use for Libp2p - pub advertise_address: Option, - /// An optional network config file to save to/load from - /// Allows for rejoining the network on a complete state loss - #[arg(short, long)] - pub network_config_file: Option, -} - -/// Asynchronously retrieves a `NetworkConfig` from an orchestrator. -/// The retrieved one includes correct `node_index` and peer's public config. -/// -/// # Errors -/// If we are unable to get the configuration from the orchestrator -pub async fn get_complete_config( - client: &OrchestratorClient, - mut validator_config: ValidatorConfig, - libp2p_advertise_address: Option, - libp2p_public_key: Option, -) -> anyhow::Result<(NetworkConfig, ValidatorConfig, NetworkConfigSource)> { - // get the configuration from the orchestrator - let run_config: NetworkConfig = client - .post_and_wait_all_public_keys::( - &mut validator_config, - libp2p_advertise_address, - libp2p_public_key, - ) - .await; - - info!( - "Retrieved config; our node index is {}. DA committee member: {}", - run_config.node_index, validator_config.is_da - ); - Ok(( - run_config, - validator_config, - NetworkConfigSource::Orchestrator, - )) -} - -impl ValidatorArgs { - /// Constructs `ValidatorArgs` from `MultiValidatorArgs` and a node index. - /// - /// If `network_config_file` is present in `MultiValidatorArgs`, it appends the node index to it to create a unique file name for each node. - /// - /// # Arguments - /// - /// * `multi_args` - A `MultiValidatorArgs` instance containing the base arguments for the construction. - /// * `node_index` - A `u16` representing the index of the node for which the args are being constructed. - /// - /// # Returns - /// - /// This function returns a new instance of `ValidatorArgs`. - /// - /// # Examples - /// - /// ```ignore - /// // NOTE this is a toy example, - /// // the user will need to construct a multivalidatorargs since `new` does not exist - /// # use hotshot_orchestrator::client::MultiValidatorArgs; - /// let multi_args = MultiValidatorArgs::new(); - /// let node_index = 1; - /// let instance = Self::from_multi_args(multi_args, node_index); - /// ``` - #[must_use] - pub fn from_multi_args(multi_args: MultiValidatorArgs, node_index: u16) -> Self { - Self { - url: multi_args.url, - advertise_address: multi_args.advertise_address, - builder_address: None, - network_config_file: multi_args - .network_config_file - .map(|s| format!("{s}-{node_index}")), - } - } -} - -impl OrchestratorClient { - /// Creates the client that will connect to the orchestrator - #[must_use] - pub fn new(url: Url) -> Self { - let client = surf_disco::Client::::new(url); - // TODO ED: Add healthcheck wait here - OrchestratorClient { client } - } - - /// Get the config from the orchestrator. - /// If the identity is provided, register the identity with the orchestrator. - /// If not, just retrieving the config (for passive observers) - /// - /// # Panics - /// if unable to convert the node index from usize into u64 - /// (only applicable on 32 bit systems) - /// - /// # Errors - /// If we were unable to serialize the Libp2p data - #[allow(clippy::type_complexity)] - pub async fn get_config_without_peer( - &self, - libp2p_advertise_address: Option, - libp2p_public_key: Option, - ) -> anyhow::Result> { - // Serialize our (possible) libp2p-specific data - let request_body = vbs::Serializer::::serialize(&( - libp2p_advertise_address, - libp2p_public_key, - ))?; - - let identity = |client: Client| { - // We need to clone here to move it into the closure - let request_body = request_body.clone(); - async move { - let node_index: Result = client - .post("api/identity") - .body_binary(&request_body) - .expect("failed to set request body") - .send() - .await; - - node_index - } - .boxed() - }; - let node_index = self.wait_for_fn_from_orchestrator(identity).await; - - // get the corresponding config - let f = |client: Client| { - async move { - let config: Result, ClientError> = client - .post(&format!("api/config/{node_index}")) - .send() - .await; - config - } - .boxed() - }; - - let mut config = self.wait_for_fn_from_orchestrator(f).await; - config.node_index = From::::from(node_index); - - Ok(config) - } - - /// Post to the orchestrator and get the latest `node_index` - /// Then return it for the init validator config - /// # Panics - /// if unable to post - #[instrument(skip_all, name = "orchestrator node index for validator config")] - pub async fn get_node_index_for_init_validator_config(&self) -> u16 { - let cur_node_index = |client: Client| { - async move { - let cur_node_index: Result = client - .post("api/get_tmp_node_index") - .send() - .await - .inspect_err(|err| tracing::error!("{err}")); - - cur_node_index - } - .boxed() - }; - self.wait_for_fn_from_orchestrator(cur_node_index).await - } - - /// Requests the configuration from the orchestrator with the stipulation that - /// a successful call requires all nodes to be registered. - /// - /// Does not fail, retries internally until success. - #[instrument(skip_all, name = "orchestrator config")] - pub async fn get_config_after_collection(&self) -> NetworkConfig { - // Define the request for post-register configurations - let get_config_after_collection = |client: Client| { - async move { - let result = client - .post("api/post_config_after_peer_collected") - .send() - .await; - - if let Err(ref err) = result { - tracing::error!("{err}"); - } - - result - } - .boxed() - }; - - // Loop until successful - self.wait_for_fn_from_orchestrator(get_config_after_collection) - .await - } - - /// Registers a builder URL with the orchestrator - /// - /// # Panics - /// if unable to serialize `address` - pub async fn post_builder_addresses(&self, addresses: Vec) { - let send_builder_f = |client: Client| { - let request_body = vbs::Serializer::::serialize(&addresses) - .expect("Failed to serialize request"); - - async move { - let result: Result<_, ClientError> = client - .post("api/builder") - .body_binary(&request_body) - .unwrap() - .send() - .await - .inspect_err(|err| tracing::error!("{err}")); - result - } - .boxed() - }; - self.wait_for_fn_from_orchestrator::<_, _, ()>(send_builder_f) - .await; - } - - /// Requests a builder URL from orchestrator - pub async fn get_builder_addresses(&self) -> Vec { - // Define the request for post-register configurations - let get_builder = |client: Client| { - async move { - let result = client.get("api/builders").send().await; - - if let Err(ref err) = result { - tracing::error!("{err}"); - } - - result - } - .boxed() - }; - - // Loop until successful - self.wait_for_fn_from_orchestrator(get_builder).await - } - - /// Sends my public key to the orchestrator so that it can collect all public keys - /// And get the updated config - /// Blocks until the orchestrator collects all peer's public keys/configs - /// # Panics - /// if unable to post - #[instrument(skip(self), name = "orchestrator public keys")] - pub async fn post_and_wait_all_public_keys( - &self, - validator_config: &mut ValidatorConfig, - libp2p_advertise_address: Option, - libp2p_public_key: Option, - ) -> NetworkConfig { - let pubkey: Vec = PeerConfig::::to_bytes(&validator_config.public_config()).clone(); - let da_requested: bool = validator_config.is_da; - - // Serialize our (possible) libp2p-specific data - let request_body = vbs::Serializer::::serialize(&( - pubkey, - libp2p_advertise_address, - libp2p_public_key, - )) - .expect("failed to serialize request"); - - // register our public key with the orchestrator - let (node_index, is_da): (u64, bool) = loop { - let result = self - .client - .post(&format!("api/pubkey/{da_requested}")) - .body_binary(&request_body) - .expect("Failed to form request") - .send() - .await - .inspect_err(|err| tracing::error!("{err}")); - - if let Ok((index, is_da)) = result { - break (index, is_da); - } - - sleep(Duration::from_millis(250)).await; - }; - - validator_config.is_da = is_da; - - // wait for all nodes' public keys - let wait_for_all_nodes_pub_key = |client: Client| { - async move { - client - .get("api/peer_pub_ready") - .send() - .await - .inspect_err(|err| tracing::error!("{err}")) - } - .boxed() - }; - self.wait_for_fn_from_orchestrator::<_, _, ()>(wait_for_all_nodes_pub_key) - .await; - - let mut network_config = self.get_config_after_collection().await; - - network_config.node_index = node_index; - - network_config - } - - /// Tells the orchestrator this validator is ready to start - /// Blocks until the orchestrator indicates all nodes are ready to start - /// # Panics - /// Panics if unable to post. - #[instrument(skip(self), name = "orchestrator ready signal")] - pub async fn wait_for_all_nodes_ready(&self, peer_config: Vec) -> bool { - let send_ready_f = |client: Client| { - let pk = peer_config.clone(); - async move { - let result: Result<_, ClientError> = client - .post("api/ready") - .body_binary(&pk) - .unwrap() - .send() - .await - .inspect_err(|err| tracing::error!("{err}")); - result - } - .boxed() - }; - self.wait_for_fn_from_orchestrator::<_, _, ()>(send_ready_f) - .await; - - let wait_for_all_nodes_ready_f = |client: Client| { - async move { client.get("api/start").send().await }.boxed() - }; - self.wait_for_fn_from_orchestrator(wait_for_all_nodes_ready_f) - .await - } - - /// Sends the benchmark metrics to the orchestrator - /// # Panics - /// Panics if unable to post - #[instrument(skip_all, name = "orchestrator metrics")] - pub async fn post_bench_results(&self, bench_results: BenchResults) { - let _send_metrics_f: Result<(), ClientError> = self - .client - .post("api/results") - .body_json(&bench_results) - .unwrap() - .send() - .await - .inspect_err(|err| tracing::warn!("{err}")); - } - - /// Generic function that waits for the orchestrator to return a non-error - /// Returns whatever type the given function returns - #[instrument(skip_all, name = "waiting for orchestrator")] - async fn wait_for_fn_from_orchestrator(&self, f: F) -> GEN - where - F: Fn(Client) -> Fut, - Fut: Future>, - { - loop { - let client = self.client.clone(); - let res = f(client).await; - match res { - Ok(x) => break x, - Err(err) => { - tracing::info!("{err}"); - sleep(Duration::from_millis(250)).await; - } - } - } - } -} diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs deleted file mode 100644 index bafdccbe2c..0000000000 --- a/crates/orchestrator/src/lib.rs +++ /dev/null @@ -1,881 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! Orchestrator for manipulating nodes and recording results during a run of `HotShot` tests - -/// The orchestrator's clients -pub mod client; - -use std::{ - collections::{HashMap, HashSet}, - fs, - fs::OpenOptions, - io::{self, ErrorKind}, - time::Duration, -}; - -use async_lock::RwLock; -use client::{BenchResults, BenchResultsDownloadConfig}; -use csv::Writer; -use futures::{stream::FuturesUnordered, FutureExt, StreamExt}; -use hotshot_types::{ - network::{BuilderType, NetworkConfig, PublicKeysFile}, - traits::signature_key::{SignatureKey, StakeTableEntryType}, - PeerConfig, -}; -use libp2p_identity::{ - ed25519::{Keypair as EdKeypair, SecretKey}, - Keypair, PeerId, -}; -use multiaddr::Multiaddr; -use surf_disco::Url; -use tide_disco::{ - api::ApiError, - error::ServerError, - method::{ReadState, WriteState}, - Api, App, RequestError, -}; -use vbs::{ - version::{StaticVersion, StaticVersionType}, - BinarySerializer, -}; - -/// Orchestrator is not, strictly speaking, bound to the network; it can have its own versioning. -/// Orchestrator Version (major) -pub const ORCHESTRATOR_MAJOR_VERSION: u16 = 0; -/// Orchestrator Version (minor) -pub const ORCHESTRATOR_MINOR_VERSION: u16 = 1; -/// Orchestrator Version as a type -pub type OrchestratorVersion = - StaticVersion; -/// Orchestrator Version as a type-binding instance -pub const ORCHESTRATOR_VERSION: OrchestratorVersion = StaticVersion {}; - -/// Generate an keypair based on a `seed` and an `index` -/// # Panics -/// This panics if libp2p is unable to generate a secret key from the seed -#[must_use] -pub fn libp2p_generate_indexed_identity(seed: [u8; 32], index: u64) -> Keypair { - let mut hasher = blake3::Hasher::new(); - hasher.update(&seed); - hasher.update(&index.to_le_bytes()); - let new_seed = *hasher.finalize().as_bytes(); - let sk_bytes = SecretKey::try_from_bytes(new_seed).unwrap(); - >::from(sk_bytes).into() -} - -/// The state of the orchestrator -#[derive(Default, Clone)] -#[allow(clippy::struct_excessive_bools)] -struct OrchestratorState { - /// Tracks the latest node index we have generated a configuration for - latest_index: u16, - /// Tracks the latest temporary index we have generated for init validator's key pair - tmp_latest_index: u16, - /// The network configuration - config: NetworkConfig, - /// Whether the network configuration has been updated with all the peer's public keys/configs - peer_pub_ready: bool, - /// A map from public keys to `(node_index, is_da)`. - pub_posted: HashMap, (u64, bool)>, - /// Whether nodes should start their HotShot instances - /// Will be set to true once all nodes post they are ready to start - start: bool, - /// The total nodes that have posted they are ready to start - nodes_connected: HashSet>, - /// The results of the benchmarks - bench_results: BenchResults, - /// The number of nodes that have posted their results - nodes_post_results: u64, - /// Whether the orchestrator can be started manually - manual_start_allowed: bool, - /// Whether we are still accepting new keys for registration - accepting_new_keys: bool, - /// Builder address pool - builders: Vec, - /// whether we are using a fixed stake table, disabling public key registration - fixed_stake_table: bool, -} - -impl OrchestratorState { - /// create a new [`OrchestratorState`] - pub fn new(network_config: NetworkConfig) -> Self { - let mut peer_pub_ready = false; - let mut fixed_stake_table = false; - - if network_config.config.known_nodes_with_stake.is_empty() { - println!("No nodes were loaded from the config file. Nodes will be allowed to register dynamically."); - } else { - println!("Initializing orchestrator with fixed stake table."); - peer_pub_ready = true; - fixed_stake_table = true; - } - - let builders = if matches!(network_config.builder, BuilderType::External) { - network_config.config.builder_urls.clone().into() - } else { - vec![] - }; - - OrchestratorState { - latest_index: 0, - tmp_latest_index: 0, - config: network_config, - peer_pub_ready, - pub_posted: HashMap::new(), - nodes_connected: HashSet::new(), - start: false, - bench_results: BenchResults::default(), - nodes_post_results: 0, - manual_start_allowed: true, - accepting_new_keys: true, - builders, - fixed_stake_table, - } - } - - /// Output the results to a csv file according to orchestrator state - pub fn output_to_csv(&self) { - let output_csv = BenchResultsDownloadConfig { - commit_sha: self.config.commit_sha.clone(), - total_nodes: self.config.config.num_nodes_with_stake.into(), - da_committee_size: self.config.config.da_staked_committee_size, - fixed_leader_for_gpuvid: self.config.config.fixed_leader_for_gpuvid, - transactions_per_round: self.config.transactions_per_round, - transaction_size: self.bench_results.transaction_size_in_bytes, - rounds: self.config.rounds, - partial_results: self.bench_results.partial_results.clone(), - avg_latency_in_sec: self.bench_results.avg_latency_in_sec, - minimum_latency_in_sec: self.bench_results.minimum_latency_in_sec, - maximum_latency_in_sec: self.bench_results.maximum_latency_in_sec, - throughput_bytes_per_sec: self.bench_results.throughput_bytes_per_sec, - total_transactions_committed: self.bench_results.total_transactions_committed, - total_time_elapsed_in_sec: self.bench_results.total_time_elapsed_in_sec, - total_num_views: self.bench_results.total_num_views, - failed_num_views: self.bench_results.failed_num_views, - committee_type: self.bench_results.committee_type.clone(), - }; - // Open the CSV file in append mode - let results_csv_file = OpenOptions::new() - .create(true) - .append(true) // Open in append mode - .open("scripts/benchmarks_results/results.csv") - .unwrap(); - // Open a file for writing - let mut wtr = Writer::from_writer(results_csv_file); - let _ = wtr.serialize(output_csv); - let _ = wtr.flush(); - println!("Results successfully saved in scripts/benchmarks_results/results.csv"); - } -} - -/// An api exposed by the orchestrator -pub trait OrchestratorApi { - /// Post an identity to the orchestrator. Takes in optional - /// arguments so others can identify us on the Libp2p network. - /// # Errors - /// If we were unable to serve the request - fn post_identity( - &mut self, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result; - /// post endpoint for each node's config - /// # Errors - /// if unable to serve - fn post_getconfig(&mut self, _node_index: u16) -> Result, ServerError>; - /// get endpoint for the next available temporary node index - /// # Errors - /// if unable to serve - fn get_tmp_node_index(&mut self) -> Result; - /// post endpoint for each node's public key - /// # Errors - /// if unable to serve - fn register_public_key( - &mut self, - pubkey: &mut Vec, - is_da: bool, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result<(u64, bool), ServerError>; - /// post endpoint for whether or not all peers public keys are ready - /// # Errors - /// if unable to serve - fn peer_pub_ready(&self) -> Result; - /// get endpoint for the network config after all peers public keys are collected - /// # Errors - /// if unable to serve - fn post_config_after_peer_collected(&mut self) -> Result, ServerError>; - /// get endpoint for whether or not the run has started - /// # Errors - /// if unable to serve - fn get_start(&self) -> Result; - /// post endpoint for the results of the run - /// # Errors - /// if unable to serve - fn post_run_results(&mut self, metrics: BenchResults) -> Result<(), ServerError>; - /// A node POSTs its public key to let the orchestrator know that it is ready - /// # Errors - /// if unable to serve - fn post_ready(&mut self, peer_config: &PeerConfig) -> Result<(), ServerError>; - /// post endpoint for manually starting the orchestrator - /// # Errors - /// if unable to serve - fn post_manual_start(&mut self, password_bytes: Vec) -> Result<(), ServerError>; - /// post endpoint for registering a builder with the orchestrator - /// # Errors - /// if unable to serve - fn post_builder(&mut self, builder: Url) -> Result<(), ServerError>; - /// get endpoints for builders - /// # Errors - /// if not all builders are registered yet - fn get_builders(&self) -> Result, ServerError>; -} - -impl OrchestratorState -where - KEY: serde::Serialize + Clone + SignatureKey + 'static, -{ - /// register a node with an unknown public key. - /// this method should be used when we don't have a fixed stake table - fn register_unknown( - &mut self, - pubkey: &mut Vec, - da_requested: bool, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result<(u64, bool), ServerError> { - if let Some((node_index, is_da)) = self.pub_posted.get(pubkey) { - return Ok((*node_index, *is_da)); - } - - if !self.accepting_new_keys { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: - "Network has been started manually, and is no longer registering new keys." - .to_string(), - }); - } - - let node_index = self.pub_posted.len() as u64; - - // Deserialize the public key - let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); - - self.config - .config - .known_nodes_with_stake - .push(staked_pubkey.clone()); - - let mut added_to_da = false; - - let da_full = - self.config.config.known_da_nodes.len() >= self.config.config.da_staked_committee_size; - - #[allow(clippy::nonminimal_bool)] - // We add the node to the DA committee depending on either its node index or whether it requested membership. - // - // Since we issue `node_index` incrementally, if we are deciding DA membership by node_index - // we only need to check that the DA committee is not yet full. - // - // Note: this logically simplifies to (self.config.indexed_da || da_requested) && !da_full, - // but writing it that way makes it a little less clear to me. - if (self.config.indexed_da || (!self.config.indexed_da && da_requested)) && !da_full { - self.config.config.known_da_nodes.push(staked_pubkey); - added_to_da = true; - } - - self.pub_posted - .insert(pubkey.clone(), (node_index, added_to_da)); - - // If the orchestrator is set up for libp2p and we have supplied the proper - // Libp2p data, add our node to the list of bootstrap nodes. - if self.config.libp2p_config.clone().is_some() { - if let (Some(libp2p_public_key), Some(libp2p_address)) = - (libp2p_public_key, libp2p_address) - { - // Push to our bootstrap nodes - self.config - .libp2p_config - .as_mut() - .unwrap() - .bootstrap_nodes - .push((libp2p_public_key, libp2p_address)); - } - } - - tracing::error!("Posted public key for node_index {node_index}"); - - // node_index starts at 0, so once it matches `num_nodes_with_stake` - // we will have registered one node too many. hence, we want `node_index + 1`. - if node_index + 1 >= (self.config.config.num_nodes_with_stake.get() as u64) { - self.peer_pub_ready = true; - self.accepting_new_keys = false; - } - Ok((node_index, added_to_da)) - } - - /// register a node on the fixed stake table, which was loaded at startup - fn register_from_list( - &mut self, - pubkey: &mut Vec, - da_requested: bool, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result<(u64, bool), ServerError> { - // if we've already registered this node before, we just retrieve its info from `pub_posted` - if let Some((node_index, is_da)) = self.pub_posted.get(pubkey) { - return Ok((*node_index, *is_da)); - } - - // Deserialize the public key - let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); - - // Check if the node is allowed to connect, returning its index and config entry if so. - let Some((node_index, node_config)) = - self.config.public_keys.iter().enumerate().find(|keys| { - keys.1.stake_table_key == staked_pubkey.stake_table_entry.public_key() - }) - else { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "You are unauthorized to register with the orchestrator".to_string(), - }); - }; - - // Check that our recorded DA status for the node matches what the node actually requested - if node_config.da != da_requested { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: format!("Mismatch in DA status in registration for node {}. DA requested: {}, expected: {}", node_index, da_requested, node_config.da), - }); - } - - let added_to_da = node_config.da; - - self.pub_posted - .insert(pubkey.clone(), (node_index as u64, added_to_da)); - - // If the orchestrator is set up for libp2p and we have supplied the proper - // Libp2p data, add our node to the list of bootstrap nodes. - if self.config.libp2p_config.clone().is_some() { - if let (Some(libp2p_public_key), Some(libp2p_address)) = - (libp2p_public_key, libp2p_address) - { - // Push to our bootstrap nodes - self.config - .libp2p_config - .as_mut() - .unwrap() - .bootstrap_nodes - .push((libp2p_public_key, libp2p_address)); - } - } - - tracing::error!("Node {node_index} has registered."); - - Ok((node_index as u64, added_to_da)) - } -} - -impl OrchestratorApi for OrchestratorState -where - KEY: serde::Serialize + Clone + SignatureKey + 'static, -{ - /// Post an identity to the orchestrator. Takes in optional - /// arguments so others can identify us on the Libp2p network. - /// # Errors - /// If we were unable to serve the request - fn post_identity( - &mut self, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result { - let node_index = self.latest_index; - self.latest_index += 1; - - if usize::from(node_index) >= self.config.config.num_nodes_with_stake.get() { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Network has reached capacity".to_string(), - }); - } - - // If the orchestrator is set up for libp2p and we have supplied the proper - // Libp2p data, add our node to the list of bootstrap nodes. - if self.config.libp2p_config.clone().is_some() { - if let (Some(libp2p_public_key), Some(libp2p_address)) = - (libp2p_public_key, libp2p_address) - { - // Push to our bootstrap nodes - self.config - .libp2p_config - .as_mut() - .unwrap() - .bootstrap_nodes - .push((libp2p_public_key, libp2p_address)); - } - } - Ok(node_index) - } - - // Assumes nodes will set their own index that they received from the - // 'identity' endpoint - fn post_getconfig(&mut self, _node_index: u16) -> Result, ServerError> { - Ok(self.config.clone()) - } - - // Assumes one node do not get twice - fn get_tmp_node_index(&mut self) -> Result { - let tmp_node_index = self.tmp_latest_index; - self.tmp_latest_index += 1; - - if usize::from(tmp_node_index) >= self.config.config.num_nodes_with_stake.get() { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Node index getter for key pair generation has reached capacity" - .to_string(), - }); - } - Ok(tmp_node_index) - } - - #[allow(clippy::cast_possible_truncation)] - fn register_public_key( - &mut self, - pubkey: &mut Vec, - da_requested: bool, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result<(u64, bool), ServerError> { - if self.fixed_stake_table { - self.register_from_list(pubkey, da_requested, libp2p_address, libp2p_public_key) - } else { - self.register_unknown(pubkey, da_requested, libp2p_address, libp2p_public_key) - } - } - - fn peer_pub_ready(&self) -> Result { - if !self.peer_pub_ready { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Peer's public configs are not ready".to_string(), - }); - } - Ok(self.peer_pub_ready) - } - - fn post_config_after_peer_collected(&mut self) -> Result, ServerError> { - if !self.peer_pub_ready { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Peer's public configs are not ready".to_string(), - }); - } - - Ok(self.config.clone()) - } - - fn get_start(&self) -> Result { - // println!("{}", self.start); - if !self.start { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Network is not ready to start".to_string(), - }); - } - Ok(self.start) - } - - // Assumes nodes do not post 'ready' twice - fn post_ready(&mut self, peer_config: &PeerConfig) -> Result<(), ServerError> { - // If we have not disabled registration verification. - // Is this node allowed to connect? - if !self - .config - .config - .known_nodes_with_stake - .contains(peer_config) - { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "You are unauthorized to register with the orchestrator".to_string(), - }); - } - - // `HashSet::insert()` returns whether the node was newly inserted (true) or not - if self.nodes_connected.insert(peer_config.clone()) { - tracing::error!( - "Node {peer_config} connected. Total nodes connected: {}", - self.nodes_connected.len() - ); - } - - // i.e. nodes_connected >= num_nodes_with_stake * (start_threshold.0 / start_threshold.1) - if self.nodes_connected.len() as u64 * self.config.config.start_threshold.1 - >= (self.config.config.num_nodes_with_stake.get() as u64) - * self.config.config.start_threshold.0 - { - self.accepting_new_keys = false; - self.manual_start_allowed = false; - self.start = true; - } - - Ok(()) - } - - /// Manually start the network - fn post_manual_start(&mut self, password_bytes: Vec) -> Result<(), ServerError> { - if !self.manual_start_allowed { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "Configs have already been distributed to nodes, and the network can no longer be started manually.".to_string(), - }); - } - - let password = String::from_utf8(password_bytes) - .expect("Failed to decode raw password as UTF-8 string."); - - // Check that the password matches - if self.config.manual_start_password != Some(password) { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "Incorrect password.".to_string(), - }); - } - - let registered_nodes_with_stake = self.config.config.known_nodes_with_stake.len(); - let registered_da_nodes = self.config.config.known_da_nodes.len(); - - if registered_da_nodes > 1 { - self.config.config.num_nodes_with_stake = - std::num::NonZeroUsize::new(registered_nodes_with_stake) - .expect("Failed to convert to NonZeroUsize; this should be impossible."); - - self.config.config.da_staked_committee_size = registered_da_nodes; - } else { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: format!("We cannot manually start the network, because we only have {registered_nodes_with_stake} nodes with stake registered, with {registered_da_nodes} DA nodes.") - }); - } - - self.accepting_new_keys = false; - self.manual_start_allowed = false; - self.peer_pub_ready = true; - self.start = true; - - Ok(()) - } - - // Aggregates results of the run from all nodes - fn post_run_results(&mut self, metrics: BenchResults) -> Result<(), ServerError> { - if metrics.total_transactions_committed != 0 { - // Deal with the bench results - if self.bench_results.total_transactions_committed == 0 { - self.bench_results = metrics; - } else { - // Deal with the bench results from different nodes - let cur_metrics = self.bench_results.clone(); - self.bench_results.avg_latency_in_sec = (metrics.avg_latency_in_sec - * metrics.num_latency - + cur_metrics.avg_latency_in_sec * cur_metrics.num_latency) - / (metrics.num_latency + cur_metrics.num_latency); - self.bench_results.num_latency += metrics.num_latency; - self.bench_results.minimum_latency_in_sec = metrics - .minimum_latency_in_sec - .min(cur_metrics.minimum_latency_in_sec); - self.bench_results.maximum_latency_in_sec = metrics - .maximum_latency_in_sec - .max(cur_metrics.maximum_latency_in_sec); - self.bench_results.throughput_bytes_per_sec = metrics - .throughput_bytes_per_sec - .max(cur_metrics.throughput_bytes_per_sec); - self.bench_results.total_transactions_committed = metrics - .total_transactions_committed - .max(cur_metrics.total_transactions_committed); - self.bench_results.total_time_elapsed_in_sec = metrics - .total_time_elapsed_in_sec - .max(cur_metrics.total_time_elapsed_in_sec); - self.bench_results.total_num_views = - metrics.total_num_views.min(cur_metrics.total_num_views); - self.bench_results.failed_num_views = - metrics.failed_num_views.max(cur_metrics.failed_num_views); - } - } - self.nodes_post_results += 1; - if self.bench_results.partial_results == "Unset" { - self.bench_results.partial_results = "One".to_string(); - self.bench_results.printout(); - self.output_to_csv(); - } - if self.bench_results.partial_results == "One" - && self.nodes_post_results >= (self.config.config.da_staked_committee_size as u64 / 2) - { - self.bench_results.partial_results = "HalfDA".to_string(); - self.bench_results.printout(); - self.output_to_csv(); - } - if self.bench_results.partial_results == "HalfDA" - && self.nodes_post_results >= (self.config.config.num_nodes_with_stake.get() as u64 / 2) - { - self.bench_results.partial_results = "Half".to_string(); - self.bench_results.printout(); - self.output_to_csv(); - } - if self.bench_results.partial_results != "Full" - && self.nodes_post_results >= (self.config.config.num_nodes_with_stake.get() as u64) - { - self.bench_results.partial_results = "Full".to_string(); - self.bench_results.printout(); - self.output_to_csv(); - } - Ok(()) - } - - fn post_builder(&mut self, builder: Url) -> Result<(), ServerError> { - self.builders.push(builder); - Ok(()) - } - - fn get_builders(&self) -> Result, ServerError> { - if !matches!(self.config.builder, BuilderType::External) - && self.builders.len() != self.config.config.da_staked_committee_size - { - return Err(ServerError { - status: tide_disco::StatusCode::NOT_FOUND, - message: "Not all builders are registered yet".to_string(), - }); - } - Ok(self.builders.clone()) - } -} - -/// Sets up all API routes -#[allow(clippy::too_many_lines)] -fn define_api() -> Result, ApiError> -where - State: 'static + Send + Sync + ReadState + WriteState, - ::State: Send + Sync + OrchestratorApi, - KEY: serde::Serialize + SignatureKey, - VER: StaticVersionType + 'static, -{ - let api_toml = toml::from_str::(include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/api.toml" - ))) - .expect("API file is not valid toml"); - let mut api = Api::::new(api_toml)?; - api.post("post_identity", |req, state| { - async move { - // Read the bytes from the body - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - - // Decode the libp2p data so we can add to our bootstrap nodes (if supplied) - let Ok((libp2p_address, libp2p_public_key)) = - vbs::Serializer::::deserialize(&body_bytes) - else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - - // Call our state function to process the request - state.post_identity(libp2p_address, libp2p_public_key) - } - .boxed() - })? - .post("post_getconfig", |req, state| { - async move { - let node_index = req.integer_param("node_index")?; - state.post_getconfig(node_index) - } - .boxed() - })? - .post("get_tmp_node_index", |_req, state| { - async move { state.get_tmp_node_index() }.boxed() - })? - .post("post_pubkey", |req, state| { - async move { - let is_da = req.boolean_param("is_da")?; - // Read the bytes from the body - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - - // Decode the libp2p data so we can add to our bootstrap nodes (if supplied) - let Ok((mut pubkey, libp2p_address, libp2p_public_key)) = - vbs::Serializer::::deserialize(&body_bytes) - else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - - state.register_public_key(&mut pubkey, is_da, libp2p_address, libp2p_public_key) - } - .boxed() - })? - .get("peer_pubconfig_ready", |_req, state| { - async move { state.peer_pub_ready() }.boxed() - })? - .post("post_config_after_peer_collected", |_req, state| { - async move { state.post_config_after_peer_collected() }.boxed() - })? - .post( - "post_ready", - |req, state: &mut ::State| { - async move { - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - // Decode the payload-supplied pubkey - let Some(pubkey) = PeerConfig::::from_bytes(&body_bytes) else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - state.post_ready(&pubkey) - } - .boxed() - }, - )? - .post( - "post_manual_start", - |req, state: &mut ::State| { - async move { - let password = req.body_bytes(); - state.post_manual_start(password) - } - .boxed() - }, - )? - .get("get_start", |_req, state| { - async move { state.get_start() }.boxed() - })? - .post("post_results", |req, state| { - async move { - let metrics: Result = req.body_json(); - state.post_run_results(metrics.unwrap()) - } - .boxed() - })? - .post("post_builder", |req, state| { - async move { - // Read the bytes from the body - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - - let Ok(urls) = - vbs::Serializer::::deserialize::>(&body_bytes) - else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - - let mut futures = urls - .into_iter() - .map(|url| async { - let client: surf_disco::Client = - surf_disco::client::Client::builder(url.clone()).build(); - if client.connect(Some(Duration::from_secs(2))).await { - Some(url) - } else { - None - } - }) - .collect::>() - .filter_map(futures::future::ready); - - if let Some(url) = futures.next().await { - state.post_builder(url) - } else { - Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "No reachable addresses".to_string(), - }) - } - } - .boxed() - })? - .get("get_builders", |_req, state| { - async move { state.get_builders() }.boxed() - })?; - Ok(api) -} - -/// Runs the orchestrator -/// # Errors -/// This errors if tide disco runs into an issue during serving -/// # Panics -/// This panics if unable to register the api with tide disco -pub async fn run_orchestrator( - mut network_config: NetworkConfig, - url: Url, -) -> io::Result<()> -where - KEY: SignatureKey + 'static + serde::Serialize, -{ - let env_password = std::env::var("ORCHESTRATOR_MANUAL_START_PASSWORD"); - - if env_password.is_ok() { - tracing::warn!("Took orchestrator manual start password from the environment variable: ORCHESTRATOR_MANUAL_START_PASSWORD={:?}", env_password); - network_config.manual_start_password = env_password.ok(); - } - - // Try to overwrite the network_config public keys - // from the file the env var points to, or panic. - { - let env_public_keys = std::env::var("ORCHESTRATOR_PUBLIC_KEYS"); - - if let Ok(filepath) = env_public_keys { - #[allow(clippy::panic)] - let config_file_as_string: String = fs::read_to_string(filepath.clone()) - .unwrap_or_else(|_| panic!("Could not read config file located at {filepath}")); - - let file: PublicKeysFile = - toml::from_str::>(&config_file_as_string) - .expect("Unable to convert config file to TOML"); - - network_config.public_keys = file.public_keys; - } - } - - network_config.config.known_nodes_with_stake = network_config - .public_keys - .iter() - .map(|keys| PeerConfig { - stake_table_entry: keys.stake_table_key.stake_table_entry(keys.stake), - state_ver_key: keys.state_ver_key.clone(), - }) - .collect(); - - network_config.config.known_da_nodes = network_config - .public_keys - .iter() - .filter(|keys| keys.da) - .map(|keys| PeerConfig { - stake_table_entry: keys.stake_table_key.stake_table_entry(keys.stake), - state_ver_key: keys.state_ver_key.clone(), - }) - .collect(); - - let web_api = - define_api().map_err(|_e| io::Error::new(ErrorKind::Other, "Failed to define api")); - - let state: RwLock> = RwLock::new(OrchestratorState::new(network_config)); - - let mut app = App::>, ServerError>::with_state(state); - app.register_module::("api", web_api.unwrap()) - .expect("Error registering api"); - tracing::error!("listening on {:?}", url); - app.serve(url, ORCHESTRATOR_VERSION).await -} diff --git a/crates/orchestrator/staging-config.toml b/crates/orchestrator/staging-config.toml deleted file mode 100644 index 61c5adb696..0000000000 --- a/crates/orchestrator/staging-config.toml +++ /dev/null @@ -1,76 +0,0 @@ -rounds = 10 -indexed_da = false -transactions_per_round = 10 -manual_start_password = "tuktu6-tohnaX-gihxib" -node_index = 0 -seed = [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 -] -transaction_size = 100 -builder = "Simple" - -[config] -start_threshold = [ 8, 10 ] -num_nodes_with_stake = 10 -staked_da_nodes = 10 -fixed_leader_for_gpuvid = 1 -next_view_timeout = 15_000 -num_bootstrap = 5 -builder_urls = [ "https://builder.staging.testnet.espresso.network/" ] - -[config.view_sync_timeout] -secs = 15 -nanos = 0 - -[config.builder_timeout] -secs = 8 -nanos = 0 - -[config.data_request_delay] -secs = 5 -nanos = 0 - -[config.upgrade] -start_proposing_view = 1 -stop_proposing_view = 0 -start_voting_view = 1 -stop_voting_view = 0 -start_proposing_time = 1 -stop_proposing_time = 0 -start_voting_time = 1 -stop_voting_time = 0 - -[combined_network_config.delay_duration] -secs = 5 -nanos = 0 diff --git a/crates/testing/src/block_builder/random.rs b/crates/testing/src/block_builder/random.rs index b5c1fb93f7..368ad40691 100644 --- a/crates/testing/src/block_builder/random.rs +++ b/crates/testing/src/block_builder/random.rs @@ -27,7 +27,7 @@ use hotshot_builder_api::v0_1::{ }; use hotshot_example_types::block_types::TestTransaction; use hotshot_types::{ - network::RandomBuilderConfig, + builder::RandomBuilderConfig, traits::{node_implementation::NodeType, signature_key::BuilderSignatureKey}, utils::BuilderCommitment, vid::VidCommitment, diff --git a/crates/testing/src/test_builder.rs b/crates/testing/src/test_builder.rs index 5bd6f38d79..ccfce00fef 100644 --- a/crates/testing/src/test_builder.rs +++ b/crates/testing/src/test_builder.rs @@ -64,8 +64,6 @@ pub struct TestDescription, V: Ver /// Whether to skip initializing nodes that will start late, which will catch up later with /// `HotShotInitializer::from_reload` in the spinning task. pub skip_late: bool, - /// number of bootstrap nodes (libp2p usage only) - pub num_bootstrap_nodes: usize, /// Size of the staked DA committee for the test pub da_staked_committee_size: usize, /// overall safety property description @@ -299,7 +297,6 @@ impl, V: Versions> TestDescription let num_nodes_with_stake = 100; Self { - num_bootstrap_nodes: num_nodes_with_stake, num_nodes_with_stake, start_nodes: num_nodes_with_stake, overall_safety_properties: OverallSafetyPropertiesDescription:: { @@ -326,7 +323,6 @@ impl, V: Versions> TestDescription pub fn default_multiple_rounds() -> Self { let num_nodes_with_stake = 10; TestDescription:: { - num_bootstrap_nodes: num_nodes_with_stake, num_nodes_with_stake, start_nodes: num_nodes_with_stake, overall_safety_properties: OverallSafetyPropertiesDescription:: { @@ -354,7 +350,6 @@ impl, V: Versions> TestDescription Self { num_nodes_with_stake, start_nodes: num_nodes_with_stake, - num_bootstrap_nodes: num_nodes_with_stake, // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the // following issue. @@ -390,7 +385,6 @@ impl, V: Versions> Default num_nodes_with_stake, start_nodes: num_nodes_with_stake, skip_late: false, - num_bootstrap_nodes: num_nodes_with_stake, da_staked_committee_size: num_nodes_with_stake, spinning_properties: SpinningTaskDescription { node_changes: vec![], @@ -451,7 +445,6 @@ where ) -> TestLauncher { let TestDescription { num_nodes_with_stake, - num_bootstrap_nodes, timing_data, da_staked_committee_size, unreliable_network, @@ -494,7 +487,6 @@ where num_nodes_with_stake: NonZeroUsize::new(num_nodes_with_stake).unwrap(), // Currently making this zero for simplicity known_da_nodes, - num_bootstrap: num_bootstrap_nodes, known_nodes_with_stake, da_staked_committee_size, fixed_leader_for_gpuvid: 1, @@ -535,7 +527,6 @@ where resource_generator: ResourceGenerators { channel_generator: >::gen_networks( num_nodes_with_stake, - num_bootstrap_nodes, da_staked_committee_size, unreliable_network, secondary_network_delay, diff --git a/crates/testing/tests/tests_1/block_builder.rs b/crates/testing/tests/tests_1/block_builder.rs index fc29b1c01d..66e262c161 100644 --- a/crates/testing/tests/tests_1/block_builder.rs +++ b/crates/testing/tests/tests_1/block_builder.rs @@ -19,7 +19,7 @@ use hotshot_testing::block_builder::{ BuilderTask, RandomBuilderImplementation, TestBuilderImplementation, }; use hotshot_types::{ - network::RandomBuilderConfig, + builder::RandomBuilderConfig, traits::{ block_contents::vid_commitment, node_implementation::NodeType, signature_key::SignatureKey, BlockPayload, diff --git a/crates/testing/tests/tests_1/test_success.rs b/crates/testing/tests/tests_1/test_success.rs index 982b7018f6..c5fb0ac7e1 100644 --- a/crates/testing/tests/tests_1/test_success.rs +++ b/crates/testing/tests/tests_1/test_success.rs @@ -140,7 +140,6 @@ cross_tests!( Ignore: false, Metadata: { let mut metadata = TestDescription::default_more_nodes(); - metadata.num_bootstrap_nodes = 10; metadata.num_nodes_with_stake = 12; metadata.da_staked_committee_size = 12; metadata.start_nodes = 12; diff --git a/crates/testing/tests/tests_1/test_with_failures_2.rs b/crates/testing/tests/tests_1/test_with_failures_2.rs index c922b7fd6e..a474be0151 100644 --- a/crates/testing/tests/tests_1/test_with_failures_2.rs +++ b/crates/testing/tests/tests_1/test_with_failures_2.rs @@ -39,7 +39,6 @@ cross_tests!( Ignore: false, Metadata: { let mut metadata = TestDescription::default_more_nodes(); - metadata.num_bootstrap_nodes = 10; metadata.num_nodes_with_stake = 12; metadata.da_staked_committee_size = 12; metadata.start_nodes = 12; @@ -78,7 +77,6 @@ cross_tests!( Ignore: false, Metadata: { let mut metadata = TestDescription::default_more_nodes(); - metadata.num_bootstrap_nodes = 10; metadata.num_nodes_with_stake = 12; metadata.da_staked_committee_size = 12; metadata.start_nodes = 12; diff --git a/crates/testing/tests/tests_3/test_with_failures_half_f.rs b/crates/testing/tests/tests_3/test_with_failures_half_f.rs index a8a2dbb14b..d62ccec475 100644 --- a/crates/testing/tests/tests_3/test_with_failures_half_f.rs +++ b/crates/testing/tests/tests_3/test_with_failures_half_f.rs @@ -23,7 +23,6 @@ cross_tests!( Ignore: false, Metadata: { let mut metadata = TestDescription::default_more_nodes(); - metadata.num_bootstrap_nodes = 17; // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the // following issue. @@ -60,7 +59,6 @@ cross_tests!( Ignore: false, Metadata: { let mut metadata = TestDescription::default_more_nodes(); - metadata.num_bootstrap_nodes = 17; // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the // following issue. diff --git a/crates/testing/tests/tests_4/test_with_failures_f.rs b/crates/testing/tests/tests_4/test_with_failures_f.rs index 4fd033d0af..221f1ad3dc 100644 --- a/crates/testing/tests/tests_4/test_with_failures_f.rs +++ b/crates/testing/tests/tests_4/test_with_failures_f.rs @@ -26,7 +26,6 @@ cross_tests!( metadata.overall_safety_properties.num_failed_views = 6; // Make sure we keep committing rounds after the bad leaders, but not the full 50 because of the numerous timeouts metadata.overall_safety_properties.num_successful_views = 20; - metadata.num_bootstrap_nodes = 14; // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the // following issue. diff --git a/crates/testing/tests/tests_5/combined_network.rs b/crates/testing/tests/tests_5/combined_network.rs index 67fb0e4030..fadffba021 100644 --- a/crates/testing/tests/tests_5/combined_network.rs +++ b/crates/testing/tests/tests_5/combined_network.rs @@ -231,7 +231,6 @@ async fn test_stress_combined_network_fuzzy() { hotshot::helpers::initialize_logging(); let mut metadata: TestDescription = TestDescription { - num_bootstrap_nodes: 10, num_nodes_with_stake: 20, start_nodes: 20, diff --git a/crates/testing/tests/tests_5/test_with_failures.rs b/crates/testing/tests/tests_5/test_with_failures.rs index 4fc96e5564..15e5c2b770 100644 --- a/crates/testing/tests/tests_5/test_with_failures.rs +++ b/crates/testing/tests/tests_5/test_with_failures.rs @@ -24,7 +24,6 @@ cross_tests!( Ignore: false, Metadata: { let mut metadata = TestDescription::default_more_nodes(); - metadata.num_bootstrap_nodes = 19; // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the // following issue. diff --git a/crates/testing/tests/tests_5/timeout.rs b/crates/testing/tests/tests_5/timeout.rs index 4466a4e2f6..2186ef9b3a 100644 --- a/crates/testing/tests/tests_5/timeout.rs +++ b/crates/testing/tests/tests_5/timeout.rs @@ -85,7 +85,6 @@ async fn test_timeout_libp2p() { let mut metadata: TestDescription = TestDescription { num_nodes_with_stake: 10, start_nodes: 10, - num_bootstrap_nodes: 10, ..Default::default() }; let dead_nodes = vec![ChangeNode { diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index e99de2e53d..6ebacde892 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -18,7 +18,6 @@ async-trait = { workspace = true } bincode = { workspace = true } bitvec = { workspace = true } blake3 = { workspace = true } -clap = { workspace = true } committable = { workspace = true } derive_more = { workspace = true, features = ["debug"] } digest = { workspace = true, features = ["rand_core"] } @@ -31,17 +30,13 @@ jf-signature = { workspace = true, features = ["bls", "schnorr"] } jf-utils = { workspace = true } jf-vid = { workspace = true } lazy_static = { workspace = true } -libp2p-identity = { workspace = true } memoize = { workspace = true } mnemonic = "1" -multiaddr = { workspace = true } primitive-types = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } serde = { workspace = true } -serde-inline-default = { workspace = true } serde_bytes = { workspace = true } -serde_json = { workspace = true } sha2 = { workspace = true } tagged-base64 = { workspace = true } thiserror = { workspace = true } diff --git a/crates/types/src/builder.rs b/crates/types/src/builder.rs new file mode 100644 index 0000000000..328de97f8e --- /dev/null +++ b/crates/types/src/builder.rs @@ -0,0 +1,20 @@ +/// Options controlling how the random builder generates blocks +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] +pub struct RandomBuilderConfig { + /// How many transactions to include in a block + pub txn_in_block: u64, + /// How many blocks to generate per second + pub blocks_per_second: u32, + /// Range of how big a transaction can be (in bytes) + pub txn_size: std::ops::Range, +} + +impl Default for RandomBuilderConfig { + fn default() -> Self { + Self { + txn_in_block: 100, + blocks_per_second: 1, + txn_size: 20..100, + } + } +} diff --git a/crates/types/src/constants.rs b/crates/types/src/constants.rs index 6a1969ae31..5634071f05 100644 --- a/crates/types/src/constants.rs +++ b/crates/types/src/constants.rs @@ -17,9 +17,6 @@ pub const BUNDLE_FETCH_TIMEOUT: Duration = Duration::from_millis(500); /// the number of views to gather information for ahead of time pub const LOOK_AHEAD: u64 = 5; -/// the default kademlia record republication interval (in seconds) -pub const KAD_DEFAULT_REPUB_INTERVAL_SEC: u64 = 28800; - /// the number of messages to cache in the combined network pub const COMBINED_NETWORK_CACHE_SIZE: usize = 200_000; diff --git a/crates/types/src/hotshot_config_file.rs b/crates/types/src/hotshot_config_file.rs index 53633d5d7a..16e000aa1b 100644 --- a/crates/types/src/hotshot_config_file.rs +++ b/crates/types/src/hotshot_config_file.rs @@ -42,8 +42,6 @@ pub struct HotShotConfigFile { pub next_view_timeout: u64, /// Duration for view sync round timeout pub view_sync_timeout: Duration, - /// Number of network bootstrap nodes - pub num_bootstrap: usize, /// The maximum amount of time a leader can wait to get a block from a builder pub builder_timeout: Duration, /// Time to wait until we request data associated with a proposal @@ -68,7 +66,6 @@ impl From> for HotShotConfig { fixed_leader_for_gpuvid: val.fixed_leader_for_gpuvid, next_view_timeout: val.next_view_timeout, view_sync_timeout: val.view_sync_timeout, - num_bootstrap: val.num_bootstrap, builder_timeout: val.builder_timeout, data_request_delay: val .data_request_delay @@ -122,7 +119,6 @@ impl HotShotConfigFile { fixed_leader_for_gpuvid: 1, next_view_timeout: 10000, view_sync_timeout: Duration::from_millis(1000), - num_bootstrap: 5, builder_timeout: Duration::from_secs(10), data_request_delay: Some(Duration::from_millis(REQUEST_DATA_DELAY)), builder_urls: default_builder_urls(), diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 93076491e1..0f2c390e10 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -39,6 +39,9 @@ pub mod simple_vote; pub mod stake_table; pub mod traits; +/// Holds test builder configuration structures +pub mod builder; + /// Holds the upgrade configuration specification for HotShot nodes. pub mod upgrade_config; pub mod utils; @@ -182,8 +185,6 @@ pub struct HotShotConfig { pub next_view_timeout: u64, /// Duration of view sync round timeouts pub view_sync_timeout: Duration, - /// Number of network bootstrap nodes - pub num_bootstrap: usize, /// The maximum amount of time a leader can wait to get a block from a builder pub builder_timeout: Duration, /// time to wait until we request data associated with a proposal diff --git a/crates/types/src/network.rs b/crates/types/src/network.rs index f2d35b6984..832dfb40aa 100644 --- a/crates/types/src/network.rs +++ b/crates/types/src/network.rs @@ -4,387 +4,13 @@ // You should have received a copy of the MIT License // along with the HotShot repository. If not, see . -use std::{fs, ops::Range, path::Path, time::Duration, vec}; +use std::time::Duration; -use clap::ValueEnum; -use libp2p_identity::PeerId; -use multiaddr::Multiaddr; -use serde_inline_default::serde_inline_default; -use thiserror::Error; -use tracing::error; - -use crate::{ - constants::{ - ORCHESTRATOR_DEFAULT_NUM_ROUNDS, ORCHESTRATOR_DEFAULT_TRANSACTIONS_PER_ROUND, - ORCHESTRATOR_DEFAULT_TRANSACTION_SIZE, REQUEST_DATA_DELAY, - }, - hotshot_config_file::HotShotConfigFile, - light_client::StateVerKey, - traits::signature_key::SignatureKey, - HotShotConfig, ValidatorConfig, -}; - -/// Configuration describing a libp2p node -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -pub struct Libp2pConfig { - /// The bootstrap nodes to connect to (multiaddress, serialized public key) - pub bootstrap_nodes: Vec<(PeerId, Multiaddr)>, -} - -/// configuration for combined network +/// The configuration for the HotShot network #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -pub struct CombinedNetworkConfig { - /// delay duration before sending a message through the secondary network - pub delay_duration: Duration, -} - -/// a network configuration error -#[derive(Error, Debug)] -pub enum NetworkConfigError { - /// Failed to read NetworkConfig from file - #[error("Failed to read NetworkConfig from file")] - ReadFromFileError(std::io::Error), - /// Failed to deserialize loaded NetworkConfig - #[error("Failed to deserialize loaded NetworkConfig")] - DeserializeError(serde_json::Error), - /// Failed to write NetworkConfig to file - #[error("Failed to write NetworkConfig to file")] - WriteToFileError(std::io::Error), - /// Failed to serialize NetworkConfig - #[error("Failed to serialize NetworkConfig")] - SerializeError(serde_json::Error), - /// Failed to recursively create path to NetworkConfig - #[error("Failed to recursively create path to NetworkConfig")] - FailedToCreatePath(std::io::Error), -} - -#[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize, Default, ValueEnum)] -/// configuration for builder type to use -pub enum BuilderType { - /// Use external builder, [config.builder_url] must be - /// set to correct builder address - External, - #[default] - /// Simple integrated builder will be started and used by each hotshot node - Simple, - /// Random integrated builder will be started and used by each hotshot node - Random, -} - -/// Node PeerConfig keys -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -#[serde(bound(deserialize = ""))] -pub struct PeerConfigKeys { - /// The peer's public key - pub stake_table_key: KEY, - /// the peer's state public key - pub state_ver_key: StateVerKey, - /// the peer's stake - pub stake: u64, - /// whether the node is a DA node - pub da: bool, -} - -/// Options controlling how the random builder generates blocks -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -pub struct RandomBuilderConfig { - /// How many transactions to include in a block - pub txn_in_block: u64, - /// How many blocks to generate per second - pub blocks_per_second: u32, - /// Range of how big a transaction can be (in bytes) - pub txn_size: Range, -} - -impl Default for RandomBuilderConfig { - fn default() -> Self { - Self { - txn_in_block: 100, - blocks_per_second: 1, - txn_size: 20..100, - } - } -} - -/// a network configuration -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -#[serde(bound(deserialize = ""))] -pub struct NetworkConfig { - /// number of views to run - pub rounds: usize, - /// whether DA membership is determined by index. - /// if true, the first k nodes to register form the DA committee - /// if false, DA membership is requested by the nodes - pub indexed_da: bool, - /// number of transactions per view - pub transactions_per_round: usize, - /// password to have the orchestrator start the network, - /// regardless of the number of nodes connected. - pub manual_start_password: Option, - /// number of bootstrap nodes - pub num_bootrap: usize, - /// timeout before starting the next view +pub struct NetworkConfig { + /// The timeout before starting the next view pub next_view_timeout: u64, - /// timeout before starting next view sync round + /// The timeout before starting next view sync round pub view_sync_timeout: Duration, - /// The maximum amount of time a leader can wait to get a block from a builder - pub builder_timeout: Duration, - /// time to wait until we request data associated with a proposal - pub data_request_delay: Duration, - /// global index of node (for testing purposes a uid) - pub node_index: u64, - /// unique seed (for randomness? TODO) - pub seed: [u8; 32], - /// size of transactions - pub transaction_size: usize, - /// name of the key type (for debugging) - pub key_type_name: String, - /// the libp2p config - pub libp2p_config: Option, - /// the hotshot config - pub config: HotShotConfig, - /// The address for the Push CDN's "marshal", A.K.A. load balancer - pub cdn_marshal_address: Option, - /// combined network config - pub combined_network_config: Option, - /// the commit this run is based on - pub commit_sha: String, - /// builder to use - pub builder: BuilderType, - /// random builder config - pub random_builder: Option, - /// The list of public keys that are allowed to connect to the orchestrator - pub public_keys: Vec>, -} - -/// the source of the network config -pub enum NetworkConfigSource { - /// we source the network configuration from the orchestrator - Orchestrator, - /// we source the network configuration from a config file on disk - File, -} - -impl NetworkConfig { - /// Get a temporary node index for generating a validator config - #[must_use] - pub fn generate_init_validator_config(cur_node_index: u16, is_da: bool) -> ValidatorConfig { - // This cur_node_index is only used for key pair generation, it's not bound with the node, - // later the node with the generated key pair will get a new node_index from orchestrator. - ValidatorConfig::generated_from_seed_indexed([0u8; 32], cur_node_index.into(), 1, is_da) - } - - /// Loads a `NetworkConfig` from a file. - /// - /// This function takes a file path as a string, reads the file, and then deserializes the contents into a `NetworkConfig`. - /// - /// # Arguments - /// - /// * `file` - A string representing the path to the file from which to load the `NetworkConfig`. - /// - /// # Returns - /// - /// This function returns a `Result` that contains a `NetworkConfig` if the file was successfully read and deserialized, or a `NetworkConfigError` if an error occurred. - /// - /// # Errors - /// - /// This function will return an error if the file cannot be read or if the contents cannot be deserialized into a `NetworkConfig`. - /// - /// # Examples - /// - /// ```ignore - /// # use hotshot_orchestrator::config::NetworkConfig; - /// # use hotshot_types::signature_key::BLSPubKey; - /// // # use hotshot::traits::election::static_committee::StaticElectionConfig; - /// let file = "/path/to/my/config".to_string(); - /// // NOTE: broken due to staticelectionconfig not being importable - /// // cannot import staticelectionconfig from hotshot without creating circular dependency - /// // making this work probably involves the `types` crate implementing a dummy - /// // electionconfigtype just to make this example work - /// let config = NetworkConfig::::from_file(file).unwrap(); - /// ``` - pub fn from_file(file: String) -> Result { - // read from file - let data = match fs::read(file) { - Ok(data) => data, - Err(e) => { - return Err(NetworkConfigError::ReadFromFileError(e)); - } - }; - - // deserialize - match serde_json::from_slice(&data) { - Ok(data) => Ok(data), - Err(e) => Err(NetworkConfigError::DeserializeError(e)), - } - } - - /// Serializes the `NetworkConfig` and writes it to a file. - /// - /// This function takes a file path as a string, serializes the `NetworkConfig` into JSON format using `serde_json` and then writes the serialized data to the file. - /// - /// # Arguments - /// - /// * `file` - A string representing the path to the file where the `NetworkConfig` should be saved. - /// - /// # Returns - /// - /// This function returns a `Result` that contains `()` if the `NetworkConfig` was successfully serialized and written to the file, or a `NetworkConfigError` if an error occurred. - /// - /// # Errors - /// - /// This function will return an error if the `NetworkConfig` cannot be serialized or if the file cannot be written. - /// - /// # Examples - /// - /// ```ignore - /// # use hotshot_orchestrator::config::NetworkConfig; - /// let file = "/path/to/my/config".to_string(); - /// let config = NetworkConfig::from_file(file); - /// config.to_file(file).unwrap(); - /// ``` - pub fn to_file(&self, file: String) -> Result<(), NetworkConfigError> { - // ensure the directory containing the config file exists - if let Some(dir) = Path::new(&file).parent() { - if let Err(e) = fs::create_dir_all(dir) { - return Err(NetworkConfigError::FailedToCreatePath(e)); - } - } - - // serialize - let serialized = match serde_json::to_string_pretty(self) { - Ok(data) => data, - Err(e) => { - return Err(NetworkConfigError::SerializeError(e)); - } - }; - - // write to file - match fs::write(file, serialized) { - Ok(()) => Ok(()), - Err(e) => Err(NetworkConfigError::WriteToFileError(e)), - } - } -} - -impl Default for NetworkConfig { - fn default() -> Self { - Self { - rounds: ORCHESTRATOR_DEFAULT_NUM_ROUNDS, - indexed_da: true, - transactions_per_round: ORCHESTRATOR_DEFAULT_TRANSACTIONS_PER_ROUND, - node_index: 0, - seed: [0u8; 32], - transaction_size: ORCHESTRATOR_DEFAULT_TRANSACTION_SIZE, - manual_start_password: None, - libp2p_config: None, - config: HotShotConfigFile::hotshot_config_5_nodes_10_da().into(), - key_type_name: std::any::type_name::().to_string(), - cdn_marshal_address: None, - combined_network_config: None, - next_view_timeout: 10, - view_sync_timeout: Duration::from_secs(2), - num_bootrap: 5, - builder_timeout: Duration::from_secs(10), - data_request_delay: Duration::from_millis(2500), - commit_sha: String::new(), - builder: BuilderType::default(), - random_builder: None, - public_keys: vec![], - } - } -} - -/// a network config stored in a file -#[serde_inline_default] -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] -#[serde(bound(deserialize = ""))] -pub struct PublicKeysFile { - /// The list of public keys that are allowed to connect to the orchestrator - /// - /// If nonempty, this list becomes the stake table and is used to determine DA membership (ignoring the node's request). - #[serde(default)] - pub public_keys: Vec>, -} - -/// a network config stored in a file -#[serde_inline_default] -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] -#[serde(bound(deserialize = ""))] -pub struct NetworkConfigFile { - /// number of views to run - #[serde_inline_default(ORCHESTRATOR_DEFAULT_NUM_ROUNDS)] - pub rounds: usize, - /// number of views to run - #[serde(default)] - pub indexed_da: bool, - /// number of transactions per view - #[serde_inline_default(ORCHESTRATOR_DEFAULT_TRANSACTIONS_PER_ROUND)] - pub transactions_per_round: usize, - /// password to have the orchestrator start the network, - /// regardless of the number of nodes connected. - #[serde(default)] - pub manual_start_password: Option, - /// global index of node (for testing purposes a uid) - #[serde(default)] - pub node_index: u64, - /// unique seed (for randomness? TODO) - #[serde(default)] - pub seed: [u8; 32], - /// size of transactions - #[serde_inline_default(ORCHESTRATOR_DEFAULT_TRANSACTION_SIZE)] - pub transaction_size: usize, - /// the hotshot config file - #[serde(default = "HotShotConfigFile::hotshot_config_5_nodes_10_da")] - pub config: HotShotConfigFile, - /// The address of the Push CDN's "marshal", A.K.A. load balancer - #[serde(default)] - pub cdn_marshal_address: Option, - /// combined network config - #[serde(default)] - pub combined_network_config: Option, - /// builder to use - #[serde(default)] - pub builder: BuilderType, - /// random builder configuration - #[serde(default)] - pub random_builder: Option, - /// The list of public keys that are allowed to connect to the orchestrator - /// - /// If nonempty, this list becomes the stake table and is used to determine DA membership (ignoring the node's request). - #[serde(default)] - pub public_keys: Vec>, -} - -impl From> for NetworkConfig { - fn from(val: NetworkConfigFile) -> Self { - NetworkConfig { - rounds: val.rounds, - indexed_da: val.indexed_da, - transactions_per_round: val.transactions_per_round, - node_index: 0, - num_bootrap: val.config.num_bootstrap, - manual_start_password: val.manual_start_password, - next_view_timeout: val.config.next_view_timeout, - view_sync_timeout: val.config.view_sync_timeout, - builder_timeout: val.config.builder_timeout, - data_request_delay: val - .config - .data_request_delay - .unwrap_or(Duration::from_millis(REQUEST_DATA_DELAY)), - seed: val.seed, - transaction_size: val.transaction_size, - libp2p_config: Some(Libp2pConfig { - bootstrap_nodes: Vec::new(), - }), - config: val.config.into(), - key_type_name: std::any::type_name::().to_string(), - cdn_marshal_address: val.cdn_marshal_address, - combined_network_config: val.combined_network_config, - commit_sha: String::new(), - builder: val.builder, - random_builder: val.random_builder, - public_keys: val.public_keys, - } - } } diff --git a/crates/types/src/traits/network.rs b/crates/types/src/traits/network.rs index 5d6670407f..3673408d22 100644 --- a/crates/types/src/traits/network.rs +++ b/crates/types/src/traits/network.rs @@ -287,7 +287,6 @@ where #[allow(clippy::type_complexity)] fn generator( expected_node_count: usize, - num_bootstrap: usize, network_id: usize, da_committee_size: usize, reliability_config: Option>, diff --git a/crates/types/src/traits/node_implementation.rs b/crates/types/src/traits/node_implementation.rs index 77a4971e24..7dbd2e0eb2 100644 --- a/crates/types/src/traits/node_implementation.rs +++ b/crates/types/src/traits/node_implementation.rs @@ -101,7 +101,6 @@ pub trait TestableNodeImplementation: NodeImplementation /// Generate the communication channels for testing fn gen_networks( expected_node_count: usize, - num_bootstrap: usize, da_committee_size: usize, reliability_config: Option>, secondary_network_delay: Duration, @@ -143,14 +142,12 @@ where fn gen_networks( expected_node_count: usize, - num_bootstrap: usize, da_committee_size: usize, reliability_config: Option>, secondary_network_delay: Duration, ) -> AsyncGenerator> { >::generator( expected_node_count, - num_bootstrap, 0, da_committee_size, reliability_config.clone(), From 51c892d25d1120adb8dead27689a676bcb5f6a25 Mon Sep 17 00:00:00 2001 From: Rob Date: Wed, 4 Dec 2024 17:27:41 -0500 Subject: [PATCH 02/16] remove example-related from CI --- .github/workflows/build-and-test.yml | 235 --------------------------- justfile | 3 - 2 files changed, 238 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index d8836d871d..cc8924233e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -52,239 +52,4 @@ jobs: env: RUST_BACKTRACE: full - test-examples: - strategy: - fail-fast: false - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - name: Checkout Repository - - - name: Install Rust - uses: mkroening/rust-toolchain-toml@main - - - uses: Swatinem/rust-cache@v2 - name: Enable Rust Caching - with: - shared-key: "examples" - cache-on-failure: "true" - save-if: ${{ github.ref == 'refs/heads/main' }} - - - uses: taiki-e/install-action@just - - - name: Test examples - run: | - just example all-push-cdn -- --config_file ./crates/orchestrator/run-config.toml - timeout-minutes: 20 - - build-release: - strategy: - fail-fast: false - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - name: Checkout Repository - - - name: Install Rust - uses: mkroening/rust-toolchain-toml@main - - - uses: Swatinem/rust-cache@v2 - name: Enable Rust Caching - with: - shared-key: "build-release" - cache-on-failure: "true" - save-if: ${{ github.ref == 'refs/heads/main' }} - - - uses: taiki-e/install-action@just - - - name: Build examples in release mode - run: just build_release --examples --package hotshot-examples --no-default-features - - - name: Upload Binaries - uses: actions/upload-artifact@v4 - with: - name: binaries-amd64 - path: | - target/release/examples/counter - target/release/examples/multi-validator-libp2p - target/release/examples/validator-libp2p - target/release/examples/validator-combined - target/release/examples/validator-push-cdn - target/release/examples/orchestrator - target/release/examples/cdn-broker - target/release/examples/cdn-marshal - - build-arm-release: - strategy: - fail-fast: false - runs-on: buildjet-4vcpu-ubuntu-2204-arm - if: ${{ github.ref == 'refs/heads/main' }} - container: ghcr.io/espressosystems/devops-rust:stable - steps: - - uses: actions/checkout@v4 - name: Checkout Repository - - uses: Swatinem/rust-cache@v2 - name: Enable Rust Caching - with: - shared-key: "build-arm-release" - cache-on-failure: "true" - save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Build examples in release mode - run: just build_release --examples --package hotshot-examples --no-default-features - - - name: Upload Binaries - uses: actions/upload-artifact@v4 - with: - name: binaries-aarch64 - path: | - target/release/examples/counter - target/release/examples/multi-validator-libp2p - target/release/examples/validator-libp2p - target/release/examples/validator-combined - target/release/examples/validator-push-cdn - target/release/examples/orchestrator - target/release/examples/cdn-broker - target/release/examples/cdn-marshal - - build-dockers: - strategy: - fail-fast: false - runs-on: ubuntu-latest - if: ${{ github.ref == 'refs/heads/main' }} - needs: [build-release, build-arm-release, test] - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Setup Docker BuildKit (buildx) - uses: docker/setup-buildx-action@v3 - - - name: Login to Github Container Repo - uses: docker/login-action@v3 - if: github.event_name != 'pull_request' - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Download AMD executables - uses: actions/download-artifact@v4 - with: - name: binaries-amd64 - path: target/amd64/release/examples - - - name: Download ARM executables - uses: actions/download-artifact@v4 - with: - name: binaries-aarch64 - path: target/arm64/release/examples - - - name: Generate validator-libp2p docker metadata - uses: docker/metadata-action@v5 - id: validator-libp2p - with: - images: ghcr.io/espressosystems/hotshot/validator-libp2p - - - name: Generate validator-combined docker metadata - uses: docker/metadata-action@v5 - id: validator-combined - with: - images: ghcr.io/espressosystems/hotshot/validator-combined - - - name: Generate validator-push-cdn docker metadata - uses: docker/metadata-action@v5 - id: validator-push-cdn - with: - images: ghcr.io/espressosystems/hotshot/validator-push-cdn - - - name: Generate orchestrator docker metadata - uses: docker/metadata-action@v5 - id: orchestrator - with: - images: ghcr.io/espressosystems/hotshot/orchestrator - - - name: Generate cdn-broker docker metadata - uses: docker/metadata-action@v5 - id: cdn-broker - with: - images: ghcr.io/espressosystems/hotshot/cdn-broker - - - name: Generate cdn-marshal docker metadata - uses: docker/metadata-action@v5 - id: cdn-marshal - with: - images: ghcr.io/espressosystems/hotshot/cdn-marshal - - - name: Build and push validator-libp2p docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/validator-libp2p.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.validator-libp2p.outputs.tags }} - labels: ${{ steps.validator-libp2p.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push validator-combined docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/validator-combined.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.validator-combined.outputs.tags }} - labels: ${{ steps.validator-combined.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push validator-push-cdn docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/validator-cdn.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.validator-push-cdn.outputs.tags }} - labels: ${{ steps.validator-push-cdn.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push orchestrator docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/orchestrator.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.orchestrator.outputs.tags }} - labels: ${{ steps.orchestrator.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push cdn-broker docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/cdn-broker.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.cdn-broker.outputs.tags }} - labels: ${{ steps.cdn-broker.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push cdn-marshal docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/cdn-marshal.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.cdn-marshal.outputs.tags }} - labels: ${{ steps.cdn-marshal.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/justfile b/justfile index a48c18021c..54e036705f 100644 --- a/justfile +++ b/justfile @@ -10,9 +10,6 @@ build: build_release *ARGS: cargo build --profile=release {{ARGS}} -example *ARGS: - cargo run --profile=release-lto --package hotshot-examples --no-default-features --example {{ARGS}} - example_fixed_leader *ARGS: cargo run --features "fixed-leader-election" --profile=release-lto --example {{ARGS}} From 17509d89e07c2fb9e34761bbb390c590b2d3f7d8 Mon Sep 17 00:00:00 2001 From: Rob Date: Wed, 4 Dec 2024 17:30:23 -0500 Subject: [PATCH 03/16] fix fmt --- justfile | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/justfile b/justfile index 54e036705f..de5feb3946 100644 --- a/justfile +++ b/justfile @@ -10,12 +10,6 @@ build: build_release *ARGS: cargo build --profile=release {{ARGS}} -example_fixed_leader *ARGS: - cargo run --features "fixed-leader-election" --profile=release-lto --example {{ARGS}} - -example_gpuvid_leader *ARGS: - cargo run --features "fixed-leader-election, gpu-vid" --profile=release-lto --example {{ARGS}} - test *ARGS: echo Testing {{ARGS}} cargo test --lib --bins --tests --benches --workspace --no-fail-fast {{ARGS}} -- --test-threads=1 --nocapture --skip crypto_test @@ -145,13 +139,11 @@ clippy_release: fmt: echo Running cargo fmt - cargo fmt -- crates/**/*.rs - cargo fmt -- crates/**/tests/**/**.rs + cargo fmt fmt_check: echo Running cargo fmt --check - cargo fmt --check -- crates/**/*.rs - cargo fmt --check -- crates/**/tests/**/**.rs + cargo fmt --check lint: clippy fmt_check From bda12647243b7649f91475d01523a5634f6503b7 Mon Sep 17 00:00:00 2001 From: Rob Date: Wed, 4 Dec 2024 19:05:07 -0500 Subject: [PATCH 04/16] give tests random IDs --- crates/hotshot/src/traits/networking/combined_network.rs | 6 +++--- crates/hotshot/src/traits/networking/libp2p_network.rs | 5 +++-- crates/hotshot/src/traits/networking/memory_network.rs | 2 +- crates/hotshot/src/traits/networking/push_cdn_network.rs | 2 +- crates/types/src/traits/network.rs | 2 +- crates/types/src/traits/node_implementation.rs | 4 +++- 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/hotshot/src/traits/networking/combined_network.rs b/crates/hotshot/src/traits/networking/combined_network.rs index d7b3c0d6a6..0639dc2f8f 100644 --- a/crates/hotshot/src/traits/networking/combined_network.rs +++ b/crates/hotshot/src/traits/networking/combined_network.rs @@ -255,7 +255,7 @@ pub struct UnderlyingCombinedNetworks( impl TestableNetworkingImplementation for CombinedNetworks { fn generator( expected_node_count: usize, - network_id: usize, + test_id: usize, da_committee_size: usize, reliability_config: Option>, secondary_network_delay: Duration, @@ -263,14 +263,14 @@ impl TestableNetworkingImplementation for CombinedNetwor let generators = ( as TestableNetworkingImplementation>::generator( expected_node_count, - network_id, + test_id, da_committee_size, None, Duration::default(), ), as TestableNetworkingImplementation>::generator( expected_node_count, - network_id, + test_id, da_committee_size, reliability_config, Duration::default(), diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index ac40cabca5..5fabe7cecb 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -203,7 +203,7 @@ impl TestableNetworkingImplementation for Libp2pNetwork { #[allow(clippy::panic, clippy::too_many_lines)] fn generator( expected_node_count: usize, - _network_id: usize, + test_id: usize, da_committee_size: usize, reliability_config: Option>, _secondary_network_delay: Duration, @@ -221,7 +221,8 @@ impl TestableNetworkingImplementation for Libp2pNetwork { // Create the bind address let bind_address = - Multiaddr::from_str(&format!("/ip4/127.0.0.1/udp/{port}/quic-v1")).unwrap(); + Multiaddr::from_str(&format!("/ip4/127.0.{test_id}.1/udp/{port}/quic-v1")) + .unwrap(); // Deterministically generate the private key from the node ID let hotshot_private_key = diff --git a/crates/hotshot/src/traits/networking/memory_network.rs b/crates/hotshot/src/traits/networking/memory_network.rs index 7c7665b051..7b11d58be4 100644 --- a/crates/hotshot/src/traits/networking/memory_network.rs +++ b/crates/hotshot/src/traits/networking/memory_network.rs @@ -180,7 +180,7 @@ impl TestableNetworkingImplementation { fn generator( _expected_node_count: usize, - _network_id: usize, + _test_id: usize, da_committee_size: usize, reliability_config: Option>, _secondary_network_delay: Duration, diff --git a/crates/hotshot/src/traits/networking/push_cdn_network.rs b/crates/hotshot/src/traits/networking/push_cdn_network.rs index 9f7391ba9e..9681d8104a 100644 --- a/crates/hotshot/src/traits/networking/push_cdn_network.rs +++ b/crates/hotshot/src/traits/networking/push_cdn_network.rs @@ -281,7 +281,7 @@ impl TestableNetworkingImplementation #[allow(clippy::too_many_lines)] fn generator( _expected_node_count: usize, - _network_id: usize, + _test_id: usize, da_committee_size: usize, _reliability_config: Option>, _secondary_network_delay: Duration, diff --git a/crates/types/src/traits/network.rs b/crates/types/src/traits/network.rs index 3673408d22..c6b4ea673c 100644 --- a/crates/types/src/traits/network.rs +++ b/crates/types/src/traits/network.rs @@ -287,7 +287,7 @@ where #[allow(clippy::type_complexity)] fn generator( expected_node_count: usize, - network_id: usize, + test_id: usize, da_committee_size: usize, reliability_config: Option>, secondary_network_delay: Duration, diff --git a/crates/types/src/traits/node_implementation.rs b/crates/types/src/traits/node_implementation.rs index 7dbd2e0eb2..8489ef41af 100644 --- a/crates/types/src/traits/node_implementation.rs +++ b/crates/types/src/traits/node_implementation.rs @@ -19,6 +19,7 @@ use std::{ use async_trait::async_trait; use committable::Committable; +use rand::Rng; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use url::Url; use vbs::version::StaticVersionType; @@ -148,7 +149,8 @@ where ) -> AsyncGenerator> { >::generator( expected_node_count, - 0, + // Generate a unique-ish test id between 0 and 255 + rand::thread_rng().gen_range(0..255), da_committee_size, reliability_config.clone(), secondary_network_delay, From 37dcb0d452729e14bb4f14348ce679a4f98d0e66 Mon Sep 17 00:00:00 2001 From: Rob Date: Wed, 4 Dec 2024 19:52:29 -0500 Subject: [PATCH 05/16] add test ID to DHT --- crates/hotshot/src/traits/networking/libp2p_network.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index 5fabe7cecb..dcdf402ef1 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -247,7 +247,7 @@ impl TestableNetworkingImplementation for Libp2pNetwork { replication_factor: 2 * expected_node_count / 3, record_ttl: None, publication_interval: None, - file_path: format!("/tmp/libp2p_dht-{node_id}.bin"), + file_path: format!("/tmp/libp2p_dht-{test_id}-{node_id}.bin"), lookup_record_value, }; From e7b4251074d0e23aadc1e9a4beacec90286af223 Mon Sep 17 00:00:00 2001 From: Rob Date: Wed, 4 Dec 2024 20:04:54 -0500 Subject: [PATCH 06/16] fix test id --- crates/hotshot/src/traits/networking/libp2p_network.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index dcdf402ef1..986e9e1543 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -57,6 +57,7 @@ use libp2p_networking::{ }, reexport::Multiaddr, }; +use rand::Rng; use serde::Serialize; use tokio::{ select, spawn, @@ -221,7 +222,7 @@ impl TestableNetworkingImplementation for Libp2pNetwork { // Create the bind address let bind_address = - Multiaddr::from_str(&format!("/ip4/127.0.{test_id}.1/udp/{port}/quic-v1")) + Multiaddr::from_str(&format!("/ip4/127.0.0.{test_id}/udp/{port}/quic-v1")) .unwrap(); // Deterministically generate the private key from the node ID @@ -241,13 +242,16 @@ impl TestableNetworkingImplementation for Libp2pNetwork { ) .expect("Failed to sign DHT lookup record"); + // Create a random file path for the DHT database + let dht_file_path = format!("/tmp/libp2p_dht-{}.bin", rand::thread_rng().gen::()); + // Configure Kademlia with our lookup record value let kademlia_config = KademliaConfig:: { // At least 2/3 of the nodes should have any given record replication_factor: 2 * expected_node_count / 3, record_ttl: None, publication_interval: None, - file_path: format!("/tmp/libp2p_dht-{test_id}-{node_id}.bin"), + file_path: dht_file_path, lookup_record_value, }; From 9fac4f41879beadd5807b78bb304456d5f3d5d26 Mon Sep 17 00:00:00 2001 From: Rob Date: Wed, 4 Dec 2024 20:09:03 -0500 Subject: [PATCH 07/16] fmt --- crates/hotshot/src/traits/networking/libp2p_network.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index 986e9e1543..94acc37d20 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -243,7 +243,8 @@ impl TestableNetworkingImplementation for Libp2pNetwork { .expect("Failed to sign DHT lookup record"); // Create a random file path for the DHT database - let dht_file_path = format!("/tmp/libp2p_dht-{}.bin", rand::thread_rng().gen::()); + let dht_file_path = + format!("/tmp/libp2p_dht-{}.bin", rand::thread_rng().gen::()); // Configure Kademlia with our lookup record value let kademlia_config = KademliaConfig:: { From 48a98d691a92750093a15eb85c4a48ab5878e54b Mon Sep 17 00:00:00 2001 From: Rob Date: Thu, 5 Dec 2024 10:31:25 -0500 Subject: [PATCH 08/16] potentially fix Libp2p restart test --- .../src/traits/networking/combined_network.rs | 3 -- .../src/traits/networking/libp2p_network.rs | 38 ++++++++++--------- .../src/traits/networking/memory_network.rs | 1 - .../src/traits/networking/push_cdn_network.rs | 1 - crates/types/src/traits/network.rs | 1 - .../types/src/traits/node_implementation.rs | 3 -- 6 files changed, 21 insertions(+), 26 deletions(-) diff --git a/crates/hotshot/src/traits/networking/combined_network.rs b/crates/hotshot/src/traits/networking/combined_network.rs index 0639dc2f8f..f73c016ed4 100644 --- a/crates/hotshot/src/traits/networking/combined_network.rs +++ b/crates/hotshot/src/traits/networking/combined_network.rs @@ -255,7 +255,6 @@ pub struct UnderlyingCombinedNetworks( impl TestableNetworkingImplementation for CombinedNetworks { fn generator( expected_node_count: usize, - test_id: usize, da_committee_size: usize, reliability_config: Option>, secondary_network_delay: Duration, @@ -263,14 +262,12 @@ impl TestableNetworkingImplementation for CombinedNetwor let generators = ( as TestableNetworkingImplementation>::generator( expected_node_count, - test_id, da_committee_size, None, Duration::default(), ), as TestableNetworkingImplementation>::generator( expected_node_count, - test_id, da_committee_size, reliability_config, Duration::default(), diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index 94acc37d20..81b0f74688 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -172,7 +172,7 @@ pub struct Libp2pNetwork { } /// Generate the expected [testing] multiaddr from a node index -fn multiaddr_from_node_index(i: usize) -> Multiaddr { +fn multiaddr_from_node_index(i: usize, bind_addresses: &[Multiaddr]) -> Multiaddr { // Generate the node's private key from the node ID let peers_hotshot_private_key = T::SignatureKey::generated_from_seed_indexed([0u8; 32], i as u64).1; @@ -182,12 +182,10 @@ fn multiaddr_from_node_index(i: usize) -> Multiaddr { .expect("Failed to derive libp2p keypair"); // Generate the multiaddress using the peer id and port - Multiaddr::from_str(&format!( - "/ip4/127.0.0.1/udp/{}/quic-v1/p2p/{}", - 48000 + i, - peers_libp2p_keypair.public().to_peer_id() - )) - .expect("Failed to create multiaddr") + bind_addresses[i] + .clone() + .with_p2p(peers_libp2p_keypair.public().to_peer_id()) + .expect("Failed to append libp2p peer id to multiaddr") } #[cfg(feature = "hotshot-testing")] @@ -204,7 +202,6 @@ impl TestableNetworkingImplementation for Libp2pNetwork { #[allow(clippy::panic, clippy::too_many_lines)] fn generator( expected_node_count: usize, - test_id: usize, da_committee_size: usize, reliability_config: Option>, _secondary_network_delay: Duration, @@ -214,16 +211,23 @@ impl TestableNetworkingImplementation for Libp2pNetwork { "DA committee size must be less than or equal to total # nodes" ); + // Generate the bind addresses each node will use + let mut bind_addresses = Vec::new(); + for _ in 0..expected_node_count { + bind_addresses.push( + Multiaddr::from_str(&format!( + "/ip4/127.0.0.1/udp/{}/quic-v1", + portpicker::pick_unused_port().expect("All ports are in use") + )) + .expect("Failed to create multiaddr"), + ); + } + // NOTE uncomment this for easier debugging Box::pin({ move |node_id| { - // The port is 48000 + the node id. This way it's deterministic and easy to connect nodes together - let port = 48000 + node_id; - - // Create the bind address - let bind_address = - Multiaddr::from_str(&format!("/ip4/127.0.0.{test_id}/udp/{port}/quic-v1")) - .unwrap(); + // Get our bind address from the list + let bind_address = bind_addresses[usize::try_from(node_id).unwrap()].clone(); // Deterministically generate the private key from the node ID let hotshot_private_key = @@ -272,7 +276,7 @@ impl TestableNetworkingImplementation for Libp2pNetwork { // Create the list of known peers let known_peers = (0..expected_node_count) - .map(|i| multiaddr_from_node_index::(i)) + .map(|i| multiaddr_from_node_index::(i, &bind_addresses)) .collect(); // Collect the `PeerConfig`s of all nodes @@ -434,7 +438,7 @@ impl Libp2pNetwork { // Spawn the network node with a copy of our config let (mut rx, network_handle) = spawn_network_node::(config.clone()) .await - .map_err(|e| NetworkError::ConfigError(format!("failed to spawn network node: {e}")))?; + .with_context(|| "failed to spawn network node")?; // Subscribe only to the global topic (DA messages are direct-broadcasted) let subscribed_topics = HashSet::from_iter(vec![GLOBAL_TOPIC.to_string()]); diff --git a/crates/hotshot/src/traits/networking/memory_network.rs b/crates/hotshot/src/traits/networking/memory_network.rs index 7b11d58be4..297f116430 100644 --- a/crates/hotshot/src/traits/networking/memory_network.rs +++ b/crates/hotshot/src/traits/networking/memory_network.rs @@ -180,7 +180,6 @@ impl TestableNetworkingImplementation { fn generator( _expected_node_count: usize, - _test_id: usize, da_committee_size: usize, reliability_config: Option>, _secondary_network_delay: Duration, diff --git a/crates/hotshot/src/traits/networking/push_cdn_network.rs b/crates/hotshot/src/traits/networking/push_cdn_network.rs index 9681d8104a..f3a068530f 100644 --- a/crates/hotshot/src/traits/networking/push_cdn_network.rs +++ b/crates/hotshot/src/traits/networking/push_cdn_network.rs @@ -281,7 +281,6 @@ impl TestableNetworkingImplementation #[allow(clippy::too_many_lines)] fn generator( _expected_node_count: usize, - _test_id: usize, da_committee_size: usize, _reliability_config: Option>, _secondary_network_delay: Duration, diff --git a/crates/types/src/traits/network.rs b/crates/types/src/traits/network.rs index c6b4ea673c..1211cc93e2 100644 --- a/crates/types/src/traits/network.rs +++ b/crates/types/src/traits/network.rs @@ -287,7 +287,6 @@ where #[allow(clippy::type_complexity)] fn generator( expected_node_count: usize, - test_id: usize, da_committee_size: usize, reliability_config: Option>, secondary_network_delay: Duration, diff --git a/crates/types/src/traits/node_implementation.rs b/crates/types/src/traits/node_implementation.rs index 8489ef41af..49b814672a 100644 --- a/crates/types/src/traits/node_implementation.rs +++ b/crates/types/src/traits/node_implementation.rs @@ -19,7 +19,6 @@ use std::{ use async_trait::async_trait; use committable::Committable; -use rand::Rng; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use url::Url; use vbs::version::StaticVersionType; @@ -149,8 +148,6 @@ where ) -> AsyncGenerator> { >::generator( expected_node_count, - // Generate a unique-ish test id between 0 and 255 - rand::thread_rng().gen_range(0..255), da_committee_size, reliability_config.clone(), secondary_network_delay, From caace83da570cbbb6976518d23cbb073f6880561 Mon Sep 17 00:00:00 2001 From: Rob Date: Thu, 5 Dec 2024 11:57:33 -0500 Subject: [PATCH 09/16] pick a random port on restart --- .../src/traits/networking/libp2p_network.rs | 43 ++++++++++++------- crates/testing/src/spinning_task.rs | 3 +- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index 81b0f74688..e37294856a 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -10,7 +10,7 @@ #[cfg(feature = "hotshot-testing")] use std::str::FromStr; use std::{ - collections::HashSet, + collections::{HashMap, HashSet}, fmt::Debug, net::{IpAddr, ToSocketAddrs}, sync::{ @@ -171,8 +171,20 @@ pub struct Libp2pNetwork { inner: Arc>, } +/// Generate a [testing] multiaddr +fn random_multiaddr() -> Multiaddr { + Multiaddr::from_str(&format!( + "/ip4/127.0.0.1/udp/{}/quic-v1", + portpicker::pick_unused_port().expect("All ports are in use") + )) + .expect("Failed to create multiaddr") +} + /// Generate the expected [testing] multiaddr from a node index -fn multiaddr_from_node_index(i: usize, bind_addresses: &[Multiaddr]) -> Multiaddr { +fn multiaddr_from_node_index( + i: usize, + bind_addresses: &HashMap, +) -> Multiaddr { // Generate the node's private key from the node ID let peers_hotshot_private_key = T::SignatureKey::generated_from_seed_indexed([0u8; 32], i as u64).1; @@ -182,7 +194,7 @@ fn multiaddr_from_node_index(i: usize, bind_addresses: &[Multiaddr] .expect("Failed to derive libp2p keypair"); // Generate the multiaddress using the peer id and port - bind_addresses[i] + bind_addresses[&i] .clone() .with_p2p(peers_libp2p_keypair.public().to_peer_id()) .expect("Failed to append libp2p peer id to multiaddr") @@ -212,22 +224,23 @@ impl TestableNetworkingImplementation for Libp2pNetwork { ); // Generate the bind addresses each node will use - let mut bind_addresses = Vec::new(); - for _ in 0..expected_node_count { - bind_addresses.push( - Multiaddr::from_str(&format!( - "/ip4/127.0.0.1/udp/{}/quic-v1", - portpicker::pick_unused_port().expect("All ports are in use") - )) - .expect("Failed to create multiaddr"), - ); + let bind_addresses = Arc::new(parking_lot::Mutex::new(HashMap::new())); + for i in 0..expected_node_count { + // Insert a random port for each node + bind_addresses.lock().insert(i, random_multiaddr()); } // NOTE uncomment this for easier debugging Box::pin({ move |node_id| { - // Get our bind address from the list - let bind_address = bind_addresses[usize::try_from(node_id).unwrap()].clone(); + // Get and replace our bind address with a newly random port. + // We need this because Libp2p does not release the listener as soon + // as we tell it to. + let mut bind_addresses_lock = bind_addresses.lock(); + let bind_address = bind_addresses_lock + .insert(usize::try_from(node_id).unwrap(), random_multiaddr()) + .unwrap(); + drop(bind_addresses_lock); // Deterministically generate the private key from the node ID let hotshot_private_key = @@ -276,7 +289,7 @@ impl TestableNetworkingImplementation for Libp2pNetwork { // Create the list of known peers let known_peers = (0..expected_node_count) - .map(|i| multiaddr_from_node_index::(i, &bind_addresses)) + .map(|i| multiaddr_from_node_index::(i, &bind_addresses.lock().clone())) .collect(); // Collect the `PeerConfig`s of all nodes diff --git a/crates/testing/src/spinning_task.rs b/crates/testing/src/spinning_task.rs index 4382f12ce0..faee282c32 100644 --- a/crates/testing/src/spinning_task.rs +++ b/crates/testing/src/spinning_task.rs @@ -215,8 +215,8 @@ where NodeAction::RestartDown(delay_views) => { let node_id = idx.try_into().unwrap(); if let Some(node) = self.handles.write().await.get_mut(idx) { - tracing::error!("Node {} shutting down", idx); node.handle.shut_down().await; + // For restarted nodes generate the network on correct view let generated_network = (self.channel_generator)(node_id).await; @@ -263,6 +263,7 @@ where node_id < config.da_staked_committee_size as u64, ); let internal_chan = broadcast(EVENT_CHANNEL_SIZE); + let context = TestRunner::::add_node_with_config_and_channels( node_id, From 9765bb6de4286b8cc96b6935c367616675a7b222 Mon Sep 17 00:00:00 2001 From: Rob Date: Thu, 5 Dec 2024 17:38:15 -0500 Subject: [PATCH 10/16] re-add examples, remove node id --- Cargo.lock | 27 +- config/ValidatorConfigExample | 32 - config/ValidatorConfigFile.toml | 3 - crates/examples/Cargo.toml | 19 +- crates/examples/src/main.rs | 622 ++++++++++++++++++ crates/hotshot/src/lib.rs | 30 +- crates/hotshot/src/tasks/mod.rs | 3 - crates/hotshot/src/tasks/task_state.rs | 12 +- .../traits/election/randomized_committee.rs | 3 +- .../election/randomized_committee_members.rs | 3 +- .../src/traits/election/static_committee.rs | 3 +- .../static_committee_leader_two_views.rs | 3 +- .../src/traits/networking/libp2p_network.rs | 20 +- .../src/traits/networking/push_cdn_network.rs | 10 +- crates/hotshot/src/types/handle.rs | 4 +- crates/task-impls/src/consensus/handlers.rs | 2 - crates/task-impls/src/consensus/mod.rs | 5 +- crates/task-impls/src/da.rs | 6 +- crates/task-impls/src/helpers.rs | 2 +- .../src/quorum_proposal/handlers.rs | 5 +- crates/task-impls/src/quorum_proposal/mod.rs | 12 +- .../src/quorum_proposal_recv/mod.rs | 8 +- crates/task-impls/src/quorum_vote/handlers.rs | 2 +- crates/task-impls/src/quorum_vote/mod.rs | 16 +- crates/task-impls/src/request.rs | 4 +- crates/task-impls/src/response.rs | 6 +- crates/task-impls/src/rewind.rs | 10 +- crates/task-impls/src/transactions.rs | 17 +- crates/task-impls/src/upgrade.rs | 6 +- crates/task-impls/src/vid.rs | 5 +- crates/task-impls/src/view_sync.rs | 16 +- crates/task-impls/src/vote_collection.rs | 8 - crates/testing/src/helpers.rs | 9 +- crates/testing/src/spinning_task.rs | 6 +- crates/testing/src/test_builder.rs | 14 +- crates/testing/src/test_runner.rs | 21 +- crates/testing/tests/tests_1/gen_key_pair.rs | 43 -- crates/testing/tests/tests_1/network_task.rs | 4 +- .../tests/tests_1/vote_dependency_handle.rs | 1 - crates/types/src/hotshot_config_file.rs | 39 +- crates/types/src/lib.rs | 24 +- crates/types/src/traits/consensus_api.rs | 4 +- crates/types/src/traits/election.rs | 5 +- crates/types/src/validator_config.rs | 54 -- 44 files changed, 779 insertions(+), 369 deletions(-) delete mode 100644 config/ValidatorConfigExample delete mode 100644 config/ValidatorConfigFile.toml create mode 100644 crates/examples/src/main.rs delete mode 100644 crates/testing/tests/tests_1/gen_key_pair.rs delete mode 100644 crates/types/src/validator_config.rs diff --git a/Cargo.lock b/Cargo.lock index e3fe618d73..9598cabbeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2215,10 +2215,6 @@ dependencies = [ "pin-project-lite 0.2.15", ] -[[package]] -name = "examples" -version = "0.5.79" - [[package]] name = "fastrand" version = "1.9.0" @@ -2924,6 +2920,29 @@ dependencies = [ "vbs", ] +[[package]] +name = "hotshot-examples" +version = "0.5.79" +dependencies = [ + "anyhow", + "cdn-broker", + "cdn-marshal", + "clap", + "futures", + "hotshot", + "hotshot-example-types", + "hotshot-testing", + "hotshot-types", + "libp2p", + "libp2p-networking", + "lru 0.12.5", + "portpicker", + "rand 0.8.5", + "tokio", + "tracing", + "url", +] + [[package]] name = "hotshot-fakeapi" version = "0.5.79" diff --git a/config/ValidatorConfigExample b/config/ValidatorConfigExample deleted file mode 100644 index 3170b75f9f..0000000000 --- a/config/ValidatorConfigExample +++ /dev/null @@ -1,32 +0,0 @@ -ValidatorConfig { - public_key: VerKey( - ( - QuadExtField(2264797523581107490935262917175769123227923636811928330606075281145117212394 + 15807017392833049888165434456991157794698032464874424842715555348468160607934 * u), - QuadExtField(7996517616082121122160563552650547601395271017260499735456299700133762512689 + 7504045709281061282278228438613345070383424761478787301859187055302953740948 * u), - QuadExtField(1515973040548822760825076242090160370742046237881440422068330135941139244581 + 20251846261653098602911417004145145971080304248810966341160788194007704966108 * u) - ) - ), - private_key: SignKey( - BigInt( - [3505488234151006356, 6655477166151225138, 3291219027844407676, 2153641080015542578] - ) - ), - stake_value: 1, - state_key_pair: StateKeyPair( - KeyPair { - sk: SignKey( - BigInt( - [2822822805887490846, 6664316196088353173, 4926510007447087464, 116097479308258694] - ) - ), - vk: VerKey( - Projective { - x: BigInt([11315198235793138814, 4744451806709910489, 6921831025042192557, 1125393823825936625]), - y: BigInt([13035879815613524256, 18225673961538637854, 12006860967936477969, 1516668567229692859]), - t: BigInt([13450777528397789701, 12242009376162249168, 12596256366242272750, 3368076418495976469]), - z: BigInt([10465708325245823445, 13967918689717629445, 14943426723808572731, 621075342718756551]) - } - ) - } - ) - } \ No newline at end of file diff --git a/config/ValidatorConfigFile.toml b/config/ValidatorConfigFile.toml deleted file mode 100644 index 1f96cf0eb2..0000000000 --- a/config/ValidatorConfigFile.toml +++ /dev/null @@ -1,3 +0,0 @@ -is_da = true -seed = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -node_id = 0 diff --git a/crates/examples/Cargo.toml b/crates/examples/Cargo.toml index 849ee8305f..5c15ed9330 100644 --- a/crates/examples/Cargo.toml +++ b/crates/examples/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "examples" +name = "hotshot-examples" version.workspace = true authors.workspace = true edition.workspace = true @@ -9,6 +9,23 @@ documentation.workspace = true repository.workspace = true [dependencies] +tokio.workspace = true +clap.workspace = true +hotshot = { path = "../hotshot" } +hotshot-types = { path = "../types" } +hotshot-example-types.path = "../example-types" +url.workspace = true +libp2p.workspace = true +portpicker.workspace = true +libp2p-networking.workspace = true +anyhow.workspace = true +rand.workspace = true +futures.workspace = true +tracing.workspace = true +hotshot-testing.path = "../testing" +lru.workspace = true +cdn-broker.workspace = true +cdn-marshal.workspace = true [lints] workspace = true diff --git a/crates/examples/src/main.rs b/crates/examples/src/main.rs new file mode 100644 index 0000000000..3d9e937c83 --- /dev/null +++ b/crates/examples/src/main.rs @@ -0,0 +1,622 @@ +//! This crate contains examples of HotShot usage +use std::{collections::HashMap, num::NonZero, sync::Arc, time::Duration}; + +use anyhow::{Context, Result}; +use cdn_broker::{reexports::def::hook::NoMessageHook, Broker, Config as BrokerConfig}; +use cdn_marshal::{Config as MarshalConfig, Marshal}; +use clap::Parser; +use futures::StreamExt; +use hotshot::{ + helpers::initialize_logging, + traits::{ + election::static_committee::StaticCommittee, + implementations::{ + derive_libp2p_keypair, CdnMetricsValue, CdnTopic, CombinedNetworks, KeyPair, + Libp2pMetricsValue, Libp2pNetwork, PushCdnNetwork, TestingDef, WrappedSignatureKey, + }, + }, + types::{BLSPrivKey, BLSPubKey, EventType, SignatureKey}, + MarketplaceConfig, SystemContext, +}; +use hotshot_example_types::{ + auction_results_provider_types::TestAuctionResultsProvider, + block_types::TestTransaction, + node_types::{CombinedImpl, Libp2pImpl, PushCdnImpl, TestTypes, TestVersions}, + state_types::TestInstanceState, + storage_types::TestStorage, + testable_delay::DelayConfig, +}; +use hotshot_testing::block_builder::{SimpleBuilderImplementation, TestBuilderImplementation}; +use hotshot_types::{ + consensus::ConsensusMetricsValue, + light_client::StateKeyPair, + traits::{election::Membership, node_implementation::NodeType}, + HotShotConfig, PeerConfig, +}; +use libp2p::Multiaddr; +use libp2p_networking::network::{ + behaviours::dht::record::{Namespace, RecordKey, RecordValue}, + node::config::{KademliaConfig, Libp2pConfig}, + GossipConfig, RequestResponseConfig, +}; +use lru::LruCache; +use rand::Rng; +use tokio::{spawn, sync::OnceCell, task::JoinSet}; +use tracing::{error, info}; +use url::Url; + +/// The command line arguments for the example +#[derive(Parser)] +struct Args { + /// The number of nodes to start + #[arg(long, default_value_t = 5)] + total_num_nodes: usize, + + /// The number of nodes which are DA nodes + #[arg(long, default_value_t = 3)] + num_da_nodes: usize, + + /// The number of views to run for. If not specified, it will run indefinitely + #[arg(long)] + num_views: Option, + + /// The number of transactions to submit to the builder per view + #[arg(long, default_value_t = 100)] + num_transactions_per_view: usize, + + /// The size of the transactions submitted to the builder per view + #[arg(long, default_value_t = 1000)] + transaction_size: usize, + + /// The type of network to use. Acceptable values are + /// "combined", "cdn", or "libp2p" + #[arg(long, default_value = "combined")] + network: String, +} + +/// This is a testing function which allows us to easily determine if a node should be a DA node +fn is_da_node(index: usize, num_da_nodes: usize) -> bool { + index < num_da_nodes +} + +/// This is a testing function which allows us to easily generate peer configs from indexes +fn peer_info_from_index(index: usize) -> PeerConfig { + // Get the node's public key + let (public_key, _) = BLSPubKey::generated_from_seed_indexed([0u8; 32], index as u64); + + // Generate the peer config + PeerConfig { + stake_table_entry: public_key.stake_table_entry(1), + state_ver_key: StateKeyPair::default().0.ver_key(), + } +} + +/// Generate a Libp2p multiaddress from a port +fn libp2p_multiaddress_from_index(index: usize) -> Result { + // Generate the peer's private key and derive their libp2p keypair + let (_, peer_private_key) = BLSPubKey::generated_from_seed_indexed([0u8; 32], index as u64); + let peer_libp2p_keypair = derive_libp2p_keypair::(&peer_private_key) + .with_context(|| "Failed to derive libp2p keypair")?; + + // Generate the multiaddress from the peer's port and libp2p keypair + format!( + "/ip4/127.0.0.1/udp/{}/quic-v1/p2p/{}", + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")?, + peer_libp2p_keypair.public().to_peer_id() + ) + .parse() + .with_context(|| "Failed to parse multiaddress") +} + +/// The type of network to use for the example +#[derive(Debug, PartialEq, Eq)] +enum NetworkType { + /// A combined network, which is a combination of a Libp2p and Push CDN network + Combined, + + /// A network solely using the Push CDN + Cdn, + + /// A Libp2p network + LibP2P, +} + +/// A helper function to start the CDN +/// (2 brokers + 1 marshal) +/// +/// Returns the address of the marshal +async fn start_cdn() -> Result { + // Figure out where we're going to spawn the marshal + let marshal_port = + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")?; + let marshal_address = format!("127.0.0.1:{marshal_port}"); + + // Configure the marshal + let marshal_config = MarshalConfig { + bind_endpoint: marshal_address.clone(), + discovery_endpoint: marshal_address.clone(), + ca_cert_path: None, + ca_key_path: None, + metrics_bind_endpoint: None, + global_memory_pool_size: Some(1024 * 1024 * 1024), + }; + + // Create and start the marshal + let marshal: Marshal> = Marshal::new(marshal_config) + .await + .with_context(|| "Failed to create marshal")?; + spawn(marshal.start()); + + // This keypair is shared between brokers + let (broker_public_key, broker_private_key) = + ::SignatureKey::generated_from_seed_indexed([0u8; 32], 1337); + + for _ in 0..2 { + // Generate one random port for the "public" endpoint and one for the "private" endpoint + let public_port = + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")?; + let private_port = + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")?; + + // Throw into address format + let private_address = format!("127.0.0.1:{private_port}"); + let public_address = format!("127.0.0.1:{public_port}"); + + // Configure the broker + let broker_config: BrokerConfig::SignatureKey>> = + BrokerConfig { + public_advertise_endpoint: public_address.clone(), + public_bind_endpoint: public_address, + private_advertise_endpoint: private_address.clone(), + private_bind_endpoint: private_address, + discovery_endpoint: marshal_address.clone(), + keypair: KeyPair { + public_key: WrappedSignatureKey(broker_public_key), + private_key: broker_private_key.clone(), + }, + + user_message_hook: NoMessageHook, // Don't do any message processing + broker_message_hook: NoMessageHook, + + metrics_bind_endpoint: None, + ca_cert_path: None, + ca_key_path: None, + global_memory_pool_size: Some(1024 * 1024 * 1024), + }; + + // Create and start it + let broker = Broker::new(broker_config) + .await + .with_context(|| "Failed to create broker")?; + spawn(broker.start()); + } + + Ok(marshal_address) +} + +/// A helper function to create a Libp2p network +async fn create_libp2p_network( + node_index: usize, + total_num_nodes: usize, + public_key: &BLSPubKey, + private_key: &BLSPrivKey, + known_libp2p_nodes: &[Multiaddr], +) -> Result>> { + // Derive the Libp2p keypair from the private key + let libp2p_keypair = derive_libp2p_keypair::(private_key) + .with_context(|| "Failed to derive libp2p keypair")?; + + // Sign our Libp2p lookup record value + let lookup_record_value = RecordValue::new_signed( + &RecordKey::new(Namespace::Lookup, public_key.to_bytes()), + libp2p_keypair.public().to_peer_id().to_bytes(), + private_key, + ) + .expect("Failed to sign DHT lookup record"); + + // Configure Libp2p + let libp2p_config = Libp2pConfig { + keypair: libp2p_keypair, + bind_address: known_libp2p_nodes[node_index].clone(), + known_peers: known_libp2p_nodes.to_vec(), + quorum_membership: None, // This disables stake-table authentication + auth_message: None, // This disables stake-table authentication + gossip_config: GossipConfig::default(), + request_response_config: RequestResponseConfig::default(), + kademlia_config: KademliaConfig { + replication_factor: total_num_nodes * 2 / 3, + record_ttl: None, + publication_interval: None, + file_path: format!("/tmp/kademlia-{}.db", rand::random::()), + lookup_record_value, + }, + }; + + // Create the network with the config + Ok(Arc::new( + Libp2pNetwork::new( + libp2p_config, + public_key, + Libp2pMetricsValue::default(), + None, + ) + .await + .with_context(|| "Failed to create libp2p network")?, + )) +} + +/// Create a Push CDN network, starting the CDN if it's not already running +async fn create_cdn_network( + is_da_node: bool, + public_key: &BLSPubKey, + private_key: &BLSPrivKey, +) -> Result>> { + /// Create a static cell to store the marshal's endpoint + static MARSHAL_ENDPOINT: OnceCell = OnceCell::const_new(); + + // If the marshal endpoint isn't already set, start the CDN and set the endpoint + let marshal_endpoint = MARSHAL_ENDPOINT + .get_or_init(|| async { start_cdn().await.expect("Failed to start CDN") }) + .await + .clone(); + + // Subscribe to topics based on whether we're a DA node or not + let mut topics = vec![CdnTopic::Global]; + if is_da_node { + topics.push(CdnTopic::Da); + } + + // Create and return the network + Ok(Arc::new( + PushCdnNetwork::new( + marshal_endpoint, + topics, + KeyPair { + public_key: WrappedSignatureKey(*public_key), + private_key: private_key.clone(), + }, + CdnMetricsValue::default(), + ) + .with_context(|| "Failed to create Push CDN network")?, + )) +} + +/// A helper function to create a Combined network, which is a combination of a Libp2p and Push CDN network +async fn create_combined_network( + node_index: usize, + total_num_nodes: usize, + known_libp2p_nodes: &[Multiaddr], + is_da_node: bool, + public_key: &BLSPubKey, + private_key: &BLSPrivKey, +) -> Result>> { + // Create the CDN network + let cdn_network = + Arc::into_inner(create_cdn_network(is_da_node, public_key, private_key).await?).unwrap(); + + // Create the Libp2p network + let libp2p_network = Arc::into_inner( + create_libp2p_network( + node_index, + total_num_nodes, + public_key, + private_key, + known_libp2p_nodes, + ) + .await?, + ) + .unwrap(); + + // Create and return the combined network + Ok(Arc::new(CombinedNetworks::new( + cdn_network, + libp2p_network, + Some(Duration::from_secs(1)), + ))) +} + +/// A macro to start a node with a given network. We need this because `NodeImplementation` requires we +/// define a bunch of traits, which we can't do in a generic context. +macro_rules! start_node_with_network { + ($network_type:ident, $node_index:expr, $total_num_nodes:expr, $public_key:expr, $private_key:expr, $config:expr, $memberships:expr, $hotshot_initializer:expr, $network:expr, $join_set:expr, $num_transactions_per_view:expr, $transaction_size:expr, $num_views:expr, $builder_url:expr) => { + // Create the marketplace config + let marketplace_config = MarketplaceConfig::<_, $network_type> { + auction_results_provider: TestAuctionResultsProvider::::default().into(), + // TODO: we need to pass a valid fallback builder url here somehow + fallback_builder_url: Url::parse("http://localhost:8080").unwrap(), + }; + + // Initialize the system context + let handle = SystemContext::::init( + $public_key.clone(), + $private_key.clone(), + $config.clone(), + $memberships.clone(), + $network, + $hotshot_initializer, + ConsensusMetricsValue::default(), + TestStorage::::default(), + marketplace_config, + ) + .await + .with_context(|| "Failed to initialize system context")? + .0; + + // If the node index is 0, create and start the builder, which sips off events from HotShot + if $node_index == 0 { + // Create it + let builder_handle = + >::start( + $total_num_nodes, + $builder_url.clone(), + (), + HashMap::new(), + ) + .await; + + // Start it + builder_handle.start(Box::new(handle.event_stream())); + } + + // Start consensus + handle.hotshot.start_consensus().await; + + // Create an LRU cache to store the size of the proposed block if we're DA + let mut proposed_block_size_cache = LruCache::new(NonZero::new(100).unwrap()); + + // Spawn the task into the join set + $join_set.spawn(async move { + // Get the event stream for this particular node + let mut event_stream = handle.event_stream(); + + // Wait for a `Decide` event for the view number we requested + loop { + // Get the next event + let event = event_stream.next().await.unwrap(); + + // DA proposals contain the full list of transactions. We can use this to cache + // the size of the proposed block + if let EventType::DaProposal { proposal, .. } = event.event { + // Insert the size of the proposed block into the cache + proposed_block_size_cache.put( + *proposal.data.view_number, + proposal.data.encoded_transactions.len(), + ); + + // A `Decide` event contains data that HotShot has decided on + } else if let EventType::Decide { qc, .. } = event.event { + // If we cached the size of the proposed block, log it + if let Some(size) = proposed_block_size_cache.get(&*qc.view_number) { + info!( + "Decided on view number {}. Block size was {}", + qc.view_number, size + ); + } else { + info!("Decided on view number {}.", qc.view_number); + } + + // If the view number is divisible by the node's index, submit transactions + if $node_index == 0 { + // Generate and submit the requested number of transactions + for _ in 0..$num_transactions_per_view { + // Generate a random transaction + let mut transaction_bytes = vec![0u8; $transaction_size]; + rand::thread_rng().fill(&mut transaction_bytes[..]); + + // Submit the transaction + if let Err(err) = handle + .submit_transaction(TestTransaction::new(transaction_bytes)) + .await + { + error!("Failed to submit transaction: {:?}", err); + }; + } + } + + // If we have a specific view number we want to wait for, check if we've reached it + if let Some(num_views) = $num_views { + if *qc.view_number == num_views as u64 { + // Break when we've decided on the view number we requested + break; + } + } + } + } + }); + }; +} + +#[tokio::main] +#[allow(clippy::too_many_lines)] +async fn main() -> Result<()> { + // Initialize logging + initialize_logging(); + + // Parse the command line arguments + let args = Args::parse(); + + // Match the network type + let network_type = match args.network.to_lowercase().as_str() { + "combined" => NetworkType::Combined, + "cdn" => NetworkType::Cdn, + "libp2p" => NetworkType::LibP2P, + _ => { + anyhow::bail!("Invalid network type. Please use one of 'combined', 'cdn', or 'libp2p'.") + } + }; + + // Generate the builder URL we plan to use + let builder_url = Url::parse( + format!( + "http://localhost:{}", + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")? + ) + .as_str(), + ) + .with_context(|| "Failed to parse builder URL")?; + + // Create the `known_nodes` and `known_da_nodes` + let known_nodes: Vec> = (0..args.total_num_nodes) + .map(peer_info_from_index) + .collect(); + let known_da_nodes: Vec> = (0..args.num_da_nodes) + .filter(|index| is_da_node(*index, args.num_da_nodes)) + .map(peer_info_from_index) + .collect(); + + // If the network type is "Libp2p" or "Combined", we need to also assign the list of + // Libp2p addresses to be used + let mut known_libp2p_nodes = Vec::new(); + if network_type == NetworkType::LibP2P || network_type == NetworkType::Combined { + for index in 0..args.total_num_nodes { + // Generate a Libp2p multiaddress from a random, unused port + let addr = libp2p_multiaddress_from_index(index) + .with_context(|| "Failed to generate multiaddress")?; + known_libp2p_nodes.push(addr); + } + } + + // Create the memberships from the known nodes and known da nodes + let memberships = StaticCommittee::new(known_nodes.clone(), known_da_nodes.clone()); + + // Create a `JoinSet` composed of all handles + let mut join_set = JoinSet::new(); + + // Spawn each node + for index in 0..args.total_num_nodes { + // Create a new instance state + let instance_state = TestInstanceState::new(DelayConfig::default()); + + // Initialize HotShot from genesis + let hotshot_initializer = + hotshot::HotShotInitializer::::from_genesis::(instance_state) + .await + .with_context(|| "Failed to initialize HotShot")?; + + // Create our own keypair + let (public_key, private_key) = + BLSPubKey::generated_from_seed_indexed([0u8; 32], index as u64); + + // Configure HotShot + let config = HotShotConfig:: { + known_nodes: known_nodes.clone(), + known_da_nodes: known_da_nodes.clone(), + next_view_timeout: 5000, + fixed_leader_for_gpuvid: 0, // This means that we don't have a fixed leader for testing GPU VID + view_sync_timeout: Duration::from_secs(5), + builder_timeout: Duration::from_secs(1), + data_request_delay: Duration::from_millis(200), + builder_urls: vec![builder_url.clone()], + start_proposing_view: u64::MAX, // These just mean the upgrade functionality is disabled + stop_proposing_view: u64::MAX, + start_voting_view: u64::MAX, + stop_voting_view: u64::MAX, + start_proposing_time: u64::MAX, + stop_proposing_time: u64::MAX, + start_voting_time: u64::MAX, + stop_voting_time: u64::MAX, + epoch_height: 0, // This just means epochs aren't enabled + }; + + // Create a network and start HotShot based on the network type + match network_type { + NetworkType::Combined => { + // Create the combined network + let network = create_combined_network( + index, + args.total_num_nodes, + &known_libp2p_nodes, + is_da_node(index, args.num_da_nodes), + &public_key, + &private_key, + ) + .await + .with_context(|| "Failed to create Combined network")?; + + // Start the node + start_node_with_network!( + CombinedImpl, + index, + args.total_num_nodes, + &public_key, + &private_key, + &config, + memberships, + hotshot_initializer, + network, + join_set, + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + builder_url + ); + } + NetworkType::Cdn => { + // Create the CDN network + let network = create_cdn_network( + is_da_node(index, args.num_da_nodes), + &public_key, + &private_key, + ) + .await + .with_context(|| "Failed to create CDN network")?; + + // Start the node + start_node_with_network!( + PushCdnImpl, + index, + args.total_num_nodes, + &public_key, + &private_key, + &config, + memberships, + hotshot_initializer, + network, + join_set, + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + builder_url + ); + } + + NetworkType::LibP2P => { + // Create the Libp2p network + let network = create_libp2p_network( + index, + args.total_num_nodes, + &public_key, + &private_key, + &known_libp2p_nodes, + ) + .await + .with_context(|| "Failed to create Libp2p network")?; + + // Start the node + start_node_with_network!( + Libp2pImpl, + index, + args.total_num_nodes, + &public_key, + &private_key, + &config, + memberships, + hotshot_initializer, + network, + join_set, + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + builder_url + ); + } + }; + } + + // Wait for all the tasks to finish + while let Some(res) = join_set.join_next().await { + res.expect("Failed to join task"); + } + + Ok(()) +} diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index 83d110e380..239b394d00 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -32,7 +32,6 @@ pub mod helpers; use std::{ collections::{BTreeMap, HashMap}, - num::NonZeroUsize, sync::Arc, time::Duration, }; @@ -140,9 +139,6 @@ pub struct SystemContext, V: Versi InactiveReceiver>>, ), - /// uid for instrumentation - pub id: u64, - /// Reference to the internal storage for consensus datum. pub storage: Arc>, @@ -172,7 +168,6 @@ impl, V: Versions> Clone external_event_stream: self.external_event_stream.clone(), anchored_leaf: self.anchored_leaf.clone(), internal_event_stream: self.internal_event_stream.clone(), - id: self.id, storage: Arc::clone(&self.storage), upgrade_lock: self.upgrade_lock.clone(), marketplace_config: self.marketplace_config.clone(), @@ -196,7 +191,6 @@ impl, V: Versions> SystemContext::PrivateKey, - nonce: u64, config: HotShotConfig, memberships: TYPES::Membership, network: Arc, @@ -225,7 +219,6 @@ impl, V: Versions> SystemContext, V: Versions> SystemContext::PrivateKey, - nonce: u64, config: HotShotConfig, memberships: TYPES::Membership, network: Arc, @@ -349,7 +341,6 @@ impl, V: Versions> SystemContext> = Arc::new(SystemContext { - id: nonce, consensus: OuterConsensus::new(consensus), instance_state: Arc::new(instance_state), public_key, @@ -376,7 +367,7 @@ impl, V: Versions> SystemContext, V: Versions> SystemContext, V: Versions> SystemContext Leaf2 { self.consensus.read().await.decided_leaf() } @@ -549,7 +540,7 @@ impl, V: Versions> SystemContext Option> { self.consensus.try_read().map(|guard| guard.decided_leaf()) } @@ -558,7 +549,7 @@ impl, V: Versions> SystemContext Arc { Arc::clone(&self.consensus.read().await.decided_state()) } @@ -570,7 +561,7 @@ impl, V: Versions> SystemContext Option> { self.consensus.read().await.state(view).cloned() } @@ -592,7 +583,6 @@ impl, V: Versions> SystemContext::PrivateKey, - node_id: u64, config: HotShotConfig, memberships: TYPES::Membership, network: Arc, @@ -611,7 +601,6 @@ impl, V: Versions> SystemContext::PrivateKey, - nonce: u64, config: HotShotConfig, memberships: TYPES::Membership, network: Arc, @@ -771,7 +759,6 @@ where let left_system_context = SystemContext::new( public_key.clone(), private_key.clone(), - nonce, config.clone(), memberships.clone(), Arc::clone(&network), @@ -784,7 +771,6 @@ where let right_system_context = SystemContext::new( public_key, private_key, - nonce, config, memberships, network, @@ -938,8 +924,8 @@ impl, V: Versions> TwinsHandlerSta impl, V: Versions> ConsensusApi for SystemContextHandle { - fn total_nodes(&self) -> NonZeroUsize { - self.hotshot.config.num_nodes_with_stake + fn total_nodes(&self) -> usize { + self.hotshot.config.known_nodes.len() } fn builder_timeout(&self) -> Duration { diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index 7e94aea326..c59356aa98 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -85,7 +85,6 @@ pub fn add_response_task, V: Versi (*handle.hotshot.memberships).clone().into(), handle.public_key().clone(), handle.private_key().clone(), - handle.hotshot.id, ); handle.network_registry.register(run_response_task::( state, @@ -319,7 +318,6 @@ where &'static mut self, public_key: TYPES::SignatureKey, private_key: ::PrivateKey, - nonce: u64, config: HotShotConfig, memberships: TYPES::Membership, network: Arc, @@ -332,7 +330,6 @@ where let hotshot = SystemContext::new( public_key, private_key, - nonce, config, memberships, network, diff --git a/crates/hotshot/src/tasks/task_state.rs b/crates/hotshot/src/tasks/task_state.rs index f97905b6a4..1d5e250aa3 100644 --- a/crates/hotshot/src/tasks/task_state.rs +++ b/crates/hotshot/src/tasks/task_state.rs @@ -61,7 +61,6 @@ impl, V: Versions> CreateTaskState membership: (*handle.hotshot.memberships).clone(), public_key: handle.public_key().clone(), private_key: handle.private_key().clone(), - id: handle.hotshot.id, shutdown_flag: Arc::new(AtomicBool::new(false)), spawned_tasks: BTreeMap::new(), } @@ -82,7 +81,6 @@ impl, V: Versions> CreateTaskState vote_collectors: BTreeMap::default(), public_key: handle.public_key().clone(), private_key: handle.private_key().clone(), - id: handle.hotshot.id, start_proposing_view: handle.hotshot.config.start_proposing_view, stop_proposing_view: handle.hotshot.config.stop_proposing_view, start_voting_view: handle.hotshot.config.start_voting_view, @@ -131,7 +129,6 @@ impl, V: Versions> CreateTaskState membership: (*handle.hotshot.memberships).clone().into(), public_key: handle.public_key().clone(), private_key: handle.private_key().clone(), - id: handle.hotshot.id, } } } @@ -151,7 +148,6 @@ impl, V: Versions> CreateTaskState vote_collectors: BTreeMap::default(), public_key: handle.public_key().clone(), private_key: handle.private_key().clone(), - id: handle.hotshot.id, storage: Arc::clone(&handle.storage), upgrade_lock: handle.hotshot.upgrade_lock.clone(), } @@ -178,7 +174,6 @@ impl, V: Versions> CreateTaskState commit_relay_map: HashMap::default().into(), finalize_relay_map: HashMap::default().into(), view_sync_timeout: handle.hotshot.config.view_sync_timeout, - id: handle.hotshot.id, last_garbage_collected_view: TYPES::View::new(0), upgrade_lock: handle.hotshot.upgrade_lock.clone(), } @@ -200,7 +195,6 @@ impl, V: Versions> CreateTaskState public_key: handle.public_key().clone(), private_key: handle.private_key().clone(), instance_state: handle.hotshot.instance_state(), - id: handle.hotshot.id, builder_clients: handle .hotshot .config @@ -243,7 +237,6 @@ impl, V: Versions> CreateTaskState membership: (*handle.hotshot.memberships).clone().into(), drb_computations: DrbComputations::new(), output_event_stream: handle.hotshot.external_event_stream.0.clone(), - id: handle.hotshot.id, storage: Arc::clone(&handle.storage), upgrade_lock: handle.hotshot.upgrade_lock.clone(), epoch_height: handle.hotshot.config.epoch_height, @@ -269,7 +262,6 @@ impl, V: Versions> CreateTaskState private_key: handle.private_key().clone(), storage: Arc::clone(&handle.storage), timeout: handle.hotshot.config.next_view_timeout, - id: handle.hotshot.id, formed_upgrade_certificate: None, upgrade_lock: handle.hotshot.upgrade_lock.clone(), epoch_height: handle.hotshot.config.epoch_height, @@ -296,7 +288,6 @@ impl, V: Versions> CreateTaskState output_event_stream: handle.hotshot.external_event_stream.0.clone(), storage: Arc::clone(&handle.storage), spawned_tasks: BTreeMap::new(), - id: handle.hotshot.id, upgrade_lock: handle.hotshot.upgrade_lock.clone(), epoch_height: handle.hotshot.config.epoch_height, } @@ -325,7 +316,6 @@ impl, V: Versions> CreateTaskState timeout_task: spawn(async {}), timeout: handle.hotshot.config.next_view_timeout, consensus: OuterConsensus::new(consensus), - id: handle.hotshot.id, upgrade_lock: handle.hotshot.upgrade_lock.clone(), epoch_height: handle.hotshot.config.epoch_height, } @@ -339,7 +329,7 @@ impl, V: Versions> CreateTaskState async fn create_from(handle: &SystemContextHandle) -> Self { Self { events: Vec::new(), - id: handle.hotshot.id, + public_key: handle.public_key().clone(), } } } diff --git a/crates/hotshot/src/traits/election/randomized_committee.rs b/crates/hotshot/src/traits/election/randomized_committee.rs index 4046123553..3523f45fd8 100644 --- a/crates/hotshot/src/traits/election/randomized_committee.rs +++ b/crates/hotshot/src/traits/election/randomized_committee.rs @@ -16,9 +16,10 @@ use hotshot_types::{ }; use primitive_types::U256; use rand::{rngs::StdRng, Rng}; +use serde::{Deserialize, Serialize}; use utils::anytrace::Result; -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] /// The static committee election diff --git a/crates/hotshot/src/traits/election/randomized_committee_members.rs b/crates/hotshot/src/traits/election/randomized_committee_members.rs index 5c85ad9c07..a979ed1dcc 100644 --- a/crates/hotshot/src/traits/election/randomized_committee_members.rs +++ b/crates/hotshot/src/traits/election/randomized_committee_members.rs @@ -21,11 +21,12 @@ use hotshot_types::{ }; use primitive_types::U256; use rand::{rngs::StdRng, Rng}; +use serde::{Deserialize, Serialize}; use utils::anytrace::Result; use crate::traits::election::helpers::QuorumFilterConfig; -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] /// The static committee election pub struct RandomizedCommitteeMembers { /// The nodes eligible for leadership. diff --git a/crates/hotshot/src/traits/election/static_committee.rs b/crates/hotshot/src/traits/election/static_committee.rs index d2b62f80b7..e423294d81 100644 --- a/crates/hotshot/src/traits/election/static_committee.rs +++ b/crates/hotshot/src/traits/election/static_committee.rs @@ -15,9 +15,10 @@ use hotshot_types::{ PeerConfig, }; use primitive_types::U256; +use serde::{Deserialize, Serialize}; use utils::anytrace::Result; -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] /// The static committee election pub struct StaticCommittee { /// The nodes eligible for leadership. diff --git a/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs b/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs index 8833d06872..eda8c4a263 100644 --- a/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs +++ b/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs @@ -15,9 +15,10 @@ use hotshot_types::{ PeerConfig, }; use primitive_types::U256; +use serde::{Deserialize, Serialize}; use utils::anytrace::Result; -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] /// The static committee election pub struct StaticCommitteeLeaderForTwoViews { diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index e37294856a..52fc3af761 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -7,7 +7,6 @@ //! Libp2p based/production networking implementation //! This module provides a libp2p based networking implementation where each node in the //! network forms a tcp or udp connection to a subset of other nodes in the network -#[cfg(feature = "hotshot-testing")] use std::str::FromStr; use std::{ collections::{HashMap, HashSet}, @@ -31,7 +30,6 @@ use hotshot_types::{ boxed_sync, constants::LOOK_AHEAD, data::ViewNumber, - light_client::StateVerKey, traits::{ election::Membership, metrics::{Counter, Gauge, Metrics, NoMetrics}, @@ -39,8 +37,11 @@ use hotshot_types::{ node_implementation::{ConsensusTime, NodeType}, signature_key::{PrivateSignatureKey, SignatureKey}, }, - BoxSyncFuture, PeerConfig, + BoxSyncFuture, }; +#[cfg(feature = "hotshot-testing")] +use hotshot_types::{light_client::StateVerKey, PeerConfig}; + use libp2p_identity::{ ed25519::{self, SecretKey}, Keypair, PeerId, @@ -49,14 +50,17 @@ pub use libp2p_networking::network::{GossipConfig, RequestResponseConfig}; use libp2p_networking::{ network::{ behaviours::dht::record::{Namespace, RecordKey, RecordValue}, - node::config::{KademliaConfig, Libp2pConfig}, + node::config::Libp2pConfig, spawn_network_node, - transport::construct_auth_message, NetworkEvent::{self, DirectRequest, DirectResponse, GossipMsg}, NetworkNodeHandle, NetworkNodeReceiver, }, reexport::Multiaddr, }; + +#[cfg(feature = "hotshot-testing")] +use libp2p_networking::network::{node::config::KademliaConfig, transport::construct_auth_message}; +#[cfg(feature = "hotshot-testing")] use rand::Rng; use serde::Serialize; use tokio::{ @@ -172,6 +176,7 @@ pub struct Libp2pNetwork { } /// Generate a [testing] multiaddr +#[cfg(feature = "hotshot-testing")] fn random_multiaddr() -> Multiaddr { Multiaddr::from_str(&format!( "/ip4/127.0.0.1/udp/{}/quic-v1", @@ -181,6 +186,7 @@ fn random_multiaddr() -> Multiaddr { } /// Generate the expected [testing] multiaddr from a node index +#[cfg(feature = "hotshot-testing")] fn multiaddr_from_node_index( i: usize, bind_addresses: &HashMap, @@ -575,8 +581,8 @@ impl Libp2pNetwork { sleep(Duration::from_secs(1)).await; } - // Wait for the network to connect to the required number of peers - if let Err(e) = handle.wait_to_connect(4).await { + // Wait for the network to connect to at least one peer + if let Err(e) = handle.wait_to_connect(1).await { error!("Failed to connect to peers: {:?}", e); return Err::<(), NetworkError>(e); } diff --git a/crates/hotshot/src/traits/networking/push_cdn_network.rs b/crates/hotshot/src/traits/networking/push_cdn_network.rs index f3a068530f..90ad20994f 100644 --- a/crates/hotshot/src/traits/networking/push_cdn_network.rs +++ b/crates/hotshot/src/traits/networking/push_cdn_network.rs @@ -39,16 +39,22 @@ use hotshot_types::{ traits::{ metrics::{Counter, Metrics, NoMetrics}, network::{BroadcastDelay, ConnectedNetwork, Topic as HotShotTopic}, - node_implementation::NodeType, signature_key::SignatureKey, }, utils::bincode_opts, BoxSyncFuture, }; + +#[cfg(feature = "hotshot-testing")] +use hotshot_types::traits::node_implementation::NodeType; + use num_enum::{IntoPrimitive, TryFromPrimitive}; #[cfg(feature = "hotshot-testing")] use rand::{rngs::StdRng, RngCore, SeedableRng}; -use tokio::{spawn, sync::mpsc::error::TrySendError, time::sleep}; +use tokio::sync::mpsc::error::TrySendError; +#[cfg(feature = "hotshot-testing")] +use tokio::{spawn, time::sleep}; +#[cfg(feature = "hotshot-testing")] use tracing::error; use super::NetworkError; diff --git a/crates/hotshot/src/types/handle.rs b/crates/hotshot/src/types/handle.rs index 9ea46b34d7..a440a786d2 100644 --- a/crates/hotshot/src/types/handle.rs +++ b/crates/hotshot/src/types/handle.rs @@ -353,13 +353,13 @@ impl + 'static, V: Versions> } /// Wrapper to get the view number this node is on. - #[instrument(skip_all, target = "SystemContextHandle", fields(id = self.hotshot.id))] + #[instrument(skip_all, target = "SystemContextHandle")] pub async fn cur_view(&self) -> TYPES::View { self.hotshot.consensus.read().await.cur_view() } /// Wrapper to get the epoch number this node is on. - #[instrument(skip_all, target = "SystemContextHandle", fields(id = self.hotshot.id))] + #[instrument(skip_all, target = "SystemContextHandle")] pub async fn cur_epoch(&self) -> TYPES::Epoch { self.hotshot.consensus.read().await.cur_epoch() } diff --git a/crates/task-impls/src/consensus/handlers.rs b/crates/task-impls/src/consensus/handlers.rs index 27fc0a7b43..4131453009 100644 --- a/crates/task-impls/src/consensus/handlers.rs +++ b/crates/task-impls/src/consensus/handlers.rs @@ -62,7 +62,6 @@ pub(crate) async fn handle_quorum_vote_recv< task_state.public_key.clone(), &task_state.membership, task_state.cur_epoch, - task_state.id, &event, sender, &task_state.upgrade_lock, @@ -102,7 +101,6 @@ pub(crate) async fn handle_timeout_vote_recv< task_state.public_key.clone(), &task_state.membership, task_state.cur_epoch, - task_state.id, &event, sender, &task_state.upgrade_lock, diff --git a/crates/task-impls/src/consensus/mod.rs b/crates/task-impls/src/consensus/mod.rs index c865ed82e1..d08f1674b0 100644 --- a/crates/task-impls/src/consensus/mod.rs +++ b/crates/task-impls/src/consensus/mod.rs @@ -77,9 +77,6 @@ pub struct ConsensusTaskState, V: /// A reference to the metrics trait. pub consensus: OuterConsensus, - /// The node's id - pub id: u64, - /// Lock for a decided upgrade pub upgrade_lock: UpgradeLock, @@ -88,7 +85,7 @@ pub struct ConsensusTaskState, V: } impl, V: Versions> ConsensusTaskState { /// Handles a consensus event received on the event stream - #[instrument(skip_all, fields(id = self.id, cur_view = *self.cur_view, cur_epoch = *self.cur_epoch), name = "Consensus replica task", level = "error", target = "ConsensusTaskState")] + #[instrument(skip_all, fields(cur_view = *self.cur_view, cur_epoch = *self.cur_epoch), name = "Consensus replica task", level = "error", target = "ConsensusTaskState")] pub async fn handle( &mut self, event: Arc>, diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index 2c503d2b13..6be0d623ae 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -69,9 +69,6 @@ pub struct DaTaskState, V: Version /// This Nodes private key pub private_key: ::PrivateKey, - /// This state's ID - pub id: u64, - /// This node's storage ref pub storage: Arc>, @@ -81,7 +78,7 @@ pub struct DaTaskState, V: Version impl, V: Versions> DaTaskState { /// main task event handler - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "DA Main Task", level = "error", target = "DaTaskState")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "DA Main Task", level = "error", target = "DaTaskState")] pub async fn handle( &mut self, event: Arc>, @@ -274,7 +271,6 @@ impl, V: Versions> DaTaskState( /// # Errors /// If any validation or state update fails. #[allow(clippy::too_many_lines)] -#[instrument(skip_all, fields(id = validation_info.id, view = *proposal.data.view_number()))] +#[instrument(skip_all, fields(view = *proposal.data.view_number()))] pub async fn validate_proposal_safety_and_liveness< TYPES: NodeType, I: NodeImplementation, diff --git a/crates/task-impls/src/quorum_proposal/handlers.rs b/crates/task-impls/src/quorum_proposal/handlers.rs index b1cc8e36fa..d7bf08ecfd 100644 --- a/crates/task-impls/src/quorum_proposal/handlers.rs +++ b/crates/task-impls/src/quorum_proposal/handlers.rs @@ -105,9 +105,6 @@ pub struct ProposalDependencyHandle { /// Lock for a decided upgrade pub upgrade_lock: UpgradeLock, - /// The node's id - pub id: u64, - /// The time this view started pub view_start_time: Instant, @@ -186,7 +183,7 @@ impl ProposalDependencyHandle { /// Publishes a proposal given the [`CommitmentAndMetadata`], [`VidDisperse`] /// and high qc [`hotshot_types::simple_certificate::QuorumCertificate`], /// with optional [`ViewChangeEvidence`]. - #[instrument(skip_all, fields(id = self.id, view_number = *self.view_number, latest_proposed_view = *self.latest_proposed_view))] + #[instrument(skip_all, fields(view_number = *self.view_number, latest_proposed_view = *self.latest_proposed_view))] async fn publish_proposal( &self, commitment_and_metadata: CommitmentAndMetadata, diff --git a/crates/task-impls/src/quorum_proposal/mod.rs b/crates/task-impls/src/quorum_proposal/mod.rs index 06150dff97..aeb774ff17 100644 --- a/crates/task-impls/src/quorum_proposal/mod.rs +++ b/crates/task-impls/src/quorum_proposal/mod.rs @@ -65,9 +65,6 @@ pub struct QuorumProposalTaskState /// Shared consensus task state pub consensus: OuterConsensus, - /// The node's id - pub id: u64, - /// The most recent upgrade certificate this node formed. /// Note: this is ONLY for certificates that have been formed internally, /// so that we can propose with them. @@ -90,7 +87,7 @@ impl, V: Versions> QuorumProposalTaskState { /// Create an event dependency - #[instrument(skip_all, fields(id = self.id, latest_proposed_view = *self.latest_proposed_view), name = "Create event dependency", level = "info")] + #[instrument(skip_all, fields(latest_proposed_view = *self.latest_proposed_view), name = "Create event dependency", level = "info")] fn create_event_dependency( &self, dependency_type: ProposalDependency, @@ -266,7 +263,7 @@ impl, V: Versions> /// dependency as already completed. This allows for the task to receive a proposable event /// without losing the data that it received, as the dependency task would otherwise have no /// ability to receive the event and, thus, would never propose. - #[instrument(skip_all, fields(id = self.id, latest_proposed_view = *self.latest_proposed_view), name = "Create dependency task", level = "error")] + #[instrument(skip_all, fields(latest_proposed_view = *self.latest_proposed_view), name = "Create dependency task", level = "error")] fn create_dependency_task_if_new( &mut self, view_number: TYPES::View, @@ -314,7 +311,6 @@ impl, V: Versions> timeout: self.timeout, formed_upgrade_certificate: self.formed_upgrade_certificate.clone(), upgrade_lock: self.upgrade_lock.clone(), - id: self.id, view_start_time: Instant::now(), highest_qc: self.highest_qc.clone(), }, @@ -326,7 +322,7 @@ impl, V: Versions> } /// Update the latest proposed view number. - #[instrument(skip_all, fields(id = self.id, latest_proposed_view = *self.latest_proposed_view), name = "Update latest proposed view", level = "error")] + #[instrument(skip_all, fields(latest_proposed_view = *self.latest_proposed_view), name = "Update latest proposed view", level = "error")] async fn update_latest_proposed_view(&mut self, new_view: TYPES::View) -> bool { if *self.latest_proposed_view < *new_view { tracing::debug!( @@ -351,7 +347,7 @@ impl, V: Versions> } /// Handles a consensus event received on the event stream - #[instrument(skip_all, fields(id = self.id, latest_proposed_view = *self.latest_proposed_view), name = "handle method", level = "error", target = "QuorumProposalTaskState")] + #[instrument(skip_all, fields(latest_proposed_view = *self.latest_proposed_view), name = "handle method", level = "error", target = "QuorumProposalTaskState")] pub async fn handle( &mut self, event: Arc>, diff --git a/crates/task-impls/src/quorum_proposal_recv/mod.rs b/crates/task-impls/src/quorum_proposal_recv/mod.rs index db62d33e5d..916646b05b 100644 --- a/crates/task-impls/src/quorum_proposal_recv/mod.rs +++ b/crates/task-impls/src/quorum_proposal_recv/mod.rs @@ -73,9 +73,6 @@ pub struct QuorumProposalRecvTaskState>>, - /// The node's id - pub id: u64, - /// Lock for a decided upgrade pub upgrade_lock: UpgradeLock, @@ -86,8 +83,6 @@ pub struct QuorumProposalRecvTaskState, V: Versions> { - /// The node's id - pub id: u64, /// Our public key pub(crate) public_key: TYPES::SignatureKey, /// Our Private Key @@ -123,7 +118,7 @@ impl, V: Versions> } /// Handles all consensus events relating to propose and vote-enabling events. - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "Consensus replica task", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "Consensus replica task", level = "error")] #[allow(unused_variables)] pub async fn handle( &mut self, @@ -140,7 +135,6 @@ impl, V: Versions> return; } let validation_info = ValidationInfo:: { - id: self.id, public_key: self.public_key.clone(), private_key: self.private_key.clone(), cur_epoch: self.cur_epoch, diff --git a/crates/task-impls/src/quorum_vote/handlers.rs b/crates/task-impls/src/quorum_vote/handlers.rs index 24629116ec..3acba8300d 100644 --- a/crates/task-impls/src/quorum_vote/handlers.rs +++ b/crates/task-impls/src/quorum_vote/handlers.rs @@ -132,7 +132,7 @@ fn handle_quorum_proposal_validated_drb_calculation_seed< } /// Handles the `QuorumProposalValidated` event. -#[instrument(skip_all, fields(id = task_state.id, view = *proposal.view_number))] +#[instrument(skip_all, fields(view = *proposal.view_number))] pub(crate) async fn handle_quorum_proposal_validated< TYPES: NodeType, I: NodeImplementation, diff --git a/crates/task-impls/src/quorum_vote/mod.rs b/crates/task-impls/src/quorum_vote/mod.rs index cf8181d771..1e29df5f81 100644 --- a/crates/task-impls/src/quorum_vote/mod.rs +++ b/crates/task-impls/src/quorum_vote/mod.rs @@ -85,8 +85,6 @@ pub struct VoteDependencyHandle, V pub upgrade_lock: UpgradeLock, /// The consensus metrics pub consensus_metrics: Arc, - /// The node's id - pub id: u64, /// Number of blocks in an epoch, zero means there are no epochs pub epoch_height: u64, } @@ -97,7 +95,7 @@ impl + 'static, V: Versions> Handl type Output = Vec>>; #[allow(clippy::too_many_lines)] - #[instrument(skip_all, fields(id = self.id, view = *self.view_number))] + #[instrument(skip_all, fields(view = *self.view_number))] async fn handle_dep_result(self, res: Self::Output) { let mut payload_commitment = None; let mut leaf = None; @@ -285,9 +283,6 @@ pub struct QuorumVoteTaskState, V: /// Output events to application pub output_event_stream: async_broadcast::Sender>, - /// The node's id - pub id: u64, - /// The consensus metrics pub consensus_metrics: Arc, @@ -303,7 +298,7 @@ pub struct QuorumVoteTaskState, V: impl, V: Versions> QuorumVoteTaskState { /// Create an event dependency. - #[instrument(skip_all, fields(id = self.id, latest_voted_view = *self.latest_voted_view), name = "Quorum vote create event dependency", level = "error")] + #[instrument(skip_all, fields(latest_voted_view = *self.latest_voted_view), name = "Quorum vote create event dependency", level = "error")] fn create_event_dependency( &self, dependency_type: VoteDependency, @@ -348,7 +343,7 @@ impl, V: Versions> QuorumVoteTaskS /// Create and store an [`AndDependency`] combining [`EventDependency`]s associated with the /// given view number if it doesn't exist. - #[instrument(skip_all, fields(id = self.id, latest_voted_view = *self.latest_voted_view), name = "Quorum vote crete dependency task if new", level = "error")] + #[instrument(skip_all, fields(latest_voted_view = *self.latest_voted_view), name = "Quorum vote crete dependency task if new", level = "error")] fn create_dependency_task_if_new( &mut self, view_number: TYPES::View, @@ -393,7 +388,6 @@ impl, V: Versions> QuorumVoteTaskS sender: event_sender.clone(), receiver: event_receiver.clone().deactivate(), upgrade_lock: self.upgrade_lock.clone(), - id: self.id, epoch_height: self.epoch_height, consensus_metrics: Arc::clone(&self.consensus_metrics), }, @@ -403,7 +397,7 @@ impl, V: Versions> QuorumVoteTaskS } /// Update the latest voted view number. - #[instrument(skip_all, fields(id = self.id, latest_voted_view = *self.latest_voted_view), name = "Quorum vote update latest voted view", level = "error")] + #[instrument(skip_all, fields(latest_voted_view = *self.latest_voted_view), name = "Quorum vote update latest voted view", level = "error")] async fn update_latest_voted_view(&mut self, new_view: TYPES::View) -> bool { if *self.latest_voted_view < *new_view { tracing::debug!( @@ -437,7 +431,7 @@ impl, V: Versions> QuorumVoteTaskS } /// Handle a vote dependent event received on the event stream - #[instrument(skip_all, fields(id = self.id, latest_voted_view = *self.latest_voted_view), name = "Quorum vote handle", level = "error", target = "QuorumVoteTaskState")] + #[instrument(skip_all, fields(latest_voted_view = *self.latest_voted_view), name = "Quorum vote handle", level = "error", target = "QuorumVoteTaskState")] pub async fn handle( &mut self, event: Arc>, diff --git a/crates/task-impls/src/request.rs b/crates/task-impls/src/request.rs index a41b04f5d5..156e018b9b 100644 --- a/crates/task-impls/src/request.rs +++ b/crates/task-impls/src/request.rs @@ -65,8 +65,6 @@ pub struct NetworkRequestState> { pub public_key: TYPES::SignatureKey, /// This nodes private/signing key, used to sign requests. pub private_key: ::PrivateKey, - /// The node's id - pub id: u64, /// A flag indicating that `HotShotEvent::Shutdown` has been received pub shutdown_flag: Arc, /// A flag indicating that `HotShotEvent::Shutdown` has been received @@ -87,7 +85,7 @@ type Signature = impl> TaskState for NetworkRequestState { type Event = HotShotEvent; - #[instrument(skip_all, target = "NetworkRequestState", fields(id = self.id))] + #[instrument(skip_all, target = "NetworkRequestState")] async fn handle_event( &mut self, event: Arc, diff --git a/crates/task-impls/src/response.rs b/crates/task-impls/src/response.rs index 1ac18d8dab..91c1a4f3bc 100644 --- a/crates/task-impls/src/response.rs +++ b/crates/task-impls/src/response.rs @@ -37,8 +37,6 @@ pub struct NetworkResponseState { pub_key: TYPES::SignatureKey, /// This replicas private key private_key: ::PrivateKey, - /// The node's id - id: u64, } impl NetworkResponseState { @@ -48,14 +46,12 @@ impl NetworkResponseState { quorum: Arc, pub_key: TYPES::SignatureKey, private_key: ::PrivateKey, - id: u64, ) -> Self { Self { consensus, quorum, pub_key, private_key, - id, } } @@ -133,7 +129,7 @@ impl NetworkResponseState { /// Get the VID share from consensus storage, or calculate it from the payload for /// the view, if we have the payload. Stores all the shares calculated from the payload /// if the calculation was done - #[instrument(skip_all, target = "NetworkResponseState", fields(id = self.id))] + #[instrument(skip_all, target = "NetworkResponseState")] async fn get_or_calc_vid_share( &self, view: TYPES::View, diff --git a/crates/task-impls/src/rewind.rs b/crates/task-impls/src/rewind.rs index 4f62359aeb..9e30e95c1d 100644 --- a/crates/task-impls/src/rewind.rs +++ b/crates/task-impls/src/rewind.rs @@ -9,7 +9,7 @@ use std::{fs::OpenOptions, io::Write, sync::Arc}; use async_broadcast::{Receiver, Sender}; use async_trait::async_trait; use hotshot_task::task::TaskState; -use hotshot_types::traits::node_implementation::NodeType; +use hotshot_types::{traits::node_implementation::NodeType, utils::mnemonic}; use utils::anytrace::Result; use crate::events::HotShotEvent; @@ -20,8 +20,8 @@ pub struct RewindTaskState { /// All events received by this node since the beginning of time. pub events: Vec>>, - /// The id of this node - pub id: u64, + /// The public key of this node + pub public_key: TYPES::SignatureKey, } impl RewindTaskState { @@ -46,8 +46,8 @@ impl TaskState for RewindTaskState { } fn cancel_subtasks(&mut self) { - tracing::info!("Node ID {} Recording {} events", self.id, self.events.len()); - let filename = format!("rewind_{}.log", self.id); + tracing::info!("Recording {} events", self.events.len()); + let filename = format!("rewind_{}.log", mnemonic(&self.public_key)); let mut file = match OpenOptions::new() .write(true) .create(true) diff --git a/crates/task-impls/src/transactions.rs b/crates/task-impls/src/transactions.rs index f6da7ffec0..6aa1ca6e75 100644 --- a/crates/task-impls/src/transactions.rs +++ b/crates/task-impls/src/transactions.rs @@ -108,9 +108,6 @@ pub struct TransactionTaskState, V /// InstanceState pub instance_state: Arc, - /// This state's ID - pub id: u64, - /// Lock for a decided upgrade pub upgrade_lock: UpgradeLock, @@ -146,7 +143,7 @@ impl, V: Versions> TransactionTask } /// legacy view change handler - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view), name = "Transaction task", level = "error", target = "TransactionTaskState")] + #[instrument(skip_all, fields(view = *self.cur_view), name = "Transaction task", level = "error", target = "TransactionTaskState")] pub async fn handle_view_change_legacy( &mut self, event_stream: &Sender>>, @@ -433,7 +430,7 @@ impl, V: Versions> TransactionTask } /// epochs view change handler - #[instrument(skip_all, fields(id = self.id, view_number = *self.cur_view))] + #[instrument(skip_all, fields(view_number = *self.cur_view))] pub async fn handle_view_change_epochs( &mut self, event_stream: &Sender>>, @@ -449,7 +446,7 @@ impl, V: Versions> TransactionTask } /// main task event handler - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "Transaction task", level = "error", target = "TransactionTaskState")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "Transaction task", level = "error", target = "TransactionTaskState")] pub async fn handle( &mut self, event: Arc>, @@ -492,7 +489,7 @@ impl, V: Versions> TransactionTask /// Get VID commitment for the last successful view before `block_view`. /// Returns None if we don't have said commitment recorded. - #[instrument(skip_all, target = "TransactionTaskState", fields(id = self.id, cur_view = *self.cur_view, block_view = *block_view))] + #[instrument(skip_all, target = "TransactionTaskState", fields(cur_view = *self.cur_view, block_view = *block_view))] async fn last_vid_commitment_retry( &self, block_view: TYPES::View, @@ -513,7 +510,7 @@ impl, V: Versions> TransactionTask /// Get VID commitment for the last successful view before `block_view`. /// Returns None if we don't have said commitment recorded. - #[instrument(skip_all, target = "TransactionTaskState", fields(id = self.id, cur_view = *self.cur_view, block_view = *block_view))] + #[instrument(skip_all, target = "TransactionTaskState", fields(cur_view = *self.cur_view, block_view = *block_view))] async fn last_vid_commitment( &self, block_view: TYPES::View, @@ -551,7 +548,7 @@ impl, V: Versions> TransactionTask } } - #[instrument(skip_all, fields(id = self.id, cur_view = *self.cur_view, block_view = *block_view), name = "wait_for_block", level = "error")] + #[instrument(skip_all, fields(cur_view = *self.cur_view, block_view = *block_view), name = "wait_for_block", level = "error")] async fn wait_for_block(&self, block_view: TYPES::View) -> Option> { let task_start_time = Instant::now(); @@ -681,7 +678,7 @@ impl, V: Versions> TransactionTask /// # Errors /// If none of the builder reports any available blocks or claiming block fails for all of the /// builders. - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view), name = "block_from_builder", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view), name = "block_from_builder", level = "error")] async fn block_from_builder( &self, parent_comm: VidCommitment, diff --git a/crates/task-impls/src/upgrade.rs b/crates/task-impls/src/upgrade.rs index a8dec2e3f3..815b75e050 100644 --- a/crates/task-impls/src/upgrade.rs +++ b/crates/task-impls/src/upgrade.rs @@ -60,9 +60,6 @@ pub struct UpgradeTaskState { /// This Nodes private key pub private_key: ::PrivateKey, - /// This state's ID - pub id: u64, - /// View to start proposing an upgrade pub start_proposing_view: u64, @@ -102,7 +99,7 @@ impl UpgradeTaskState { } /// main task event handler - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "Upgrade Task", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "Upgrade Task", level = "error")] pub async fn handle( &mut self, event: Arc>, @@ -239,7 +236,6 @@ impl UpgradeTaskState { self.public_key.clone(), &self.quorum_membership, self.cur_epoch, - self.id, &event, &tx, &self.upgrade_lock, diff --git a/crates/task-impls/src/vid.rs b/crates/task-impls/src/vid.rs index f19f6e7528..259e20e0b5 100644 --- a/crates/task-impls/src/vid.rs +++ b/crates/task-impls/src/vid.rs @@ -49,14 +49,11 @@ pub struct VidTaskState> { /// Our Private Key pub private_key: ::PrivateKey, - - /// This state's ID - pub id: u64, } impl> VidTaskState { /// main task event handler - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "VID Main Task", level = "error", target = "VidTaskState")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "VID Main Task", level = "error", target = "VidTaskState")] pub async fn handle( &mut self, event: Arc>, diff --git a/crates/task-impls/src/view_sync.rs b/crates/task-impls/src/view_sync.rs index 6bb83dfc53..47ebe86444 100644 --- a/crates/task-impls/src/view_sync.rs +++ b/crates/task-impls/src/view_sync.rs @@ -81,9 +81,6 @@ pub struct ViewSyncTaskState { /// Our Private Key pub private_key: ::PrivateKey, - /// Our node id; for logging - pub id: u64, - /// How many timeouts we've seen in a row; is reset upon a successful view change pub num_timeouts_tracked: u64, @@ -156,9 +153,6 @@ pub struct ViewSyncReplicaTaskState { /// Timeout task handle, when it expires we try the next relay pub timeout_task: Option>, - /// Our node id; for logging - pub id: u64, - /// Membership for the quorum pub membership: Arc, @@ -191,7 +185,7 @@ impl TaskState for ViewSyncReplicaTaskState ViewSyncTaskState { - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view), name = "View Sync Main Task", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view), name = "View Sync Main Task", level = "error")] #[allow(clippy::type_complexity)] /// Handles incoming events for the main view sync task pub async fn send_to_or_create_replica( @@ -238,7 +232,6 @@ impl ViewSyncTaskState { public_key: self.public_key.clone(), private_key: self.private_key.clone(), view_sync_timeout: self.view_sync_timeout, - id: self.id, upgrade_lock: self.upgrade_lock.clone(), }; @@ -254,7 +247,7 @@ impl ViewSyncTaskState { task_map.insert(view, replica_state); } - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "View Sync Main Task", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "View Sync Main Task", level = "error")] #[allow(clippy::type_complexity)] /// Handles incoming events for the main view sync task pub async fn handle( @@ -319,7 +312,6 @@ impl ViewSyncTaskState { membership: Arc::clone(&self.membership), view: vote_view, epoch: self.cur_epoch, - id: self.id, }; let vote_collector = create_vote_accumulator( &info, @@ -364,7 +356,6 @@ impl ViewSyncTaskState { membership: Arc::clone(&self.membership), view: vote_view, epoch: self.cur_epoch, - id: self.id, }; let vote_collector = create_vote_accumulator( @@ -409,7 +400,6 @@ impl ViewSyncTaskState { membership: Arc::clone(&self.membership), view: vote_view, epoch: self.cur_epoch, - id: self.id, }; let vote_collector = create_vote_accumulator( &info, @@ -513,7 +503,7 @@ impl ViewSyncTaskState { } impl ViewSyncReplicaTaskState { - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "View Sync Replica Task", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "View Sync Replica Task", level = "error")] /// Handle incoming events for the view sync replica task pub async fn handle( &mut self, diff --git a/crates/task-impls/src/vote_collection.rs b/crates/task-impls/src/vote_collection.rs index e0fc96040b..bca808ce50 100644 --- a/crates/task-impls/src/vote_collection.rs +++ b/crates/task-impls/src/vote_collection.rs @@ -61,9 +61,6 @@ pub struct VoteCollectionTaskState< /// The epoch which we are collecting votes for pub epoch: TYPES::Epoch, - /// Node id - pub id: u64, - /// Whether we should check if we are the leader when handling a vote pub check_if_leader: bool, } @@ -179,8 +176,6 @@ pub struct AccumulatorInfo { pub view: TYPES::View, /// Epoch of the votes we are collecting pub epoch: TYPES::Epoch, - /// This nodes id - pub id: u64, } /// Generic function for spawning a vote task. Returns the event stream id of the spawned task if created @@ -225,7 +220,6 @@ where accumulator: Some(new_accumulator), view: info.view, epoch: info.epoch, - id: info.id, check_if_leader, }; @@ -254,7 +248,6 @@ pub async fn handle_vote< public_key: TYPES::SignatureKey, membership: &Arc, epoch: TYPES::Epoch, - id: u64, event: &Arc>, event_stream: &Sender>>, upgrade_lock: &UpgradeLock, @@ -271,7 +264,6 @@ where membership: Arc::clone(membership), view: vote.view_number(), epoch, - id, }; let collector = create_vote_accumulator( &info, diff --git a/crates/testing/src/helpers.rs b/crates/testing/src/helpers.rs index cc468a1859..9722e46c1d 100644 --- a/crates/testing/src/helpers.rs +++ b/crates/testing/src/helpers.rs @@ -100,7 +100,7 @@ pub async fn build_system_handle_from_launcher< .unwrap(); // See whether or not we should be DA - let is_da = node_id < config.da_staked_committee_size as u64; + let is_da = node_id < config.known_da_nodes.len() as u64; // We assign node's public key and stake value rather than read from config file since it's a test let validator_config: ValidatorConfig = @@ -108,15 +108,12 @@ pub async fn build_system_handle_from_launcher< let private_key = validator_config.private_key.clone(); let public_key = validator_config.public_key.clone(); - let memberships = TYPES::Membership::new( - config.known_nodes_with_stake.clone(), - config.known_da_nodes.clone(), - ); + let memberships = + TYPES::Membership::new(config.known_nodes.clone(), config.known_da_nodes.clone()); SystemContext::init( public_key, private_key, - node_id, config, memberships, network, diff --git a/crates/testing/src/spinning_task.rs b/crates/testing/src/spinning_task.rs index faee282c32..b037aac261 100644 --- a/crates/testing/src/spinning_task.rs +++ b/crates/testing/src/spinning_task.rs @@ -171,11 +171,10 @@ where node_id, 1, // For tests, make the node DA based on its index - node_id < config.da_staked_committee_size as u64, + node_id < config.known_da_nodes.len() as u64, ); TestRunner::add_node_with_config( - node_id, network.clone(), memberships, initializer, @@ -260,13 +259,12 @@ where node_id, 1, // For tests, make the node DA based on its index - node_id < config.da_staked_committee_size as u64, + node_id < config.known_da_nodes.len() as u64, ); let internal_chan = broadcast(EVENT_CHANNEL_SIZE); let context = TestRunner::::add_node_with_config_and_channels( - node_id, generated_network.clone(), (*memberships).clone(), initializer, diff --git a/crates/testing/src/test_builder.rs b/crates/testing/src/test_builder.rs index ccfce00fef..3813cf24e8 100644 --- a/crates/testing/src/test_builder.rs +++ b/crates/testing/src/test_builder.rs @@ -4,7 +4,7 @@ // You should have received a copy of the MIT License // along with the HotShot repository. If not, see . -use std::{collections::HashMap, num::NonZeroUsize, rc::Rc, sync::Arc, time::Duration}; +use std::{collections::HashMap, rc::Rc, sync::Arc, time::Duration}; use anyhow::{ensure, Result}; use hotshot::{ @@ -185,7 +185,7 @@ pub async fn create_test_handle< .unwrap(); // See whether or not we should be DA - let is_da = node_id < config.da_staked_committee_size as u64; + let is_da = node_id < config.known_da_nodes.len() as u64; let validator_config: ValidatorConfig = ValidatorConfig::generated_from_seed_indexed([0u8; 32], node_id, 1, is_da); @@ -202,7 +202,6 @@ pub async fn create_test_handle< .spawn_twin_handles( public_key, private_key, - node_id, config, memberships, network, @@ -221,7 +220,6 @@ pub async fn create_test_handle< .spawn_handle( public_key, private_key, - node_id, config, memberships, network, @@ -236,7 +234,6 @@ pub async fn create_test_handle< let hotshot = SystemContext::::new( public_key, private_key, - node_id, config, memberships, network, @@ -483,19 +480,16 @@ where ); // let da_committee_nodes = known_nodes[0..da_committee_size].to_vec(); let config = HotShotConfig { - start_threshold: (1, 1), - num_nodes_with_stake: NonZeroUsize::new(num_nodes_with_stake).unwrap(), + known_nodes: known_nodes_with_stake, // Currently making this zero for simplicity known_da_nodes, - known_nodes_with_stake, - da_staked_committee_size, fixed_leader_for_gpuvid: 1, next_view_timeout: 500, view_sync_timeout: Duration::from_millis(250), builder_timeout: Duration::from_millis(1000), data_request_delay: Duration::from_millis(200), // Placeholder until we spin up the builder - builder_urls: vec1::vec1![Url::parse("http://localhost:9999").expect("Valid URL")], + builder_urls: vec![Url::parse("http://localhost:9999").expect("Valid URL")], start_proposing_view: u64::MAX, stop_proposing_view: 0, start_voting_view: u64::MAX, diff --git a/crates/testing/src/test_runner.rs b/crates/testing/src/test_runner.rs index ffee9b39e5..02c32c25a6 100644 --- a/crates/testing/src/test_runner.rs +++ b/crates/testing/src/test_runner.rs @@ -334,7 +334,7 @@ where let builder_url = Url::parse(&format!("http://localhost:{builder_port}")).expect("Invalid URL"); let builder_task = B::start( - config.num_nodes_with_stake.into(), + config.known_nodes.len(), builder_url.clone(), B::Config::default(), metadata.changes.clone(), @@ -349,7 +349,7 @@ where Url::parse(&format!("http://localhost:{fallback_builder_port}")).expect("Invalid URL"); let fallback_builder_task = B::start( - config.num_nodes_with_stake.into(), + config.known_nodes.len(), fallback_builder_url.clone(), B::Config::default(), self.launcher.metadata.fallback_builder.changes.clone(), @@ -420,14 +420,10 @@ where tracing::debug!("launch node {}", i); let memberships = ::Membership::new( - config.known_nodes_with_stake.clone(), + config.known_nodes.clone(), config.known_da_nodes.clone(), ); - config.builder_urls = builder_urls - .clone() - .try_into() - .expect("Non-empty by construction"); - + config.builder_urls = builder_urls.clone(); let network = (self.launcher.resource_generator.channel_generator)(node_id).await; let storage = (self.launcher.resource_generator.storage)(node_id); let mut marketplace_config = @@ -474,14 +470,13 @@ where .unwrap(); // See whether or not we should be DA - let is_da = node_id < config.da_staked_committee_size as u64; + let is_da = node_id < config.known_da_nodes.len() as u64; // We assign node's public key and stake value rather than read from config file since it's a test let validator_config = ValidatorConfig::generated_from_seed_indexed([0u8; 32], node_id, 1, is_da); let hotshot = Self::add_node_with_config( - node_id, network.clone(), memberships, initializer, @@ -545,7 +540,7 @@ where ) .await; - match node_id.cmp(&(config.da_staked_committee_size as u64 - 1)) { + match node_id.cmp(&(config.known_da_nodes.len() as u64 - 1)) { std::cmp::Ordering::Less => { if let Some(task) = builder_tasks.pop() { task.start(Box::new(handle.event_stream())) @@ -575,7 +570,6 @@ where /// if unable to initialize the node's `SystemContext` based on the config #[allow(clippy::too_many_arguments)] pub async fn add_node_with_config( - node_id: u64, network: Network, memberships: TYPES::Membership, initializer: HotShotInitializer, @@ -591,7 +585,6 @@ where SystemContext::new( public_key, private_key, - node_id, config, memberships, network, @@ -608,7 +601,6 @@ where /// if unable to initialize the node's `SystemContext` based on the config #[allow(clippy::too_many_arguments, clippy::type_complexity)] pub async fn add_node_with_config_and_channels( - node_id: u64, network: Network, memberships: TYPES::Membership, initializer: HotShotInitializer, @@ -629,7 +621,6 @@ where SystemContext::new_from_channels( public_key, private_key, - node_id, config, memberships, network, diff --git a/crates/testing/tests/tests_1/gen_key_pair.rs b/crates/testing/tests/tests_1/gen_key_pair.rs deleted file mode 100644 index 301542a0a1..0000000000 --- a/crates/testing/tests/tests_1/gen_key_pair.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -#![allow(clippy::panic)] - -#[cfg(test)] -mod tests { - use core::panic; - use std::{env, fs::File, io::prelude::*}; - - use hotshot::types::{BLSPubKey, SignatureKey}; - use hotshot_types::{validator_config::ValidatorConfigFile, ValidatorConfig}; - #[test] - fn gen_key_pair_gen_from_config_file() { - let config_file = ValidatorConfigFile::from_file("config/ValidatorConfigFile.toml"); - let my_own_validator_config = ValidatorConfig::::from(config_file.clone()); - if config_file.seed == [0u8; 32] && config_file.node_id == 0 { - assert_eq!( - my_own_validator_config.public_key, - ::from_private(&my_own_validator_config.private_key) - ); - } - - let current_working_dir = match env::current_dir() { - Ok(dir) => dir, - Err(e) => { - panic!("get_current_working_dir error: {:?}", e); - } - }; - let filename = current_working_dir.into_os_string().into_string().unwrap() - + "/../../config/ValidatorConfigOutput"; - match File::create(filename) { - Err(why) => panic!("couldn't create file for output key pairs: {}", why), - Ok(mut file) => match write!(file, "{my_own_validator_config:?}",) { - Err(why) => panic!("couldn't generate key pairs and write to the file: {}", why), - Ok(()) => println!("successfully wrote to file for output key pairs"), - }, - } - } -} diff --git a/crates/testing/tests/tests_1/network_task.rs b/crates/testing/tests/tests_1/network_task.rs index a4e1c29a03..e93e43e4f8 100644 --- a/crates/testing/tests/tests_1/network_task.rs +++ b/crates/testing/tests/tests_1/network_task.rs @@ -56,7 +56,7 @@ async fn test_network_task() { let validator_config = launcher.resource_generator.validator_config.clone(); let public_key = validator_config.public_key; - let all_nodes = config.known_nodes_with_stake.clone(); + let all_nodes = config.known_nodes.clone(); let membership = ::Membership::new(all_nodes.clone(), all_nodes); let network_state: NetworkEventTaskState, _> = @@ -224,7 +224,7 @@ async fn test_network_storage_fail() { let config = launcher.resource_generator.config.clone(); let validator_config = launcher.resource_generator.validator_config.clone(); let public_key = validator_config.public_key; - let all_nodes = config.known_nodes_with_stake.clone(); + let all_nodes = config.known_nodes.clone(); let upgrade_lock = UpgradeLock::::new(); let membership = ::Membership::new(all_nodes.clone(), all_nodes); diff --git a/crates/testing/tests/tests_1/vote_dependency_handle.rs b/crates/testing/tests/tests_1/vote_dependency_handle.rs index 1b12e0b0f0..2b40074e41 100644 --- a/crates/testing/tests/tests_1/vote_dependency_handle.rs +++ b/crates/testing/tests/tests_1/vote_dependency_handle.rs @@ -96,7 +96,6 @@ async fn test_vote_dependency_handle() { sender: event_sender.clone(), receiver: event_receiver.clone().deactivate(), upgrade_lock: handle.hotshot.upgrade_lock.clone(), - id: handle.hotshot.id, epoch_height: handle.hotshot.config.epoch_height, }; diff --git a/crates/types/src/hotshot_config_file.rs b/crates/types/src/hotshot_config_file.rs index 16e000aa1b..f9a0b2623d 100644 --- a/crates/types/src/hotshot_config_file.rs +++ b/crates/types/src/hotshot_config_file.rs @@ -4,10 +4,9 @@ // You should have received a copy of the MIT License // along with the HotShot repository. If not, see . -use std::{num::NonZeroUsize, time::Duration}; +use std::time::Duration; use url::Url; -use vec1::Vec1; use crate::{ constants::REQUEST_DATA_DELAY, traits::signature_key::SignatureKey, @@ -15,32 +14,24 @@ use crate::{ }; /// Default builder URL, used as placeholder -fn default_builder_urls() -> Vec1 { - vec1::vec1![Url::parse("http://0.0.0.0:3311").unwrap()] +fn default_builder_urls() -> Vec { + vec![Url::parse("http://0.0.0.0:3311").unwrap()] } -/// Holds configuration for a `HotShot` +/// Contains configuration values for `HotShot` #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(bound(deserialize = ""))] pub struct HotShotConfigFile { - /// The proportion of nodes required before the orchestrator issues the ready signal, - /// expressed as (numerator, denominator) - pub start_threshold: (u64, u64), - /// Total number of staked nodes in the network - pub num_nodes_with_stake: NonZeroUsize, - #[serde(skip)] - /// The known nodes' public key and stake value - pub known_nodes_with_stake: Vec>, - #[serde(skip)] + /// The known nodes' public key and stake values + #[serde(rename = "known_nodes", alias = "known_nodes_with_stake")] + pub known_nodes: Vec>, /// The known DA nodes' public key and stake values pub known_da_nodes: Vec>, - /// Number of staking DA nodes - pub staked_da_nodes: usize, /// Number of fixed leaders for GPU VID pub fixed_leader_for_gpuvid: usize, - /// Base duration for next-view timeout, in milliseconds + /// The duration for the timeout before the next view, in milliseconds pub next_view_timeout: u64, - /// Duration for view sync round timeout + /// The duration for view sync round timeout pub view_sync_timeout: Duration, /// The maximum amount of time a leader can wait to get a block from a builder pub builder_timeout: Duration, @@ -48,7 +39,7 @@ pub struct HotShotConfigFile { pub data_request_delay: Option, /// Builder API base URL #[serde(default = "default_builder_urls")] - pub builder_urls: Vec1, + pub builder_urls: Vec, /// Upgrade config pub upgrade: UpgradeConfig, /// Number of blocks in an epoch, zero means there are no epochs @@ -58,11 +49,8 @@ pub struct HotShotConfigFile { impl From> for HotShotConfig { fn from(val: HotShotConfigFile) -> Self { HotShotConfig { - start_threshold: val.start_threshold, - num_nodes_with_stake: val.num_nodes_with_stake, known_da_nodes: val.known_da_nodes, - known_nodes_with_stake: val.known_nodes_with_stake, - da_staked_committee_size: val.staked_da_nodes, + known_nodes: val.known_nodes, fixed_leader_for_gpuvid: val.fixed_leader_for_gpuvid, next_view_timeout: val.next_view_timeout, view_sync_timeout: val.view_sync_timeout, @@ -111,10 +99,7 @@ impl HotShotConfigFile { .collect(); Self { - num_nodes_with_stake: NonZeroUsize::new(10).unwrap(), - start_threshold: (1, 1), - known_nodes_with_stake: gen_known_nodes_with_stake, - staked_da_nodes, + known_nodes: gen_known_nodes_with_stake, known_da_nodes, fixed_leader_for_gpuvid: 1, next_view_timeout: 10000, diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 0f2c390e10..1278b897ff 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -5,7 +5,7 @@ // along with the HotShot repository. If not, see . //! Types and Traits for the `HotShot` consensus module -use std::{fmt::Debug, future::Future, num::NonZeroUsize, pin::Pin, time::Duration}; +use std::{fmt::Debug, future::Future, pin::Pin, time::Duration}; use bincode::Options; use displaydoc::Display; @@ -13,7 +13,6 @@ use light_client::StateVerKey; use tracing::error; use traits::signature_key::SignatureKey; use url::Url; -use vec1::Vec1; use crate::utils::bincode_opts; pub mod bundle; @@ -45,8 +44,6 @@ pub mod builder; /// Holds the upgrade configuration specification for HotShot nodes. pub mod upgrade_config; pub mod utils; -/// Holds the validator configuration specification for HotShot nodes. -pub mod validator_config; pub mod vid; pub mod vote; @@ -167,30 +164,23 @@ impl Default for PeerConfig { #[derive(Clone, derive_more::Debug, serde::Serialize, serde::Deserialize)] #[serde(bound(deserialize = ""))] pub struct HotShotConfig { - /// The proportion of nodes required before the orchestrator issues the ready signal, - /// expressed as (numerator, denominator) - pub start_threshold: (u64, u64), - /// Total number of nodes in the network - // Earlier it was total_nodes - pub num_nodes_with_stake: NonZeroUsize, /// List of known node's public keys and stake value for certificate aggregation, serving as public parameter - pub known_nodes_with_stake: Vec>, + #[serde(rename = "known_nodes", alias = "known_nodes_with_stake")] + pub known_nodes: Vec>, /// All public keys known to be DA nodes pub known_da_nodes: Vec>, - /// List of DA committee (staking)nodes for static DA committee - pub da_staked_committee_size: usize, /// Number of fixed leaders for GPU VID, normally it will be 0, it's only used when running GPU VID pub fixed_leader_for_gpuvid: usize, - /// Base duration for next-view timeout, in milliseconds + /// The duration for the timeout before the next view, in milliseconds pub next_view_timeout: u64, - /// Duration of view sync round timeouts + /// The duration for view sync round timeouts pub view_sync_timeout: Duration, /// The maximum amount of time a leader can wait to get a block from a builder pub builder_timeout: Duration, - /// time to wait until we request data associated with a proposal + /// Time to wait until we request data associated with a proposal pub data_request_delay: Duration, /// Builder API base URL - pub builder_urls: Vec1, + pub builder_urls: Vec, /// View to start proposing an upgrade pub start_proposing_view: u64, /// View to stop proposing an upgrade. To prevent proposing an upgrade, set stop_proposing_view <= start_proposing_view. diff --git a/crates/types/src/traits/consensus_api.rs b/crates/types/src/traits/consensus_api.rs index 4c4daff9a7..31bceaead5 100644 --- a/crates/types/src/traits/consensus_api.rs +++ b/crates/types/src/traits/consensus_api.rs @@ -6,7 +6,7 @@ //! Contains the [`ConsensusApi`] trait. -use std::{num::NonZeroUsize, time::Duration}; +use std::time::Duration; use async_trait::async_trait; @@ -23,7 +23,7 @@ use crate::{ #[async_trait] pub trait ConsensusApi>: Send + Sync { /// Total number of nodes in the network. Also known as `n`. - fn total_nodes(&self) -> NonZeroUsize; + fn total_nodes(&self) -> usize; /// The maximum amount of time a leader can wait to get a block from a builder. fn builder_timeout(&self) -> Duration; diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index 5b72ea4f84..a01fa444f2 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -7,13 +7,16 @@ //! The election trait, used to decide which node is the leader and determine if a vote is valid. use std::{collections::BTreeSet, fmt::Debug, num::NonZeroU64}; +use serde::{Deserialize, Serialize}; use utils::anytrace::Result; use super::node_implementation::NodeType; use crate::{traits::signature_key::SignatureKey, PeerConfig}; /// A protocol for determining membership in and participating in a committee. -pub trait Membership: Clone + Debug + Send + Sync { +pub trait Membership: + Clone + Debug + Send + Sync + Serialize + for<'de> Deserialize<'de> +{ /// The error type returned by methods like `lookup_leader`. type Error: std::fmt::Display; diff --git a/crates/types/src/validator_config.rs b/crates/types/src/validator_config.rs deleted file mode 100644 index 2622c0d23a..0000000000 --- a/crates/types/src/validator_config.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -use std::{env, fs, path::PathBuf}; - -use toml; -use tracing::error; - -use crate::{traits::signature_key::SignatureKey, ValidatorConfig}; - -/// Holds configuration for a validator node -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] -#[serde(bound(deserialize = ""))] -pub struct ValidatorConfigFile { - /// The validator's seed - pub seed: [u8; 32], - /// The validator's index, which can be treated as another input to the seed - pub node_id: u64, - // The validator's stake, commented for now - // pub stake_value: u64, - /// Whether or not we are DA - pub is_da: bool, -} - -impl ValidatorConfigFile { - /// read the validator config from a file - /// # Panics - /// Panics if unable to get the current working directory - pub fn from_file(dir_str: &str) -> Self { - let current_working_dir = match env::current_dir() { - Ok(dir) => dir, - Err(e) => { - error!("get_current_working_dir error: {:?}", e); - PathBuf::from("") - } - }; - let filename = - current_working_dir.into_os_string().into_string().unwrap() + "/../../" + dir_str; - let contents = fs::read_to_string(filename.clone()).expect("Could not read file"); - let data: ValidatorConfigFile = - toml::from_str(&contents).expect("Unable to load data from file"); - data - } -} - -impl From for ValidatorConfig { - fn from(val: ValidatorConfigFile) -> Self { - // here stake_value is set to 1, since we don't input stake_value from ValidatorConfigFile for now - ValidatorConfig::generated_from_seed_indexed(val.seed, val.node_id, 1, val.is_da) - } -} From ca2f647cc1e9a741bf8450b1737da66f64069fda Mon Sep 17 00:00:00 2001 From: Rob Date: Mon, 9 Dec 2024 17:10:36 -0500 Subject: [PATCH 11/16] refactor --- CHANGELOG.md | 45 ---- Cargo.lock | 5 + Cargo.toml | 1 + README.md | 228 ++---------------- crates/examples/Cargo.toml | 24 ++ crates/examples/{src/main.rs => all.rs} | 18 +- crates/examples/cdn/broker.rs | 105 ++++++++ crates/examples/cdn/marshal.rs | 68 ++++++ crates/examples/common.rs | 32 +++ crates/examples/coordinator.rs | 131 ++++++++++ crates/examples/src/lib.rs | 1 - crates/hotshot/Cargo.toml | 2 +- crates/task-impls/src/da.rs | 2 +- docker/cdn-broker.Dockerfile | 18 -- docker/cdn-marshal.Dockerfile | 18 -- docker/orchestrator.Dockerfile | 18 -- docker/validator-cdn-local.Dockerfile | 16 -- docker/validator-cdn.Dockerfile | 18 -- docker/validator-combined.Dockerfile | 18 -- docker/validator-libp2p.Dockerfile | 18 -- hotshot-emoji.png | Bin 0 -> 54333 bytes .../aws_ecs_benchmarks_cdn.sh | 147 ----------- .../aws_ecs_benchmarks_cdn_gpu.sh | 168 ------------- .../benchmarks_start_cdn_broker.sh | 19 -- .../benchmarks_start_leader_gpu.sh | 19 -- scripts/benchmarks_results/README.md | 19 -- scripts/benchmarks_results/results_upload.csv | 160 ------------ scripts/count_fds.sh | 9 - scripts/nix_bump_pr_changes.py | 27 --- scripts/runfail.sh | 11 - 30 files changed, 394 insertions(+), 971 deletions(-) delete mode 100644 CHANGELOG.md rename crates/examples/{src/main.rs => all.rs} (97%) create mode 100644 crates/examples/cdn/broker.rs create mode 100644 crates/examples/cdn/marshal.rs create mode 100644 crates/examples/common.rs create mode 100644 crates/examples/coordinator.rs delete mode 100644 crates/examples/src/lib.rs delete mode 100644 docker/cdn-broker.Dockerfile delete mode 100644 docker/cdn-marshal.Dockerfile delete mode 100644 docker/orchestrator.Dockerfile delete mode 100644 docker/validator-cdn-local.Dockerfile delete mode 100644 docker/validator-cdn.Dockerfile delete mode 100644 docker/validator-combined.Dockerfile delete mode 100644 docker/validator-libp2p.Dockerfile create mode 100644 hotshot-emoji.png delete mode 100755 scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh delete mode 100755 scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh delete mode 100755 scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh delete mode 100755 scripts/benchmark_scripts/benchmarks_start_leader_gpu.sh delete mode 100644 scripts/benchmarks_results/README.md delete mode 100644 scripts/benchmarks_results/results_upload.csv delete mode 100755 scripts/count_fds.sh delete mode 100755 scripts/nix_bump_pr_changes.py delete mode 100755 scripts/runfail.sh diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 8bcdeb6baf..0000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,45 +0,0 @@ - -## [0.0.4] - 2021-11-01 -### Features -- Downgrade possibly-temporary network faults to warnings -- Improve logging when an invalid transaction is submitted - - - -## [0.0.3] - 2021-10-27 -### Features -- Implement janky catchup - -### BREAKING CHANGE - -Adds new type parameter, corresponding to the state type, to Message - - -## [0.0.2] - 2021-10-19 -### Bug Fixes -- Fix leaders not sending themselves commit votes -- Fix state not getting stored properly - -### Features -- StatefulHandler trait -- Reexport traits from traits module -- State Machine + Node Implementation -- state machine mvp megasquash -- Replace tokio broadcast queue with unbounded equivalent - -### BREAKING CHANGE - -Changes queue type in hotshot methods - - - -## [0.0.1] - 2021-08-20 - - -## 0.0.0 - 2021-07-07 - -[Unreleased]: https://github.com/EspressoSystems/hotshot/compare/0.0.4...HEAD -[0.0.4]: https://github.com/EspressoSystems/hotshot/compare/0.0.3...0.0.4 -[0.0.3]: https://github.com/EspressoSystems/hotshot/compare/0.0.2...0.0.3 -[0.0.2]: https://github.com/EspressoSystems/hotshot/compare/0.0.1...0.0.2 -[0.0.1]: https://github.com/EspressoSystems/hotshot/compare/0.0.0...0.0.1 diff --git a/Cargo.lock b/Cargo.lock index 9598cabbeb..cd7e65424d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2925,6 +2925,7 @@ name = "hotshot-examples" version = "0.5.79" dependencies = [ "anyhow", + "bytes", "cdn-broker", "cdn-marshal", "clap", @@ -2936,11 +2937,15 @@ dependencies = [ "libp2p", "libp2p-networking", "lru 0.12.5", + "parking_lot", "portpicker", "rand 0.8.5", + "reqwest", + "sha2 0.10.8", "tokio", "tracing", "url", + "warp", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bd28b52311..4deaac0398 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,7 @@ clap = { version = "4", features = ["derive", "env"] } url = { version = "2", features = ["serde"] } vec1 = { version = "1", features = ["serde"] } reqwest = { version = "0.12", features = ["json"] } +parking_lot = "0.12" libp2p = { package = "libp2p", version = "0.53", default-features = false, features = [ "macros", diff --git a/README.md b/README.md index dc1b35ed97..24d7724b4a 100644 --- a/README.md +++ b/README.md @@ -1,229 +1,31 @@ ![GitHub Release](https://img.shields.io/github/v/release/EspressoSystems/HotShot) +# HotShot HotShot Emoji -# License -## Copyright -**(c) 2021-2024 Espresso Systems**. -`HotShot` was developed by Espresso Systems. +HotShot is a Byzantine Fault Tolerant (BFT) consensus protocol that builds upon HotStuff 2. It is modified for proof-of-stake settings and features a linear view-synchronization protocol and a data-availability layer. -# HotShot Consensus Module +## Paper +The HotShot protocol is described in the [Espresso Sequencing Network paper](https://eprint.iacr.org/2024/1189.pdf). -HotShot is a BFT consensus protocol based off of HotStuff, with the addition of proof-of-stake and -VRF committee elections. +## Usage and Examples -## Disclaimer - -**DISCLAIMER:** This software is provided "as is" and its security has not been externally audited. Use at your own risk. - -# Usage - -Please see the rustdoc for API documentation, and the examples directory for usage. - -## Dependencies - -### Unix-like - -#### Nix (macos and linux) - -``` -nix develop -``` - -#### Brew (macos) - -``` -brew install cmake protobuf -``` - -#### Apt-get (linux) - -``` -apt-get install cmake protobuf -``` - -### Windows - -#### Chocolatey - -``` -choco install cmake protoc -``` - -#### Scoop - -``` -scoop bucket add extras -scoop install protobuf cmake -``` - -## Building - -Once dependencies have been installed, to build everything: - -```sh -just build -``` - - - -# Static linking - -HotShot supports static linking for its examples: - -```sh -# Nix-shell is optional but recommended -nix develop .#staticShell - -just build -``` - -# Testing - -To test: - -```sh -RUST_LOG=$ERROR_LOG_LEVEL RUST_LOG_FORMAT=$ERROR_LOG_FORMAT just test -``` - -- `RUST_LOG=$ERROR_LOG_LEVEL`: The basic levels of logging include `warn`, `error`, `info`. -- `RUST_LOG_FORMAT=$ERROR_LOG_FORMAT`: The types of logging include `full`, `json`, and `compact`. -- Internally, the inclusion of the `--nocapture` flag indicates whether or not to output logs. -- Internally, we run at `--test-threads=1` because the tests spawn up a lot of file handles, and unix based systems consistently run out of handles. - -To stress test, run the ignored tests prefixed with `test_stress`: -```sh -RUST_LOG=$ERROR_LOG_LEVEL RUST_LOG_FORMAT=$ERROR_LOG_FORMAT just run_test test_stress -``` - -## Careful - -To double-check for UB: - -```bash -nix develop .#correctnessShell -just careful -``` - -## Testing on CI - -To test as if running on CI, one must limit the number of cores and ram to match github runners (2 core, 7 gig ram). To limit the ram, spin up a virtual machine or container with 7 gigs ram. To limit the core count when running tests: - -``` -ASYNC_STD_THREAD_COUNT=1 RUST_LOG=$ERROR_LOG_LEVEL RUST_LOG_FORMAT=$ERROR_LOG_FORMAT just tokio test -``` - -# Tokio-console - -To use tokio-console, drop into the console shell: - -``` -nix develop .#consoleShell -``` - -Then, run an example. - -On a separate terminal, also drop into the console shell and start tokio-console: -``` -nix develop .#consoleShell -c tokio-console -``` - -This second window should now display task usage. - -# Open Telemetry + Jaeger Integration - -To view distributed logs with just the centralized server and one client, first edit the `centralized_server/orchestrator` file to include have a threshold and num_nodes of 1. - -Then open 3 terminals. +Usage examples are provided in the [examples directory](./crates/examples). +### Running the examples +To run a full example network, use the command ```bash -# Terminal 1 -# Start the jaeger instance to view spans -docker run -d -p6831:6831/udp -p6832:6832/udp -p16686:16686 -p14268:14268 jaegertracing/all-in-one:latest - -# Terminal 2 -# Start the CDN - -# Terminal 3 -# Start the client +RUST_LOG=info cargo run --example all ``` -# Resource Usage Statistics - -To generate usage stats: -- build the test suite -- find the executable containing the test of interest -- run profiling tools - -The executable `cargo` uses is shown in the output of `cargo test`. - -For example, to profile `test_stress_dht_many_round`: - +You can see the list of supported command-line arguments by running ```bash -# bring profiling tooling like flamegraph and heaptrack into scope -nix develop .#perfShell - -# show the executable we need run -# and build all test executables (required for subsequent steps) -cargo test --verbose --release --lib --bins --tests --benches --workspace -- --test-threads=1 -# the output cargo test contains the tests path: -# Running `/home/jrestivo/work/crosscross/target/release/deps/counter-880b1ff53ee21dea test_stress --test-threads=1 --ignored` -# running 7 tests -# test test_stress_dht_many_rounds ... ok -# ... - -# a more detailed alternative to flamegraph -# NOTE: only works on linux -heaptrack $(fd -I "counter*" -t x | rg release) --ignored -- test_stress_dht_many_round --nocapture -# palette provides memory statistics, omission will provide cpu cycle stats as colors -# NOTE: must be run as root on macos -flamegraph --palette=mem $(fd -I "counter*" -t x | rg release) --ignored -- test_stress_dht_one_round -# code coveragte statistics -cargo-llvm-cov llvm-cov --test=test_stress_dht_many_round --workspace --all-targets --release --html --output-path lcov.html -``` - -This will output: -- `heaptrack.counter-$HASH` which is viewable by heaptrack. This provides a plethora of useful statistics about memory and cpu cycles. -- `flamegraph.svg` which is a (moderately) less detailed version of heaptrack. -- `lcov.html` generates a summary of code coverage. - -# Debugging - -A debugging config file is provided for vscode and vscodium in [`.vscode/launch.json`](https://github.com/EspressoSystems/HotShot/blob/main/.cargo/config). This is intended to be used with [vadimcn/vscode-lldb](https://open-vsx.org/extension/vadimcn/vscode-lldb) but may work with other rust debuggers as well. - -To bring `lldb` into scope with nix, run `nix develop .#debugShell`. - -# Git Workflow - -For espresso developers we have written up a description of our workflow [here](./WORKFLOW.md). - -# Extra Editor Configuration - -Choose an async runtime to use before launching a text editor. This may be done by setting the environment RUSTFLAGS. For example: - -``` -nvim # launch text editor of choice. We choose neovim in this example -unset RUSTFLAGS # Unset rustflags so we may continue to use the justfile. The justfile sets these particular config options -``` - -# Debugging - -We support the [CodeLLDB Debugger](https://github.com/vadimcn/vscode-lldb). - -## Neovim - -Install [`dap`](https://github.com/mfussenegger/nvim-dap) and [`rust-tools`](https://github.com/simrat39/rust-tools.nvim). Install the CodeLLDB debugger listed above. -Follow the instructions [here](https://github.com/mfussenegger/nvim-dap/discussions/671#discussioncomment-4286738) to configure the adapter. To add our project-local configurations, run: - +cargo run --example all -- --help ``` -lua require('dap.ext.vscode').load_launchjs(nil, { ["codelldb"] = {"rust"} }) -``` - -Finally, place a breakpoint and run `:DapContinue` to begin debugging. -NOTE: Do NOT configure dap at all with rust-tools. Do it manually. +## Audits +The HotShot protocol has been internally audited. The report is available [here](./audits/internal-reviews/EspressoHotshot-2024internal.pdf). -[Example configuration](https://github.com/DieracDelta/vimconfig/blob/master/modules/lsp.nix#L280). +## Disclaimer -## Vscode +**DISCLAIMER:** This software is provided "as is" and its security has not been **externally** audited. Use at your own risk. -Install the extension and load the `launch.json` file. Then run the desired test target. diff --git a/crates/examples/Cargo.toml b/crates/examples/Cargo.toml index 5c15ed9330..e75b714e99 100644 --- a/crates/examples/Cargo.toml +++ b/crates/examples/Cargo.toml @@ -26,6 +26,30 @@ hotshot-testing.path = "../testing" lru.workspace = true cdn-broker.workspace = true cdn-marshal.workspace = true +sha2.workspace = true +warp = { version = "0.3", default-features = false } +reqwest.workspace = true +bytes = "1" +parking_lot.workspace = true +[[example]] +name = "all" +path = "all.rs" + +[[example]] +name = "cdn-broker" +path = "cdn/broker.rs" + +[[example]] +name = "cdn-marshal" +path = "cdn/marshal.rs" + +[[example]] +name = "coordinator" +path = "coordinator.rs" + +# [[example]] +# name = "single-validator" +# path = "single-validator.rs" [lints] workspace = true diff --git a/crates/examples/src/main.rs b/crates/examples/all.rs similarity index 97% rename from crates/examples/src/main.rs rename to crates/examples/all.rs index 3d9e937c83..787cf7ee7b 100644 --- a/crates/examples/src/main.rs +++ b/crates/examples/all.rs @@ -1,4 +1,5 @@ -//! This crate contains examples of HotShot usage +//! This file contains an example of running a full HotShot network, comprised of +//! a CDN, Libp2p, a builder, and multiple validators. use std::{collections::HashMap, num::NonZero, sync::Arc, time::Duration}; use anyhow::{Context, Result}; @@ -45,7 +46,7 @@ use tokio::{spawn, sync::OnceCell, task::JoinSet}; use tracing::{error, info}; use url::Url; -/// The command line arguments for the example +/// This example runs all necessary HotShot components #[derive(Parser)] struct Args { /// The number of nodes to start @@ -131,10 +132,13 @@ async fn start_cdn() -> Result { portpicker::pick_unused_port().with_context(|| "Failed to find unused port")?; let marshal_address = format!("127.0.0.1:{marshal_port}"); + // Generate a random file path for the SQLite database + let db_path = format!("/tmp/marshal-{}.db", rand::random::()); + // Configure the marshal let marshal_config = MarshalConfig { bind_endpoint: marshal_address.clone(), - discovery_endpoint: marshal_address.clone(), + discovery_endpoint: db_path.clone(), ca_cert_path: None, ca_key_path: None, metrics_bind_endpoint: None, @@ -169,7 +173,7 @@ async fn start_cdn() -> Result { public_bind_endpoint: public_address, private_advertise_endpoint: private_address.clone(), private_bind_endpoint: private_address, - discovery_endpoint: marshal_address.clone(), + discovery_endpoint: db_path.clone(), keypair: KeyPair { public_key: WrappedSignatureKey(broker_public_key), private_key: broker_private_key.clone(), @@ -388,11 +392,11 @@ macro_rules! start_node_with_network { // If we cached the size of the proposed block, log it if let Some(size) = proposed_block_size_cache.get(&*qc.view_number) { info!( - "Decided on view number {}. Block size was {}", - qc.view_number, size + block_size = size, + "Decided on view {}", *qc.view_number ); } else { - info!("Decided on view number {}.", qc.view_number); + info!("Decided on view {}", *qc.view_number); } // If the view number is divisible by the node's index, submit transactions diff --git a/crates/examples/cdn/broker.rs b/crates/examples/cdn/broker.rs new file mode 100644 index 0000000000..4e4669983c --- /dev/null +++ b/crates/examples/cdn/broker.rs @@ -0,0 +1,105 @@ +//! The broker is the message-routing component of the CDN +//! +//! This is meant to be run externally, e.g. when running benchmarks on the protocol. +//! If you just want to run everything required, you can use the `all` example + +use anyhow::{Context, Result}; +use cdn_broker::{reexports::def::hook::NoMessageHook, Broker, Config}; +use clap::Parser; +use hotshot::{ + helpers::initialize_logging, + traits::implementations::{KeyPair, ProductionDef, WrappedSignatureKey}, +}; +use hotshot_example_types::node_types::TestTypes; +use hotshot_types::traits::node_implementation::NodeType; +use hotshot_types::traits::signature_key::BuilderSignatureKey; +use sha2::Digest; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +/// The main component of the push CDN. +struct Args { + /// The discovery client endpoint (including scheme) to connect to. + /// This is a URL pointing to a `KeyDB` database (e.g. `redis://127.0.0.1:6789`). + #[arg(short, long)] + discovery_endpoint: String, + + /// The user-facing endpoint in `IP:port` form to bind to for connections from users + #[arg(long, default_value = "0.0.0.0:1738")] + public_bind_endpoint: String, + + /// The user-facing endpoint in `IP:port` form to advertise + #[arg(long, default_value = "local_ip:1738")] + public_advertise_endpoint: String, + + /// The broker-facing endpoint in `IP:port` form to bind to for connections from + /// other brokers + #[arg(long, default_value = "0.0.0.0:1739")] + private_bind_endpoint: String, + + /// The broker-facing endpoint in `IP:port` form to advertise + #[arg(long, default_value = "local_ip:1739")] + private_advertise_endpoint: String, + + /// The endpoint to bind to for externalizing metrics (in `IP:port` form). If not provided, + /// metrics are not exposed. + #[arg(short, long)] + metrics_bind_endpoint: Option, + + /// The path to the CA certificate + /// If not provided, a local, pinned CA is used + #[arg(long)] + ca_cert_path: Option, + + /// The path to the CA key + /// If not provided, a local, pinned CA is used + #[arg(long)] + ca_key_path: Option, + + /// The seed for broker key generation + #[arg(short, long, default_value_t = 0)] + key_seed: u64, +} + +#[tokio::main] +async fn main() -> Result<()> { + // Initialize logging + initialize_logging(); + + // Parse the command line arguments + let args = Args::parse(); + + // Generate the broker key from the supplied seed + let key_hash = sha2::Sha256::digest(args.key_seed.to_le_bytes()); + let (public_key, private_key) = + ::SignatureKey::generated_from_seed_indexed(key_hash.into(), 1337); + + // Create config + let broker_config: Config::SignatureKey>> = Config { + ca_cert_path: args.ca_cert_path, + ca_key_path: args.ca_key_path, + + discovery_endpoint: args.discovery_endpoint, + metrics_bind_endpoint: args.metrics_bind_endpoint, + keypair: KeyPair { + public_key: WrappedSignatureKey(public_key), + private_key, + }, + + user_message_hook: NoMessageHook, + broker_message_hook: NoMessageHook, + + public_bind_endpoint: args.public_bind_endpoint, + public_advertise_endpoint: args.public_advertise_endpoint, + private_bind_endpoint: args.private_bind_endpoint, + private_advertise_endpoint: args.private_advertise_endpoint, + // Use a 1GB memory pool size + global_memory_pool_size: Some(1_073_741_824), + }; + + // Create the new `Broker` + let broker = Broker::new(broker_config).await?; + + // Run the broker until it is terminated + broker.start().await.with_context(|| "Broker exited") +} diff --git a/crates/examples/cdn/marshal.rs b/crates/examples/cdn/marshal.rs new file mode 100644 index 0000000000..f4a5d0d969 --- /dev/null +++ b/crates/examples/cdn/marshal.rs @@ -0,0 +1,68 @@ +//! The marshal is the component of the CDN that authenticates users and routes +//! them to the appropriate broker +//! +//! This is meant to be run externally, e.g. when running benchmarks on the protocol. +//! If you just want to run everything required, you can use the `all` example + +use anyhow::{Context, Result}; +use cdn_marshal::{Config, Marshal}; +use clap::Parser; +use hotshot::{helpers::initialize_logging, traits::implementations::ProductionDef}; +use hotshot_example_types::node_types::TestTypes; +use hotshot_types::traits::node_implementation::NodeType; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +/// The main component of the push CDN. +struct Args { + /// The discovery client endpoint (including scheme) to connect to. + /// This is a URL pointing to a `KeyDB` database (e.g. `redis://127.0.0.1:6789`). + #[arg(short, long)] + discovery_endpoint: String, + + /// The port to bind to for connections (from users) + #[arg(short, long, default_value_t = 1737)] + bind_port: u16, + + /// The endpoint to bind to for externalizing metrics (in `IP:port` form). If not provided, + /// metrics are not exposed. + #[arg(short, long)] + metrics_bind_endpoint: Option, + + /// The path to the CA certificate + /// If not provided, a local, pinned CA is used + #[arg(long)] + ca_cert_path: Option, + + /// The path to the CA key + /// If not provided, a local, pinned CA is used + #[arg(long)] + ca_key_path: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + // Parse command-line arguments + let args = Args::parse(); + + // Initialize tracing + initialize_logging(); + + // Create a new `Config` + let config = Config { + discovery_endpoint: args.discovery_endpoint, + bind_endpoint: format!("0.0.0.0:{}", args.bind_port), + metrics_bind_endpoint: args.metrics_bind_endpoint, + ca_cert_path: args.ca_cert_path, + ca_key_path: args.ca_key_path, + // Use a 1GB memory pool + global_memory_pool_size: Some(1_073_741_824), + }; + + // Create new `Marshal` from the config + let marshal = + Marshal::::SignatureKey>>::new(config).await?; + + // Start the main loop, consuming it + marshal.start().await.with_context(|| "Marshal exited") +} diff --git a/crates/examples/common.rs b/crates/examples/common.rs new file mode 100644 index 0000000000..dc8cdbdd33 --- /dev/null +++ b/crates/examples/common.rs @@ -0,0 +1,32 @@ +/// The type of network to use for the example +#[derive(Debug, PartialEq, Eq)] +enum NetworkType { + /// A combined network, which is a combination of a Libp2p and Push CDN network + Combined, + + /// A network solely using the Push CDN + Cdn, + + /// A Libp2p network + LibP2P, +} + +/// This is a testing function which allows us to easily determine if a node should be a DA node +fn is_da_node(index: usize, num_da_nodes: usize) -> bool { + index < num_da_nodes +} + +/// This is a testing function which allows us to easily generate peer configs from indexes +fn peer_info_from_index(index: usize) -> hotshot_types::PeerConfig { + // Get the node's public key + let (public_key, _) = + hotshot::types::BLSPubKey::generated_from_seed_indexed([0u8; 32], index as u64); + + // Generate the peer config + hotshot_types::PeerConfig { + stake_table_entry: public_key.stake_table_entry(1), + state_ver_key: hotshot_types::light_client::StateKeyPair::default() + .0 + .ver_key(), + } +} diff --git a/crates/examples/coordinator.rs b/crates/examples/coordinator.rs new file mode 100644 index 0000000000..1756ba6467 --- /dev/null +++ b/crates/examples/coordinator.rs @@ -0,0 +1,131 @@ +//! This service helps coordinate running multiple nodes where each needs +//! to be assigned a unique index +//! +//! This is meant to be run externally, e.g. when running benchmarks on the protocol. +//! If you just want to run everything required, you can use the `all` example + +use std::{ + collections::HashSet, + net::SocketAddr, + str::FromStr, + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, +}; + +use anyhow::{Context, Result}; +use bytes::Bytes; +use clap::Parser; +use hotshot::helpers::initialize_logging; +use libp2p::{multiaddr::Protocol, Multiaddr}; +use parking_lot::RwLock; +use warp::Filter; + +#[derive(Parser)] +struct Args { + /// The address to bind to + #[arg(long, default_value = "127.0.0.1:3030")] + bind_address: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + // Initialize logging + initialize_logging(); + + // Parse the command-line arguments + let args = Args::parse(); + + // Parse the bind address + let bind_address = args + .bind_address + .parse::() + .with_context(|| "Failed to parse bind address")?; + + // Create a shared counter + let counter = Arc::new(AtomicU32::new(0)); + let counter = warp::any().map(move || counter.clone()); + + // Create a shared set of multiaddrs for Libp2p + let libp2p_multiaddrs = Arc::new(RwLock::new(HashSet::new())); + let libp2p_multiaddrs = warp::any().map(move || libp2p_multiaddrs.clone()); + + // `/index` returns the node index we are assigned + let index = warp::path!("index") + .and(counter.clone()) + .map(|counter: Arc| counter.fetch_add(1, Ordering::SeqCst).to_string()); + + // POST `/libp2p_info` submits libp2p information to the coordinator + let submit_libp2p_info = warp::path!("libp2p-info") + .and(warp::post()) + .and(warp::body::bytes()) + .and(libp2p_multiaddrs.clone()) + .map( + |body: Bytes, libp2p_multiaddrs: Arc>>| { + // Attempt to process as a string + let Ok(string) = String::from_utf8(body.to_vec()) else { + return "Failed to parse body as string".to_string(); + }; + + // Attempt to parse the string as a Libp2p Multiaddr + let Ok(mut multiaddr) = Multiaddr::from_str(&string) else { + return "Failed to parse body as Multiaddr".to_string(); + }; + + // Pop off the last protocol + let Some(last_protocol) = multiaddr.pop() else { + return "Failed to get last protocol of multiaddr".to_string(); + }; + + // Make sure it is the P2p protocol + let Protocol::P2p(_) = last_protocol else { + return "Failed to get P2p protocol of multiaddr".to_string(); + }; + + // Add it to the set + libp2p_multiaddrs.write().insert(multiaddr); + + "Ok".to_string() + }, + ); + + // GET `/libp2p_info` returns the list of libp2p multiaddrs + let get_libp2p_info = warp::path!("libp2p-info") + .and(libp2p_multiaddrs.clone()) + .map(|libp2p_multiaddrs: Arc>>| { + // Get the list of multiaddrs + let multiaddrs = libp2p_multiaddrs.read().clone(); + + // Convert the multiaddrs to a string, separated by newlines + multiaddrs + .iter() + .map(|m| m.to_string()) + .collect::>() + .join("\n") + }); + + // `/reset` resets the state of the coordinator + let reset = warp::path!("reset") + .and(counter) + .and(libp2p_multiaddrs) + .map( + |counter: Arc, libp2p_multiaddrs: Arc>>| { + counter.store(0, Ordering::SeqCst); + libp2p_multiaddrs.write().clear(); + "Ok" + }, + ); + + // Run the server + warp::serve( + index + .or(reset) + .or(submit_libp2p_info) + .or(get_libp2p_info), + ) + .run(bind_address) + .await; + + Ok(()) +} diff --git a/crates/examples/src/lib.rs b/crates/examples/src/lib.rs deleted file mode 100644 index a7001964d2..0000000000 --- a/crates/examples/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -//! This crate contains examples of HotShot usage diff --git a/crates/hotshot/Cargo.toml b/crates/hotshot/Cargo.toml index 75aa2384e3..be270cdced 100644 --- a/crates/hotshot/Cargo.toml +++ b/crates/hotshot/Cargo.toml @@ -41,7 +41,7 @@ libp2p-identity = { workspace = true } libp2p-networking = { workspace = true } lru = { workspace = true } num_enum = "0.7" -parking_lot = "0.12" +parking_lot = { workspace = true } portpicker = "0.1" primitive-types = { workspace = true } rand = { workspace = true } diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index 6be0d623ae..f25d7012c2 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -286,7 +286,7 @@ impl, V: Versions> DaTaskState 1 { diff --git a/docker/cdn-broker.Dockerfile b/docker/cdn-broker.Dockerfile deleted file mode 100644 index 348c7355d5..0000000000 --- a/docker/cdn-broker.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/cdn-broker /usr/local/bin/cdn-broker - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["cdn-broker"] diff --git a/docker/cdn-marshal.Dockerfile b/docker/cdn-marshal.Dockerfile deleted file mode 100644 index 9ea1b497c1..0000000000 --- a/docker/cdn-marshal.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/cdn-marshal /usr/local/bin/cdn-marshal - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["cdn-marshal"] diff --git a/docker/orchestrator.Dockerfile b/docker/orchestrator.Dockerfile deleted file mode 100644 index eda3846754..0000000000 --- a/docker/orchestrator.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/orchestrator /usr/local/bin/orchestrator - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["orchestrator"] diff --git a/docker/validator-cdn-local.Dockerfile b/docker/validator-cdn-local.Dockerfile deleted file mode 100644 index 156fc50571..0000000000 --- a/docker/validator-cdn-local.Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - - -COPY --chmod=0755 ./target/release-lto/examples/validator-push-cdn /usr/local/bin/validator-push-cdn - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["validator-push-cdn"] diff --git a/docker/validator-cdn.Dockerfile b/docker/validator-cdn.Dockerfile deleted file mode 100644 index 50d927ce74..0000000000 --- a/docker/validator-cdn.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-push-cdn /usr/local/bin/validator-push-cdn - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["validator-push-cdn"] diff --git a/docker/validator-combined.Dockerfile b/docker/validator-combined.Dockerfile deleted file mode 100644 index 7e8afa9f16..0000000000 --- a/docker/validator-combined.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-combined /usr/local/bin/validator-combined - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["validator-combined"] diff --git a/docker/validator-libp2p.Dockerfile b/docker/validator-libp2p.Dockerfile deleted file mode 100644 index 42970c1aab..0000000000 --- a/docker/validator-libp2p.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-libp2p /usr/local/bin/validator-libp2p - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["validator-libp2p"] diff --git a/hotshot-emoji.png b/hotshot-emoji.png new file mode 100644 index 0000000000000000000000000000000000000000..a219bd53e0e35d98e290f7cca4eb4d6f6a30dd08 GIT binary patch literal 54333 zcmdqJ^$Cf`_pIcf$)Ol%CF1cy0P@|MgW zX{~FziaSwBZ%UY$n4JxozU=p&crbIB3ZIR)5IwzmzX38O1S3iK<3U0H|31l~0Tl#$ zp^X3AWx$n;(MIBC8o9#!vYQ3V8$ur?Rwnk#tfWp|xSK{twv0+|es&IBOvt~Ifx3aC z4I1~oi{fv&O!qTnenvkpSAUQ?L8kDX`!t`yD8=&b0-i7v44@JN;{jRyyLOA{{iGQ! zL{8|+6B$=mny$*0SBDxTL?BUGo(eS7{|pNj8>Zs$9b{bNQ&<``0P$lLbUBA8q8?Lx z98kRFNux4J*eQ{$g8=`TwVnpJnU>?6q&CyB`P#FCC0{vb((9?e@^QcDOMM30dpYv| zX)(6}!r>syW=S~9en%Nnr^@cOhIIL7UimdPjCk(@F%9~EH>ZGdz}Y-dOozISB-Q4X zUd{a(|F&OTKn#g;eT9}7`0vJZmGL+g=?A7X1%qS3pq%!Yd{|#~k{XX1N!@)ab zrDJ>NO6$x~ycc%o2xyS|Z?UythLV16SM%A0O?|`2k|Qf!+O!{&8}gO^ zDZ-TjSfiJh_HixT_QokEd`kCKv!c<-z%;Zb5g41=%?`+u5%n)E0$=hmR4C|mf( zU(rHf#B=nr*Z&zwuowW;{c+{@ocUdvRJgXMSE6R*{~kjpsV;^xPErSj78a|f!bN6! z2<9Mn0nPvB?ZO3`ukJz zuBQ3Ym6hpCCN*!$T>g`SyBL)0KJsH>wD;ApwQH7iZ~o)U$>;y-6hO63iKO4U;8*0t zYa}m)updZO%C0RzZTJffn}q-DA_xy4sUW~n@lsN2zVL_zvA5A%I@Rj`Hg^;#Hrd>k zF3G8VbuzWJ92hRm-HH8wZRUVlbwF=!MBKlTM)|m7u*l>8eNmAbu&l&wqLkaUTBTp5MVfpVVT$u-58|AouYrHRGSRkR`LMkdtunG1I+$& zXIAo{=zM4WkD#~Jr0sUIx1*_H;nH6}11?|xr|d*`cY1sz z{n3?pk#8w)Mt}G!^um1BFcQQ6695XHY9t)DNKI*V;K7Z18)h=8^t;UQKk=qY0qn>! zei061*X`J_hHEzqQx4Jp&&9HUopeKSbraRymX%6$q*&B1-$0H3b4*jQ08)Er`@RP2 zRDya&f;bO;^Rm!z>5NV?8jAm(xP3f8!6?s7l@m&Fp-&50RY!z&vj`c}U&r&0^phbR z^!pN51eN}>U+akxp6TbZI2ZoE*9w6F=jVZS97NrokU(43Ho~YnEJzf?WMSif_g^AL z($Jj@ju6DkEuP!RpM4u<-Q%baBld+9od2iW=Mi07Eh|JEYE4V)Zj$$AiChG=a`^W9 zQU3|X9AeeDS+?F)X50Ux_O5XK^3N=d$xmE%CguNx9so{WHDmtr&V4s2eVm3?2ohBo zU+nwe0^Z>x)l^#P+e?!P_2CeVov%xsmVUH@8c`q{96T_8+wr zv(Pt8N90N0^iIp<^rzA@#hwmOEKWDMQq(cEpe)7T850YQaT<`f*>tf(92?Mh&aBjK|WYpnv8Zqy}t3m#bO$ zBl}q@OGh|)+;YaEi^}sxxD4#mmM$evGc77*5h(<~!aM^4RDZs?0o(2o%dM!a=N;Gjhp8onE^a$e2Dg>SOykQrcEr zkqRQpViSv)Y##z>aF!o^mPfucIbuZbeD3sYp2^8lQtvCr;3u3u%xKFLWPUbeOOce; zLfCfb+2!IdPe()La@l=)A8^+P^1I->%id03Wq;V%c6{NJS0;ec?QcMlrviwMrwm3>_j&V7Po7l4P%Slpyk3v)+UxWZ&&!f=7dp0$H>F*;j+6ur7Gs7P z9iPpAElK0qHY+O4F}0zhOA*&VR)@l_mRmaNjlLqOIFFpwYw=_Y_o_p@*&*?ymbT5A?TM5@Rr=K!~6)R3b1 zzFPJi!r25anC*~w>qD0_sKLw)Tvm_c+*hg#slA9GE5);$fw#*6vU!KTeSRjJazPj; z0$qa8aad%sk>NIOE?6nWfIqGMDLc&8yWJT)kvGLWO6c}*%IcFnS4}RUGJ()$%n(ye z0Kr|j!+U`5091vKe>fM6aD)i`)xwcogco zpkB35URHh%DN&Nh6~+UkuwQfE8fb5DcsSO7uYo-asSR@E6;5r;kO5mOWR}NA8oG>U zNR{7m1C{lkG9n%sj#%VO6Q(g0op1?G9&QM)+-M;oQh;XtYDAGSRov1TB3dobOI)G{&`|?KKP4SFzugVJ67NxE+bqnN zWL^0q0^Y>wcsN6!(!A-byc|lCmp-|^;esY$EN(KJb@_AGiS@#`7|N(5pgJI@GERL8;GINkYJwnGY4r$vwVp)g);OlFeVIJAI_;y}3&;qIZbs}GMZR*SQu7MekL zR^-4y_l=}_m1kR=q(aelKkowWLtkL1XYSV%19#9{$gefTLo=#yRc7f@FMo8t|6(hS ziy~+G6N!dN0?b2!G@+85yrR4rXX7_C3&+f4eQkT?h*h?5+Ea8=B6`q!!mX_qL#KVh zkrcKUi>&h6>&kfHIgjU=7*Z^D{&{8+ zP;1c0<+_9W#ZAHD>j`IB!d(iXVGiFhDSDy68uDyNPcWL1e5*6BHVpXhY-`86G`J^ITOu5)ZF2$$e$=mzr=W zgRm0=`DnF+ zW>!dcoF0fhzhye){X@YDSp`P?GY4gs4&+VkADj;g4t(NH`V;I3wr3_y->^rv{rm_l z;8w8bv?+-`Q+y2{{G|Dm!{amgYBn+BWnhz3FN|EEj7S3N3kp~54dX0*&)eXsf3k!v z8dXhV^O6;;`JnO8s-`)ZWD$%j&0Ama!38(OK9Psm7*co_?2ifD{_|V7>1V3ilxj0z zN2t}Va8Xevqa(q#4`Zham-+&m%PID8Z*N4KT>6U?OTJjb5)?!O&Q-H0%UzXFe1Y!| zlP@Q={voMS4GZAzi-C&IV~1ezn0{1Q@%PptR7QY5ptMa89v+^Ugz@n?!@2&8`fCL- z`XNQGsc8IBo+gw=!j*w^J4O>fIaEm00%tJ}7>N?xmD&{Xy;Zb#-luaQa?5=g&9x&c zF`?LT;9VW?@Wv=({X-uPX3;~14led{!s!Qtr2cMp6U~kSNKJ>D8xJWu zc1~x517AzjJk|>LZzTzRb$)azKL zo;*rZSRJ1(2ArHG+|t>Hu}}2CV1W@ql`E>wK(zMq;9E@Ij~sl0G(l%+fL&s8+nriL zVaI0z3_+)HqS`qYvEF>%&lK|60ATa6;{7iIKx-c6x!NQ^K8mjEY4R_mYsvFTR4%s- z&uZEY`c1_L!K5@&n;a7dqzd;_j{3$_-&gHtsg;-(ZWX|uk5g3;o{ARkFL}ZL^xhkX z$u3-#`f`CurA@UKa>BKPrQ`RTh=8Z=yjPy{Y1#&@A!kXl%QjA|9G{rVtdMl%du2fJ zN~cA!U$n(_mGhFF3kO+gwQHt0-b-zG&&r`2U8FqX-b{vsTR4+2;NQGevK_W<83YY z8hAqqu#}_abpgerWs@lmf486*CY07U%kk1MTCrDTjgLAaCqDXEWR{F<{9UFO#uBL6 zxs3;9mA*ijAcphg(H|Ws1b`-E1g|Dad8xWQG4lZd63j_$L|49qn!LMo$wWIApZxSK zX*~|j#S--y*KAlT&u{fuMsG6Y0m;z{IpZu%RVWBZjfg@Vu9~&8f9Dp=Ax((9`(-3( zv}q=uZNltNBzVWEZBy2~)peqlE(P>Xd-4cEwiu18wyKHw! z9g6J@|4FLPpL#&+@U?3&P@Ygg9>Apf8po6TENJe}RMV%e>Dv)2?F0n2LKsU4Insq} zHf4{Ul3B-yU+o~fw8rOD&Tf0ArK@1|iSTc+=rwQGN7kiia5v*p$5S~Yf{eZbhOzZ9 zCuis-vI&4%Ad4KsPe!;p8h=)&o0cdIG+RROO0&-1qZzsxOiP$F6_p+pm-(L%Pyb)| z2UtRz3T@ba+th<<9{pSN<~~wcElhm#g^nelQdt|{Ism(qM4D=Y?B8jX!qUVqoPRUu0qz?kxbR16T+nl z8PqAt!~k@`XR@s)s#bf{MB~W80RGi>+i7K{5&Nta)&>W&strdw%l)D{WzdHtrP29t zkUSQWsV$*CHS=BN<>FV074-{C8AbBn zeMe>>!qEldzMBYbsQ_5Rq$a4JU3i;LxY{>ch1!1KwDjPa3geOCy{paGBVGf(cUphS zL+;DE_A)OMS4pswNoA^>skmYAm<)g}sv`B)!#DODQN1m@@*2u!Y(T?nzr02K^wz+c z5$u)qbFLoMv5l~iN%u_)QNn%ES#YWJb`b-JCgz_VM^KC5RO5%On1)G4quSAMq>0-@X2j;B)HQ|nvQ2@| zbABv3PonHbWqVHf!+bu37%=*#O3}+OMM-Lbk5?M(iVw^%@TQGCOAM-DP%Y%c7Y9Ti z)4e!&v@9rG&5{e7iLeAU%NTvlSJ9h5&}?#ZR;z-cC*6O&WKB{Kg`NTCFRSL+Ja0t4 z?<4^w|D`>FY|WD8RNOWL1vnOM###7^8}Mjc$brP8AqB0T%4@=BNf_Gu42gtNN{wS~|$5MTm%p9rHLGtZfLIA@3oLJ$4bTC?nv+|eXN)? z(Ia!vcc z1f?0FR57tKwD(a2H6Nh1rX#op#lI-)1s8$0)eI5UIy#bR;mORy#o>$BA;Ls3lb`5x zOoV7a0+Py(mgMn=@K~WZP3I8y%Qfz;Vf56!z&~flxM$Or2pBaa;oFFbf-pR} z=fM6RaRbt<&^$qN^%qolB*688{6b4#DaV@kRL{&?z0tQuxb8ajV}4Gys5oHECY&)^|O7Z%zt4((bn{~}^ZSifvVB8Ht*Z*Z#*h7gsT4*S9&8bZN>drdcJ<){8F z|H?2x8^2bJh9MRO2}V@gl_6GeN39ZDGjS=nb|zjmPvA*D6M_uPl7libB!OqRpn2<3 zc@hjz%IZFCyVV8U%NlIXlJ=@^zU3VHcjE#w(!DSo&9YG@V9kSgMkLcSRu^NlD8o7e zVs~$bkVC)IUkZ(c?ohRlZD7p-sTjE#x!{V;jJf=ZZvb^9B-0D>R0%;bMPr;SKZeM& zn~JbdoDH93&VNm8ACI*4wEju5Fdrw87(RLOqSPtLx*+xAW1HycB+J$M2v%WD47jwyb%<>E2WN9!_fbm z9mNyYPGbUL%$gyE8aXf)tx~eynNLvGI*0}p;by0by!-v3v2>v{`xCYhFU{~XF(_1< zmsn0<2tM^X!N@gISehDdC3OEWMweo`Kpw#|V8_lVpdd|XPNYuF{Shox*gMGoy;Bv} zcIW2Y%9bxNUrVJ*ODOh?W?8f^R$843ADdw_7w&I2@h({&WyQ^CfcPZJWd>mwQcXQe zbjs3SM4e2*9)X%ijFC&DTT~6RpKUP**zqITIP!l(xf9V@nDDVlMy2VKFcz@$_oO>2 z$PA$-5X#2qv*&+{HzA)f#b5C1-jrPkA3b4g^k_|;$#Or9t&v(YTVsljYe81)^FEF8 z5{fZEp=G=HRijv$xHBpXeW^Bg?urs&tLLlG>pnOZ&ee`RwlUA-tsnEX5&CG7DY^Wn{rusXRuH-w{s5L4fm!(R>n_b07x z2iwyUNHLMCYViCCb8xt$U_y~YND{zP-o3COr1iO%j=M8=6?k-JuNt}Z{bd=cUEJgZ zgL)XiiZS)vaRJ~_md&nMvUQU$wcR3aFryCrhPN5ooeG-K(@a!8$tXpsB|NIv)o1-f z6T#W4`kh89SL7Xbz`1p*@axmh5h0ayLW7D3UT_eW%eK==7&h~ow=uX|z8<>epA@aa zKUX1cgLR1jce4%mH&p}uFcG`uI&&yd=#P5}7?J(0r?qOZm@7D!H6%n+3@x`3Z}U8R zlpY^r#>B`@V@0Wo)FCy7HyguU<;C%xf=Mr;93=);c%s3>3iW(+Tc#Z-e+BVq zH$xl>uIO)&H0w>F>1Y^Q7r7F}V&Hmm;A#7tx1|pZE*^uzNtm^aG~}dFA3R5SiL(zu zn8sZT1IZof!IN}XvIy!ncL5SmJFMk1#*l!_A{!4In)#7mb3AUX^s?SpUod~Cj7$Q` z2O6<2j0!oipKE6x-;$Wo%-!h@)tkVV6F$yJ=xi6E{UNkOgwNh#KPh-qz8f59`|G>a_g+i&5(@u>)Ojkvvclz{l8LR0Teimpc7?o$pMvTBPzL^F zF6DQeJJkAP)!{3`z*Gr7LK+fM#UW^Vm$rIgYA_A*AWDB5=)-))H_%&|#v9QCjg3xf z6NqR(>oRh)hgNZ&hx6m>M{E=C$R(Q)H=kEMYga6yV@aIu3d#gP2vn7hC<|*dF)YcV zJ3up+L5_)|-}SpTl*l|tdH5;8blZJ=hLQ~7PV)+7b-aWXS}gb3XTe=K@dzx7as-ID z<}|O?(z&*zEe!OqwL3ZF#@>+5d50V2Z(aByT8t@IoQBjrMpEQ)FXlBY4@{#o{DSCd^Iox2F5Na(~AAS1j6b$9LMH*5BdPV#J?I%;N=ekQ&Hv0?ys1 z6ey(A8r$MA!{}6Pk)&4!P^_8v1Gdm49gcmBddBxPl3HD!LKIbq_sa&s$-=f7^m>W4 z7QN#z7x>V}N6v!?O|kjAuTm^9U&K=_2iliIso&miAZ)-_@sT78b8ZH~@Ud-O3M)%x zrHn&5%U{c1jSau0)`=)>DyCAss*wL~A$tp0|K5B+mr|zK=Hh9(VpQ}iKRIv-2I{8D zeNn<;UR8+wNjxJLUOH{`j0WG}$t*h0!iHIl7)E^gy3q7QsTJeAf^6|3%slR}l{dA$ z^zd-#bx)Dy4K8i=`*k!fCJJhxSnMpyHVzh9M0liL?p$~Oj>I#V4JR}vI!oHDS?H2( zNh_B|iUltMCw35BD;W2)7j#5#Acp&vPn~{KspLh}0sJQ&iUbq|{Hf67gEC&oIB9KS z)VsHhZ{3zKoM5svdR+>Bg^NoVc%k7Tr*eX4_*q7^gCn@jA2eGInQlP(BUjw*B~5qQ zkv={w+#&|;Z|Ui^E$r$)t{q|?z3MnbY8AFXTRXR43wkwun4tO;4Dh?y*3F;od5K^2 zkwy@Quq{kR$1_Z~-Pqfm;aJw3@YHV4`FpFD&xT9Jrf6x$cOFQ|TWMrCNu20>u1!~NRubP@ObujnH_fQ=vVj&f5Otx(eYv& zss;y+RtdhyAnU#V^@9RCuj^adi$Zg}-oT9=s8P!vag$U~dc3qPQX;JcTQ0Q_HX4~d zP|$({t04Dx>Lcn&=5vC-HCZe5ry5W`MYLsXq1usLI zjT}@)OX?KK_+g0$Nsar?EvtX#BKDa5q`_u4mwQd=mv}-O2dx8s z11QVHveyPvk)9HlWPhn$DOzE0p%b0gxxyd`)a+V_v`xqZkulCvfh1sn!VS$IsBxpJ z^>&+Bi0x7FXJRs#mZbQ(AD%Bp;h3<~0tD7xHz$qKCN8o+O}c*}7XS?z6xWUHFR>9OX#(Ii zm|}F$8yX^2+9#JOEdUN-Mv&MdQtFm0m4fq69v&zT-YXk6E3!f?l6ftx%N*WIpGM-7 zX$YcF{M|)P3d3np_%DNAUcA}K$?f`tlhuu3yCfQHPh;j2Z~UvA*Y3Eu#C8d@3OJV4 z!&~jRiJFcHlAyXmA;+Y54lqy9Gm73z)L!dvAbOqHC;`N#rSq0s;9acd$Gnyzc;u=&?5H00=G-ca-N~3Zb%$7K+c>3LVR~X8(&C?}teN$o`G?JT>t^qS2Lm*X zI$sQ;91;dV-SSb$An};g!d13LQp#BU&#e~?Y)}HiLx>Xi+A05b9<2O0wW#iiN^aV=fo+3u5ahV`O{KdtLdS#8o zQoRj#?FX7?TKPymo>UYy+@`_zwQ`AiuIwLTSYNpbX^>Cq4d2A2oxffsj*5UOL? zZFt6RqqMWI#fq{NAf}M zY*udQD03O~T03z4NsBIZGRhIzHxrYAIzEd%db5S-(Sl*{DLN9JU`pNfC3v5NY8=tOjFl@QoTj$$=V_{SwJtI(S! zkV*D}B<{QdpI&&V99Qm%;D_c&0zJoZ)a&IeGbNT+Eb+r9OnIN$E4f~wtr4hpGs8>k zEhs+Jw@&a|>YVxBIy_r$WDLR!2Ov4kSB_LLmfO>N9Wpb({qOKcL+t%H2;kOi%`DkR zo!zASLSIi`ABuPj?8Vrw)%+fkOGFU^q>9a`56 z+bn8Qjv;tX*b1V@OR|ua)|gwN3?F2-G1CpuR?i{@7LzP;Z}%_7W?;ro*z16X22y39 z`rUc<_sGsa#kdsw)^DF%nNBh(LQk8i040SJ7T_*?xt}RjhD`4U)1nlmIX_*-k)2b* z#^cZU(r+vmTy(709%Rcbuh~*R^CqP@uTD7_FYne}~b>UW6|Mx zfNJT(6ZxZ=IgO#m@}rmt4M}2S^gK-rEtWZVi^kWx@ArO}zu7Yo?IDAcXELH+ZV zqLn+v2hWUAp69#m0hbbNh}+!PP4}b@DOMbHc}d<~gLIvluCqRjd`bCn+v`|BTN?Sx zf*uLwC{<=6w3}sfJz%;x>+xeM(Q(4VJFSocObO@<(7F>R$_MyzVo)~NeHdnB8}M!L z7&xqHGH(v~veO+k*E=QawNmUy5>8|ZBi537_u%`tv5`xAVIm&0~0vXC><2hz5BWb?ddxYKJ})ut(% zQX`-{L|=#)pfR;vv3KyZ2NM-ZBSAYu!E*$(%!LC?AOvv#=TzAwU68pM>`Fcd7;)uQ zuinY{$V^OU!7fAFPtcP(Ymf$>6DH~ zWgAbZdJ8zzYNb&8)E>6CjDjERqpAe%jR3F&4CuM*HXsp^#p@Zd5RJy_0rI-=h~WNL zs&hCr+XyhLdh%z_EjT1@nOs(=Ra>{@UJCN5+3Ipu;W~Kp{jL}TVWrjMU~p8VQ!pL5 zRwAc{xqe#b5OMW~nAaPp!#j9w@u;Q8MbF|f!R|gECjtrie{(LOsD~zqBuR@2C0)<= zJ<*^h&!R8G@DnkjSr@l;g`BB~V*~Y!yf`%6vX~t=WB>NoU{P5WE%Yasq=bsq0f9y} z)-{J~@I0i#Zbs-Ho@vRs!G*uo`saB;Z55%P(1W`ZUHBQ9-)0#43wkdWXq5C(Fbm~$ zZdO{IyHBKB-+{vY-Iz!RNWz^6%-=UiWyBW1Qe_39xf&^^;j2y+V}cs#b$s=1IraKl zU)tBT&sQgK72+mDqea*VS7q|fe-mbV(O@(C)3s`4{VQx*xDloR%BrU@=Ap=`CtLX; zq5tZe^T~&xGT9#Hx3ue^xY}$wq{vOFlX|6};+y{Z_l>RVe_pTsm`+&UEdMkq?Si+i z2;G0*Jq40OB51)KMZw4+MzBgZHU5=F6(ydd6$GX5LOp;zi-ihgZnWhbLXMPY+g62WOz zN#Xz;QskHro`C@rLRT$|CCU&SxAwz@_*;1VfXS~-2|uk9uw#Qlv8&6{I50r=n9z!` z7d@MJ5IIPh@Xes-E$;WJpQbc~8&eU>&QTiGHtrd2t2gAsK2DU=TuvPRVP^s<;`#nd zm?dLW0dEE7&Mw=p=PEB{A7+-CG9)+6Zr96*h&Z=5^TkAfmv}t?3;C4 zmO4+0O8{p@frpTU;oa?K^HX2hk`Be&Kw2TD-{gZNPy=8PjPFk z1Z#*LsmTi)&R){0J}(^#<0&(8>?AwQ#x{u#_hY2F%gH09J(r(wKJq}p+h;uJm)oxv z9K=wnmoN?LIdaT3o(4$Dmmf#VapYU%A;9&ozz+`D+TuJU?rxqvZW+NXu{YQZ%B!g| zhwf2OcunKt&UNK}b@y>CqW&gHx4nlpX2_dFROg3c0^^L2uuBUIMmA+ZtG)R(*HQ7h zi6_C%isZQw{LA%Bd2<3-$5xIL3{Ci7ymbiMV->8qY|?LiB$n ziivF)e=QnBF`{GQSWea6^|)T+|2<370h7P_*(IAq7<6|Y1h8X&<0b15D%i~2^V+AW zP7XS91?OZ~L0o|vBh~nv$ZX@B^u=4Y zqCX{klL&O-aZY|^WP^gmLJEoMn&7R@|L`JQODKUJ$2p)t5*WZ)Or6)|8qK*HVO44w zS9fUMeU4_Q*F;S+kw@??JW=nH&soq=7SM*x-{|z9%<(kBA!r0RRt6bzkp##E`*z!; zie8awWeLTlq&^!Xtlx3@GIBBd$}n@OA<9j(DWR5r{_79c>W`(f`O;##0b?~x1hoSA zhFd@;g(Z&Vs3bixG%*gdE#UdmaTcB_8vk6II^x9uU3I#sTaH)a7}hdIzw6%o`gIrT zhsl)>Da>qpkB@sufsXPcs#cCGIc4Q_pRB>p&If!IW@?g!ctn)sa=3LkLXh%0O8F^W z`^EkF5^Uj`nrabSCL0qyWI}(G_}mW9lnh$}$!dnf&_si~Rq&wX>BZvgiKYOM-%WGT$?@Ch3s|nWI%_7;)Tn_1?pkDd9 zlkw=q_evv7nPL^Z*3U=&h6308uv>$2Y;2{8uAHi0TF^|Y=To||{^ymEWC!h?csxu} zP42-Ea5({deVgvd8Zyn_FpQfmaB5Z^^#{4a^GM?PdJ7U)n`2 zgQ4cZBb9sec1d6Y`vWMh$TY*hD0V}a-E1>Wk1Rq426htTK=pmpNcFLhjNdOSokmc) zMdy-ixdunftc_DfO=_d{jFbv4YWB~z${^S7RK!Q?Rl43%?ufd`VZ6NY-S38B10(69 zQj2inRJ$#a#u&yZdJ1WT-KkEgcp1xL^v+Q;YkY+R)YWe8bsk31E4IjyA<;5Q6^vpG z#Z~P&%GH7e*1_FO8V)_171wBM=INXcX_E>_4MeLCmTjt;!3Uv6l5FJl#M*#0y3y6?q}yGlzR?E62vs$25=e?ro-a}_7#W_6PYo$Ymyai_ZSUU}_t@AWFzkUyG z?hJRcQi4Lc*tYu(M>sc6c4p*IW{^qtPU+E|SlTYFJmqU+<(abTGh!Y#MZZQURx8A$ zc4KtpP#B?!p6T$N{Wu`o%<$I&m(RL+%t<>8cjnBDqhPJ{Ka_ooMHa7PjKh$PuJzPE z+>ye>xLAc%`T){>Jx&%iK9^H?WUe)P6D)iGX61Tykz4!HYu}F9sRYIDLP{4^N%5n$ z`#Wr(+)TIktr)2U9Ttn{W-POM{_E}tAYe$(KN_qn0;FT~SjP>kJ3e{M^_Dd_2DPW* z#noyo=jUA6$3qR4&N~*DFj>M@p+tXlJ9Z1dOUyF&Q5?%aGq%PL>`Li7z|ZN^LV*RJ zF@K1i8N7`6-Z)Kh5a(n_MACH7s4lc(-;wU#$JkB=&TRxnAmgL|eoocz6ypbF-sxj( z%GRR3&fyOgM0W5vT?z; zUCkpDyXUJ<@aj*6K!6>QI#ONWI9B}gRL3dubtSxIKn5oLQHYca)vUpS73{%=O;x4D z16lTZEf7NAI7?ppD3_<{ZCR3l;mL=13E>eN#Xh9lb=rtP9@x*&A}0XFKB;}yyFr-@ zxmXw3&&?9*OMXdI&6isnACI{^O#S>&Ay)Cm7tig5ZHl*Va7#YMJ8Zm_SHRWS zYv%@rr!z;e>i3FYrw#6dl)mmSm;Yp5DE%^B{P}*aY4fthfv|7gaZOKaxc0;?WWAqd z`eyT_%t6w$GKs_Oo4MTufNkmJzqB*ICr#s)YX|9ajxT|xy;j9KDV&gHheqxVymaKigtWmB~*8B zz*X8f6aIE|cxfB_d&u;HNZvQwXSSr96ZX%WS~kr2yQmF>3&b{j9R1!xN2qfyk$DbR zS*zQy=~VO|;x1J7+~Nge>LTCWAFo*4*W)a|PHBD+vVN+{R)nJ4o6BkBm zZqy}5;O8EzoHfGm_O7|lWXIcG&lyE$0o5bMs7(EHk%AE(#a2^5IVs~5UYvNJJp94 zjI$dFQFcV8(A_I>#%um=@2FN_I*itkh6@v5)v9kVB^`wp8Zadq{X+ZfYWz#K6}1NL zfA)>*_1B=tuoxYXwWaeU27M-eS?QAmer@S@YF-aS2PJ+mMA#sbre{wuRva3Dx9k9= zaID;-HA(L65GLP`Q7y=2H6oIuT5t|DgC2Ce36uEUE;An(Q zrc<0}PlJXNE&5UYE0`UC#wvYP!gS~C8CA;IWoCrt1yehHmOY#J}-Kf@2Ero5tsz*HvY@RX+XObo7 ze`Y&vbOQ9&>%uj?AP87X4}>!i-7j1j8V7*;02wMsffa#7;eqAYz$_LlA0i-rbQmbS5>D*Q=c*zldcRL?6TExp<2*ixI*9R9!ns$VmoB% z5rBVMdKq?P;72xHV%tg_uWIDyg3FU*Vb_O!yy_&GxV}q+IS|qZPc8Y#V8^`?Sq#LY z$b{gi5G~|HF!H5-w&>Al(6N;9Ajw>v?n%YI>a`HVFb$)$6*q&G^aSFoYi*7cxa zcuh&v`(FX|UFVVTrm?N)Lfwxo#t)Y^73icO{+?~k9-X)RttFZI?`JUhI;)HG@P|?1 zp)S(vVhtm&AF1Uo#oT6y65rU8I3qnr`X_m?f7d`F5H)aPAhiXdWC3S~5 z)eT3&T|7+lO=}E~K;FvHpXBi@hVAj4B6~%8Z(P24bSK*eA+v&ar7)|^W9vs;JF=iI zpHM!mERS`Rj3dgYpKX4?eX#6&%i6MuYpzKvZUkPAJG<4O#5T~3HB7B*GncJWiZRS} zoC~%-rLfxF?~*d_rEc@ajiW>=J13rD(bJ=>t}jLTc1YomW;$9gnxy*NG)24|8_N2g zMutXH*E2tZ`T|OrYtF%o&1bH}aFk_*lqNnJV$KUwk$q>ee9Dd;9P<6{!h5C8oJixQ zP&8UdpsYDSE`<9SpZriSz0=G|HCkKz4%RLNA!;-Zlnf-J+n_M-YO^j=Bb#;;E& zWtK|zL9Bd8TsbQpL93Tj#3~$F7TPD+1#{hAF?h7M=SeN95==Gn6MAw$wlJ|&PgX`7 zzee!6ZFEseh}4V$SbnES-zW3jZ5TF9tYvC1l;=gQ(N_8NZS4s?0mGxq{>2wBX(os; zaFOg@RNI~i9oQdFW$+eLi_PC#3{M`M`kiNlA2alcfpyn(pDHydBVNySk7Z5t>r#Xq z(>i&0C~tTnFd|e3buc8;xTJ=~M2X>W^zvC7J=TxRL)ea0YUjIDK-64sE%4g1l7?{2NDf;NRBy&a1uUOlJ9m z-zvQ@+Hsq|+3pv8-O--@S5Y+hs5B7m#gDk=RAPb#!^fi%KSWwmJGvY2xYMU-AC{Um z8>op)8U)n?$i$fZjXXK_S)xZwgebj)m}5pcY>~yCs8P6a?D})c-8jB|*#QpeHV}E9 z$Z(v)Ka2>NponH}kZQd$6JZ|9{=V>Vi}fS=59rhBDUDjLgN<*pIiI~*`f988Ro}1l z^C1gEGinEQB=TS{!yowY&~BWs_S6Lyes8N@)kh}j>y)@&sXa#LhQhK_0A zR$DQz3aUX4l zsjEqCP}!2Aqb6c=hMs?7|1vzx=Dd=oGsWnh;nxW(p@PYJgQ-MjKv$Pdgrl75)&b=Xuk@N6Bb%OwOK|QX`!iqb}h@OejUc1speKdYmOGd^7f18Cv7h z$%%(+zx<`*{-p|^?>%(*=x12pB4|Mi-O{g@s4I12?3cms$|3=iad$oe-!4MiNj--r zZZDK&VaGDfk)M0#?wS*tN`x48ZK^!j;;#+c;P-zJ7D`9eqvYXMW2i zH~&3A6?0aPhoE?;JHSj#F%~s@A!XWqW~oyob!Y8l-~33T81Ap9qa)XpJnunX@w?T6 zHH1`1`}LXqpKCj-6+fIh^%`k^9^%FcnlvO`CB;^EDMcnRZzzU}_y|dLHdwl-eI(K6 zp((F%#@MMkLynE^D`s=rsIz#bV%3uSSGfODGf~5oIV(O7Nqo4FOuXi9h+ei22p`Kf z8XM>M{{R_5=Dv9yFQqZm!+<0M+ro6a7&R7pPan%KJ(*3Odp;`)L;^Jo7~8cIRXV(B z!`T<({EuC7CF>e`g7rN5P{Ug{yorOS$++G;VCm!DmtXO`+hg^XjqK9h+u7`w|H!{v zjT23=e}KJt!NqLHTP`W-YPPzfFQX3V{{o23%x-bU_K))PsxuhllfO(R|E)FFP=PO5 zMRXwzX8GDzwy{y3;;OlTT6Wt}A^>s-g`RxFtMOO z$Mw4rRGyBkH+t#GU+JFz`sY(Ucq`dh!&UAxgu})v!cWbaQ#TAC7A5YC%8p=QVK>bM zl8id-g`l~Z-oLbkwIFu8QzrHQhQh~L7@g58`!I;uZ07CJeCm;2^ zd;|~%ccE1L0Qsm0jxG_4;Mhe{)427xoWA?j zH}tXInq^)jEyvf0ISH@fuOi(grVSrtr5R<-1!ycL`wx+JY)33mb0Nmo2Bf4yC^S8K zNFAvVvP|0wnZ^N$!2IohVA@{36UP#Q5DaL-SUkYSPQK106V38WX&H*=TbLyK?!U`4 zO7v^n;iVI3A9iMd5v5yz<&Hmj_Hov`zjRcWA9e@7n*0 z&4@Gi5x&iIuEWo?S8^c^+H!46F2r)pAQ!y<(mI{#+eJa*z2?iR-|0Pj>Ql7yBOK1O z6cP@<(6uMm<*vvr&CSO*W~~m zG*b2(se`mysl%6897zQf6SNTxJ)yP&%V5k4ikgemE{lMi33=VbogaN`pSqph-+&|~ z_(Xt+g-^huF5bQR*$?V_gSj1cBO`t$M$h8{dxKf!(%Lk)v*?~_YAr^W9LrBMEvKuw zih+gDD3A&aGJ*klGE#wcAhv2Q+(-r1MN$FxcnvXVE384p6tC@Y)CXs=vA0}b_lEEA zZfWhRWA6Rz{5bEGhuGU6eliklmfiLL^V3M@x-N?hPjhRb?qI~iBqV(IP8>m9QMQGE zwjH$!0%53=1JW^A$^e-VLt4p&F%Hiq?xhfJq?s#R-*?~8Ln0=09UK877QwL#T@#Ep z^}So8H`=U7h7+SzP`ioJTF&-c~Av>_? zEL%;5&}4%lC**;~!m6zh>%cYO1Q;TYh}*tz*{R33M+c$d%)(Wn_gzH z&C@Bs(#M~!f535tD4=9|pL8rt}SCl#YwK1e>lUMfMM@E|Av8=TajHcJ>(^Uur zbTxBBi0Xl>;$&%!y)>xxjhMwuZADkvP;+8nQC(F%-^Jk18c=J2_7-?AmP0auHWy+Z zUgt+A`Y5BETCztmZ70Lix#4qFQz*Y z5HEc|qiEDw?LfGYAZ@x77mApdcCWNhztV;gEl;yb!0*95_rw`=g!>=yD1a~884+hQgGxiFVNuz)27a^bWrf&@dXh8hc87QqXgYbMl8qg@4%3OjNEji9Xs#%9C9 z2a=3(+48UKo~Pbw9zhOHv2bgtP{q?F_^m+WO{^dz7(ggs$pg>G@H@F;+wBnhB~m~t z+_sdOMw-95H+FbHYccPICmd}pri!K5tc_akQ9I&n+red& z?G^!z1Hg%KrETBjV$vd1B)7&6rMLLDK5}{$37U3|5V5dBCgyabpuE?q3+trrR5mFR zVrndU7ji@N9sD?e-E4T)>Z({kFs$SPmqRdi(q2>>l3XCc*gM2E6@}Uo3wqr2OECK1 z+*H?#m1LCW9>NY@_Nlt%4yfRFa;@v6gKjqL-0MmzqBPR<0&V}s7afTOEQbUoj!;i_ zc8eFe?8>Oi`Hm1y5bP#)r#!PwFXX~ObAh`?-23rQEMh$>h-0*_6PEHJieF3^b6Zb{_e%d1#vw0xjgwNc518^M5l zJ8i|zQ!irdou4+(>>xrR0oOwkxXAvm7hZNO*$h$v0y&+Df&%khr<_>eUGrkFuTU%Y z6{tA~MJ`|{Huei) zqE}Dm-h)nJbJzmDBNsdqE@bx< z7-lvUq6Ak2z*hwvC^lWR=(B~I%_0>jLif%$*wlh$c$?Z-5q5*Xgbu-JVt2|b*XExV z!ca}OURVeS_KyU^j^LQhxEXX>$c0pE0S&>d#7(x0{b4D7#WnYXLkCUyTKL2G zGkhK&!o~dWWTW)LnfNF_a$(h6jBh`jP48Ob+Lv8{>OWqk0B9>hg}<# z?hOGr`h5SS{J=SA0Sq;xa(v}YBLAhPOJH$WnaTZEkF17<$D~(U#7W(WKse=91#=@8 z4*5thT>FWLT!hRNm-sXl0LO1APNboz=%?&s`j9>!kP`yj+8^rqlT!N$vjdB5Ny@QJ2)espF*X zs4d_FbDi{I@DAj`#0K**HWSMqUI1s>?glJ3S(BNFgWvJ?9C(A1qFL;~JHJ@nKHAVh zN)kr*?=i{40lqDE*S6hk_p6)OPCj}234=2R*x>xdoMg;rN3TAO%{_8uY-D42s&T6^ z$_joeu1jD`DfZW_SoLtQnitu?+a?kTYk4uBRkV3gh(&#|j`#s`ej*t-K!XV+PP*0>{7%Bw`uL__n6c2=wxVLpDI*?kUoR%pc~s(a8xyf*X{+y2rEfk=c*j*;O}HZuGI+s51c z<6{rATvv`QJmEBU#u;a^rN=gsHdz~9)6QGIki9uN*5F~Y>io)ULot_&)4oF6w<{9q z)#a6DBpL&22X>RYU3y)8xArj~P+;&uE}%+Pd*Ne8h@!B~tfzQQKlr9t ziyfywHti;|!=E&@6O5V0Kb9p9`5+87q73IQO^ZE+Oi1TnZ&?jW3yc6b#ZrpTC$bwv z6cQ(4F%7Q0Ov+*%qZfbm%BfNU;jmN0iCkbmqi-C|^znYYb&YXsc4L8j@mvpRFplkk zu7U~$fY^*3*vFn({}|i;a(VI>vSYc-^YbuX|HJRhza=Y=W@lggj>rfG;Q6kj4`)Ao z_Qk551&|7G(;cTrO+P^8=f(SPR~R0S#5zMg4Bqtl$Rs;7#PQlr>Xkx}55Z2eVP;}K z=8fc|(uTDxE{Q<8qZ|P=7bu0k7sv^`x;qCx5q~G2FMWR2J*{J)StAs~{|_JHU{yQe zJk)Y77->$-cq_N3c-2esW|bjgks$JY=Wlqg@rwB0>Z5b6`ieT>q>Sy_P0pA<+V4$v zJDUq=$BtY;-qhYjtZ*RMnF}};aSZz08EkCzb!=cZZ<9~v%?_}{@!GS$XU{(JTi;6? zQOXWljGyqo6VE=6o%7!JN3ui$?y#>p_9*sm&un5#cD!zqil`6e3WLGeeD|G2U0U$} zvE1H0VZNOfbw27Vd8WFKAuh%$+ z{fp1xo3hShU6r|_Sy~EQ=6IGbN5F)hrT|MMV83zY$NpQHe1#NlW_Oldb@Y+!gr9#e z@{>M8ipItcKf<*M{I0kbv=mVDqmhhPUCJ94CvCj8YylO!x1H20g{&k%tXsXV5Qgf# zc!8P|r!u^}n;d479+yS*oo<>6PlrRTNIrrq83)%@->Ue(LgWT33_G22Ig>lFQqtO5SfBy%2-)H_N z5^}M8MjxBH_Ij54=GPp0O+M?u>2iyA z+q`wkGYi|$Rc6{iW z{Gau=h)eE@hs^Ks*QIyyQyF31S2lWYX$relIfi}NTj6w-B8^1?&jY+|8oOS)B7v^L zbzvnMmDY^@xpnQzNCgx?%>-OqiG(Z<(kAyThT`@C2?msq5e%NTf5G{tJ%zQAOl{t^ zW<_=%HK9NvF>iK1n>TMJClKELMWNiBIfLfkq01Jq=U&|A+dvd-E{?t6e71Pup^@No z@Drb9eSiCT*1M&72JFTD0k&`D$zY)5$TGV#j@gcQsLzHcPwEQZh0BC(3gJ*jBoI!M zdt~GS1hQEYfh;E(AYZmDVpQC5*iwKm3S;} zt7W1@A3+fSZ=3xq%C4l*Cq&S;s?Mr{HsH4&*JA9IxbQ}Dfw9;ukP09TR&oJl#Jq_; zOKZ-z#0F&7NHDDA;$B}3#fJ`h|G@H%gnh78WVFQqi z70c)QCKk{yjOp6H_-t4a%=&=;)MvlIGGF|>r$fLZ>D1E5n-^TnbT#YB8U_Pux4>}I z%K>K=u^Mi*qbn>Ji37$BC3ynoVQeXb6S@XMms=Z= z6Av3L1`Gj6g)E5L32Q#)O^?he+f`T#INjs22*jOG?yFBM$M=(zzl_OZWUv%mTU4_gRb8uOVA+KL@- zxx`gl0nG;0SY$RuV_%^jDX7}ZihT^LU$q%PI+dR4t-Qbf-(>!nhPKl(1&|J zeC!pU7sY4uxAmXfI^Hu)kC~#DqKroH&o&?xFnA&s!M_6DYmTrBycZ5v(vElpB^>#l zvQ`hI61`)hl(<7grnx{8A=6ys57aNOV0(^ng_yaK3dc4AX2Q17r+$4e+wj1>zU>#p zF2U9I^2Kv}Epv$Vyw#fWCAE8~L+OvB89Qig?vlfPF8VgW=p}|6^WJtfNB#`0Yf2T0Vu>64xGPgCL10KYZ)9`LK)$3BO9_h zRzd8v7GgQ;vdHxCT<=RDLPH?WzIhi(?J0y}=4DU|e``B%4|Wvy-SMxXwv8sKop@=p zfBOn?+~5tAvFL~ew4L-$5!eZ`{oJ>U!?L}e!Iz?f1lR$SdGdEbs@By}SJ)Vi!ekDU zvoKlLRfHRyyd@LAIGKyr);65-s%4#IKsi7Li5y67?D|nx80*zTw5u*eKr^t&}|A}>2(tFTWfMc&9LEH9fG>)ar2T3hT-fA8+Ze=txI-{gOM!Xp)eV2FUZ?cJ;C?BL z8Ts4!Zj~s*w_A1hhaCt+EbNepD!s^7G0oyht`zOJGAdD_9g%Fnq_5{1&sb1%)3#lr z1_PH!#DYLBUsJLqs`7Z~8dvkJ*@W=Nad*sRiC%CWG2mz?ga;c|;sq;Oh(; zZrZhg6SK|4ZB`TW#A}|36FX`S0xyd|N0zqDzQqsB$5_b2{y`pPd09&#WOJXCxfl61 z&AOzt+`!I0_TEijr zn1Hqdf*rX)LLt*$6eb#XJRr$%(p(rPe}wI>c+&KMPN*VbwG?1OapKX-64OxZ*>jM+ z_{y81>^@|G4R6~SIk5mchPaJ6!C>?CcVFkpU=zkYyC?WENF>`#7xPJWM)^g<;0BBe z-z5RDZIDdZ)v}mpquCI(1>$wJKEJR_7t44i7j>1&Z7T~bT8c^;Lp5GhPfX_-809lQ zHvN!`8ugO4=~1)QoL^ZUtMrzLg%p-(63g}#&m7~tgFZlCPQ)Za(t(pcN?6MGFfCVm zqmV6$pvD60h;eazK)f}t&E*!Zkr4aBG?Q%b?WFsCl8PYyD?m%}!V?d&2YzyAph+4C z#ZgBrOk{`0*vKRU<4kaB6<655C%Eyan@n*Ub8flzI#1h6LB_$ZEDLIj1Ir*$Y%^7l zt7$KhRA6k2BN7N%0my@3vwQ-?0fL>@0`ejy7bq)_l|>RQ1$4**H=m7Wdri5y$UaSb zir(S1yurq|qHw*eZ>6(8{0^0fgq?7-~cWXpTD&Xkz z(u~Vnzo=R+y)mMz9-%!7=Ga)HD`|Fpvrew&VgPYzR&hJC<=EA&#Vh zH}973Dg(3uwiNT#eauj7Nrj>D<&o2dUp3oH$%kpAS<-IjXOu)M5s-?B8E;iu;OO%( z11Dk;20fR$#;(i>3Jc*Y-lhG}*;e9UB^s4`CCfDkh5D}wY9Nqch_NUa5RAf@w?o0A zAQfWYPS>#O0nWw%H`abb+{+%Y)bZx-eSyyW@H>`fmIneDl`$tQfp|G+48{vPL1u*5 zIs{I!GGpOStksTnM&Fao1vFttFdRq)_Kf5KV@$^w(jho4oyhuE=5VqAS_*LT0Hi`O zdh6nHv*xF2ut$a4re4+=X{uayJwkB|Ho^|{T&tXuE)o^t2&+xi&lc5Wx`+Y+V?290 zOv)`2tVBZ)+-@95FrcV)IRw%m-%3g#T^5S{iyXaa%IXA3gQX8R%`=D=;PAs1u$3zo z^KQ#;l|lNK)k%7(LncEr(xNct^t%LdNN+B%LoC23bI&g#flB4C9kNN;+PJOe0t(uZ z3e3aUfn?Z`3aBdX?#Y=Qb_08-%;QnCzEAr=40Mdb$fJcK#x!)&1T zVeq37xc1|N$~lwkYpLyfMZ_YeXFGAp@k`b2jg5OXgY*}%HqNsMTw|f;!HAEa>A^J? zKrXBXoY)o;3@f>S@?HYiRE({@F7Wb)mp-5jIJ^3dBAi2QU z`luX8g;-QZFi>Lw;c zat`~iaILgXZ6X#iJ*%F}H=fB?)1nD(M<^vgJnS?U)&YtH1GN=U*GX#uq#}$>g;fUG zJ^H42;+Ovax8u-K?AtfOw~6iuWrztj9&ie3{ZZ=W(T|Nxh1xkX*oU*749)1%rdOT8 zrY^XQb^qo*UyX12z)hZafxr?7Xe+{mo<|=FQ@vW3ztvZ_nO1jj*boEMSYTXjM7+Sd z7+dq;9`i63%D52-H_e4IXfA!oNd@$SS_{jPh*~n5?i8!yp8H-dWz^eA5#euhQt_{N z(+M#Vi@2ULIL1mUI1!I9q=PSwog;O=7{cx48aBroZ}< zXN|xcSH9b`BPt>dHW^cJ-_%q$-U*Nj*jDW<7Vyr(BIA z{hS2?lEN;KFiRlN#=y^i3Q;6MJ5gezg9O87@<&pEu~ZPOi?OT_=3y+9K>{Ho7eX;} zS%meNT8fu=`gA1^NCgHt!4NbSrcFniXZZ$s6Y3l0g~U(kx6wJ2r#*u;$=aXbkXwwAv9k()es19lZb z&jZ0uqSRKvX{OpT39-P=QmnZ^5i6tumOf-bPAafuWm!bsyy2IalysZ`slXi|@j8S_ z8aR*r5wY+Hl%HbYaZ~LcipFr$Psc&ax)2RxoT~sNHkhcfz}TsKBotT%+;j)^Q-N*R z#nYT9y*9L-{ptt)mNj7_KYi7ak+|cAT^?KB*coUUA}T_G`1JDS5I+I>@1GTp%=qa~ zEDPiU>@6nVdp+y@$$yAh)oZ_7OT*e#K*v#PE7TK&WsvHqgbUClK#*t@AQ?0k7;Lo{ zE^V0gv3-oOQ64*KE|f#eT_0`}CXx$`h1U5cEQvgD_|wf;B8Z(-z(^I=F*`u&A8h%~ z-!G1=1xjl`!1TD;klI2C7NUVWLu4b6kqVi%0*yZ*)=#*}x0`NR-*#zlKKm4V@JIjK zE?b%oyFN}`bwni2uC{OA8|eHFn9!Fjn$3bNc}NFUNt~GUr#U-h`QddoDo$<3kq6{r z{E{nK_QmHLY#RJ-EnPIw6M8JL6C^6z3cxFKYK2(9u{*ekU*##k811l*%-g}U(SSV_ejPyCuaw{auu?k*pe zhO#)tu~mXu+cbbp$2n&nYx>CWYe(TUHrO5F*I&o%x#>{0 zf%7cbPKp8SDqg$h19WX1KBp7v>!VQ$evKUvgQF8&YdbIK+_fH{eDu5P&^iCrXNd;V&LlcZP1wR#W?f!tV zt!F(g_T@jB_J7l zXI{p3dp!Ca>v@f*xn0G^-`vYydi71S%HaIPo~oX9@=89L*V~VHs%R+$#+mEZKj(WA z2Q3B206RFMBorVXxC4K3=w*wf?zc;h@P?gxOTkNk)&iYpK`VakDxxsfgnoN}`qM}b z8-unWXeaRxoQ5%W(^|L?10)qdGNb{?0>-jJ*ayUxU}V^C+{M^22M{&Ru(|O=yF2 zOyftp-4HOLM`cd|OCO-6@EdA7Pb_OsejBos&*w{l?K_~UB`cHLDQywG;TD`{fxX3> z1O1*|c1K~X34*{9h+n@N4Fj+wXs1}-W&7s`<)j$lwV`Ix2Z}qyo}FV-cKOfVQIB zP?are(r&G<-Sjyw`=3{a6<73a<=D=)s}O;8nxe#VbnOuVb4N&j6L*BzJ<$jV$Gn42 zust)+@Q4H_@O=5B(6x*WzvX7nb^H``A{Ee%OltvJ0$6r{*NdH1HJ)dZie<|7x-kN&fPk@lX~a(dVrneZm!H0)uFaTl;FMBMDsGLf8cx(9 zg5`v%-8AG>hw>Z{$o4BRYo#w(pL77CkvqukEPN0PAQf=_M;rl2#kyawWyXQp@`tP+ zQEO4w5-gDw^PLuyRDcYN7e_-XfKY^W3vM0w;+5W4@vL>@ya$^L*x|9|;JCCKAQe&U z0D;DP*Zxvn~?IO>s)!&_>pIKq!XC%Xy@TGr{tl1;aM5$L;U z!%9}RAM{%_+48a{??yIwJ{WQCb+NU;fy8yFm^N{ct$XA#S5g6TfYt)k`*UaaMPj50 zZGuh3tFP^ZOJ4!xD4&e#tD3a%OPjX`l0n*-kht`SBZRwjU1n9)^q{$C!n6Ub6Am;Q zkRLXHBdp$E{3vWQ()yXx9OT0(Sd5_+JG%?WLxKR;lHh1_0d+A5?I_EYZHmt7*}9cn zpvD5@M$HAZj3dEb80UsRtKP(XVXW&plNpo$#!nQDx2FT_zjeYf=zc+T_@?`9ipm25 z6Tdnon%B{0zzJjn|7891ysmvu$Z{Iez*j{a4zNH|alc1WfkGW~&V`YY3h<;E#84At zSssC=0t_=Bd-A1NNW~Jatw@`F1wKQJ!Gk6kXNm!~m4`)1zz*s1X8Vl552wAmDfmDE|f@d78WeQh7E@VMYDzId2L|j|J zZa?^Q%!o5K9bnCx{4v+G493|HZ~Nz+cDR^VJc(F%MapBj-ix1d-a6s2ou(HSt=`9qX#<>8CioY}psH0v-iIq>!7*ZJ<4fB{Y+4HJM={9)*oK<>f4 z^n1*z6TB2_dKu78z|seVsJ4~D!F6C)k=qme`8olkol;%~QTiN~L&OGZQ?ffDkx%(% zYhK8O(#wrKVfV+O3J?pkVm6f>Ea_^ty60Hc3#IPDh}P4;-K^fPZ8a8TpI7?zm$A9V80;)y+PBzS zKtYjQ*zFeKTT1ss*=hiCp-Wn`E{k9VXU)YF+r9CAEE;S4-c7Kr^vqcMfW~PF#1Ud) zx^OH`1S4knA>sZP7Oo-xI!x|gNS2?rH`J*_h+$)=WFm+ zVRtTx=SCz2!Kv3?zRPCp_@mhPe(sN>tGY^BQ(KfZ6ID_H%>c=OvJ-`J`F({GmDi`< z&VX>t>Uv%~qGuf&C_Pi(Ba#Y?>vK(Vfk{X-@Se?OFI2C^3qRxn?Jc0bg=8oOyGuE7 z_<-1S9M|R!5ElEkQV^Y%KwKUmPI)fHDBCLnrVopm5&E#eCuA3cV2~#xAYO$IQjYg$ z-F_O^4iNCDDLTJ$6IVtmu#QPC0uqb8`{g?lux+<^WO$T4_u{t7c2~F3$>yJM_Ib&6 zn3CR1W6cQtRe%ff{f8gTcARrQn_3X9HXaBECm9U>dZ+iXW9I#u%`E-4h9nX#8*+gK zMXiOhNO_eLfpReTFNsopEh?w^As0}?LMnL5G~2uBr{%`t`vhZ6V@5vS9Uve+EP+hS z;7cIEHJ|aJke-%tx=N2kEi8$-zL(!rJKP|NvEUQ5#@M%&bYOY}F>F7b+qKv{QavtX zO*s3*E2)@~%dq)N4yi7G*bPx#u#)LJU$V=#P4DgL3cVBoODRLVhq5mX+peS;QS>is z7`%`kw(1mFR$Hf`v1Vp$G}JgzGZ6=wuoDb48c&V9j}wfG*$H$1H>)m_;F5?9xj;is zbH~UcWm^Lg2um(3nhUP2_(|N_3gh=P4rjf?KUQjbAUoK?r_&OMCk$~q@>G}#IU_Ln zm@>a7dh-J3PVI8VCo5jtl@FNSwQD_7F1|IEgT>&KQat;^i+Em01-1jmoZ5lc@YYj& z>`_bD#N>3-$c5kI&pi!_7drZsvm)75;kCasltW|9YzSjbI27&83ol~(R-RmEBa!6z zfM7f^`Wd#o`)Z50 z-ea`tp)wNIPg(-8B4TqN3oN^4!y~;VEDOuBu!!)0 zW!IzyOvc6}v(L|f!N@@%2e6zJY{|APTUHu{8EK>$P3%h0&7-Bro(ev zNG>?3D2izyVE_pbwOD*ma>2*r@i1-;TUx;)GS=jQwwN;6Mk)~T`EaPsUNsI~=(ZIC zYEJF8x)QpKgPRIfh z(u>VUKFZGN`vz;t_fHRNBo}>b-}Fs>WF!^hW)N}#;_{v0o5fIJ)QGIf#|=9Nh_-a4 zA-{$ajJbb``qzs4UewYQfwos|?qhxp_vL=!H(j6OMdPQzfoKd|cMHoU7nO(YY)oqT z9D)chhNm*@x!nitzu$TzY|Y>mUYCKD)=adIX*(r*uqXXt(C3 zredlFyEb0$%{HS=KOP_^5(=*{N7f$XV$0xrrzIC+<`TJ>N)>pVu0*8@L`W<|TfK_J zid>+r(H&I;(c&S%6-D`Om~ksKy9#yq7EX%7FDl!^a-ClzgH88PQ-Cb@SUI)~6m=8A zRJT8lE0NRyM8ilnN<$B=zJX25y}V)~NG_Thl8o~UUj?x$9{;7&UBG8xd$4x)GHbQbSp;QN+Z0t(x<9#a`DkK&tdP1RHDE$Jt@KRJB5Y@LX@OOT8 z-pOq2v=5oW406$&0$pSF}e^vE`GA(VP@vhs^& z&`B=2=2e;+E9f}&&_7!!uwIwpy5Taq=XKY!!PDGl@^!)h5SDuP^0f2ah(5Vf9YxRqQ$TEUcCLzPxM z!#>GLQ20Txia>3kbm-zE1R)oYTtEVVSQTChTu9xPv585yO&!}|ttY?aD#vy+ zrNRE=6Ksr=g_FMj9qTPMDs0a+Z(s*5yIP8u2gpPxcjawOCp{-fD+U0A{(>j(;I>Op z5!$7UH`^5AT|A$BUD;cNlM3V4o@r8{AHb*@5%P2wM*OC9w~yc4pW2j$d6^ z*!4s*ncoZB3S$!?wZQdhk|87(qBcq{&{`iz3$_*@7r4A*;wNc#gB&tXHF`)mU`%erg(wjTbo%qGI~7k z)&nGLQX$soFJYOY#kde-xTzsE=RS6oGvw~Tc;z?! z7ys<98AyaE2yV6Kb#G)_-urKYRES%iYGhnRmepmQtr`)Kk-}ZS0H_$3LMm3ueWSl; zcZ_JA&rQ6ZHFYN{Q=wOOnkE&M*WKA<=~3E6;@bB5oM|J|{F$#H+x*N4UY}$Bl5% zxjO`iSh&M!=C%?9L)2zF#XT;uKlJkN`qs%8r5BY11IC)0A9~Q7aTwMS6{*0qB5|2v z2@a?HKaoc@$f+w9IE*=c`^b^wEW^d&GRka`Rsbv5C>YC*zke44p@4DbwvT>-^}qBr z#wM=M@uFYe)yn3z%1PaBH!;WslCOCs7f^{D8(Gz7Gd3g_;-|71q!!|*@(y0U(-Lnk zwNU!Ee$?1R)_KXb^bNAca~vnZzC;F_jwo3+nEqf`HlZ7{3IY9VUi?qy_5PP}6^Mgf zg&UkeFgiI&$#EO9t!wXcWvqz+SYfI^OxNu(6DnT(UdsIWb392a_U^AtKIvsHeOrX@ zo-NV}>>DrQea9dpCA`rjXIwmd>wm9#|F)yhud#j ziN36vt6m)<7K6HEccCimw~32}cj|A+I@VMm>hM=N!HK58z4R9=KhIdW#@NwprZ%&X z#6lwOT4qy>C+M22TH+c#4ONn2OwISF0uKu(SV%4Owim@)LTVxGF7%xX>{w!8TWGcwEJ_tfg8$P# z%03sSSBoCHnR-mbLiUrDMJB{JSf4mH*0R@bG2Xy=pY^Q*F&nb~z%F+GuYRT7g8E?J zbI-B^Pd#QzFls?6vQykQ&w~MHfB#`mYLJW1J(&mC(u;=7P@2Am5aKclq$Fp(>{T*` zs;e;|wSZA&kb?fOp@7PxR)j*oeh?<2(Fc0c~dvJ?ZcDr!-@Bqtl?9tTf7z0377FFOAe&mp|G zzp&r+_;wA?d)1qy5=6VX;&ji?H+->vyuABbibk1n^P&M^@={=HaeVmXK0zu_i(|t$ z7`U2TKw2SOQHA89pgg6_568V!Q-O4|FB;Ys?kPztq~I&_H6&$0G>8ub#8VVChQKj* z4=vZY=CNdZPu3EQ$y9Bw|Jje2lUN^8j8j(>wCtmWQbf#rQ;$!hO81*C3?K3&sQ{^X zbn_0^6Rf^urDsy{-1GfaB+T`Mx`xOJa>hkb#T3#_9FxPMN+i!lAdPZaKq%nm31Z^4 zq|@B~f=d|{rPqs7zk}J+)poXn38{s?HuTlA7C|)NW?H;n zl3s*qu&Jp)SVYk^v3b~5XwnL@tuTBRJfG@cMV>oThnONDo~o$*ER**=`qGd4`lo6& z-QK6Su_ymj9Yes_{=^gJbHm0`wBC|RBFXCl8iqhLyb=m5L0X2pjqen<7APw*$p#YI+ZEfW$t~6_EvVYE#lap+8 zbd-&ajRnRN0k{UoD~(g_1|1w;rJOwH*Zl9gXlH@ihFoAMJ;9hBniCA(@28;|Y!(xY zMW+J6&JenL%{F<=ZyK#xcNFB+GwZ;r+YK9^G9(!eW$3)>Tp$izY;Pk-*4iI`+qMmE z!VUn*y(~)w(Li`&E>AJ!`G9o8sw5|M-W49~D5l3#8KekqU-*a_apAcuJlRxWE|7}H zHa<5!ZGfY*Uvl}Z%V25+m(z>btCRv{qOGmXkxb~<1LA?7?(S|jFfb5EiUEnr;JAKt z`(qe!LLJiHxuYD{LXZrboE z!0f3Dvcd0VQcEmzf%x6-Z-0D?qk$t{!sni~%)=BVCC$V!-e+22lhn?whM8_>)xzq! z7bykCfaK+rle~MhjE+od(h#e4c)1QtSH0SQoCYs*)k9lEv9t=04dYC=QVK{WKn$Fu z6Ly#+lS#Ho{6%%|^8lnCW5T682Fb==u@1(@65FkAL*apog zQ=FUCP#55vy`CVLwKKnFBn!rK^ImSRf>6Mn)NKGP?B$%5%RR-Z5NiRcfZc|yU|5&E z;jOH_$1XOKn0fZpX0a`>dnEmGLo7fjIyy?KItj30ED)7Lhq#T8kJ88q3X=Z0Eu;A* z6@MUtP@umc87AaH44>femca4q{YzaVHi1 zWs?dRY|f^%=mJNl8swp)vD%Om-};?`Ud@{~FOZbNYHg5wU~E6+f@?!Om7`Oqi_mvf z{XjBo$ptss4Ao$>XX48htfy*)AzVnXACPT@TwLSzpIq+VYBQQJ1ou*lFB@)&NQgU= zX|M?~?L)u#k>fZ03S-R>_vUVBl_74#ZvNv|LVpb+HB%r*g8P+@3nREdO5D#TKu z7J;{jSGQDnvUo|Wo@Gn;l>52+ifYM71i>h)Me@qSp{YQg?CfUA;b>!40Ca`@I5b-k z@la=BO~fLMhi*m+Fpt%BJwZSlt8>v`tXZ5}^%3n{)`!metjm_p%@7Fdf!lAkZHSk$ zf4d6Xpr+=OtchI(0$c4LIO174As~g6wMGKbeFVV3L z0v$YOi);e_qh z=Bb6jCMF(g=Bk$o{Z{E%5&}dlBms$Gl;yc2HM0Q{L_*X~L<0*h{cjvP^9GhnF2WTl z<*r+QU~8v<7qqu{75?(| zBWy-%uvy&p24(fc20a3FEDr%97V@wJ`Q0ybxIkkGf+1=QWPF3%GXGd$T;mb}?)B19d7gb7x#(WUXnES)XPS2kzHL;zsw97nPaRvefG*qCa>5TsoFqg3)=&9$!LRktc|J znt03I+a{UCX4Td!Z|fu|A09?qCU2B!m(Nq6Od^G2A2!;8h(vGY&l>o{=YDNL&g!zI zmEIwuJT*a!57~=`=T(IDOO&yGuoec-mb^@EA6cP>3njiX(o=h#1`kqtXx-ZM2HEpx zLC7Nd>wE#be>xpaihZCnG>XW~8@lRk?EPSf#n; zoFF&n(p;>WomHEL$gn12g86@{E(9UDaXjs8M2Ekpx)238z2i9btTstErRuk$e+O3P zgCd8nfw?68n%6$M<8dzXgA%(`GABm&xp9%l7MsBl&6Mc79m%#)?ye?^&f?q%I=?jZ zM{R8Eq`waEREUtR|I1rRuOo&-^p$%p<>Jf5v`;)|`&{45#BoY+N9?F&{76!Yl0ka>H-T>Hb-*KKy#4-7{{} zOWoG)!D5Q9a@jh-a=$dkUG_o_ z3E?xWM@)RxGU#XptLf#xj7C&e)|6n>e|gnMKIV(#mOfU~yV)(_y$$=OV0KC`>^Dky z8bN1X1S>gQ#L4!E`%x7mJ7q7<+*!3LY z4benO!U&!Cb%hFM)Ogj}sD3eMu&3J-aOCX5t)5h{8&9A{c|mqOIrS>-|0WeL#NHBg zh&7AHnp)3M05obzJX9Ux%gae8Uvxas;|G6yyf$K1`)#@L(oEaBPeh2W5;G&DYm03r z?es;z;#Y`<02zzpWOtfmeS%aimFUv;HzLnt(({MX%ABvJnSZQ(%{Bb(G^o_m>Fj)J zqwz7eI-0hHwec9*RJ>yqx#^v>ss0x$`NM`rrL{}xVU)W79`cei2VDoW2Rkiw@qfd> zG#=TbGo~6kQqe*=%Z^;qW5s2=2h1T?h@q3+eI(M`Zp2mWsp-Ouxd03wPaGh0Ux`O* zDpR6q${*{Xbt}pr8O!R_!WaHD1v&IY4jq?!tyYsi@sMAXRADuH{0nQ|Mw<+8Qpr0n zyW}6?#W$r`d_~v>#>UN`fA1x(`AGt}y^lZ9Oj=cc>bCLgHStM3ycV?tyO?-AJqdiKi>+HzVrm-5BYXEAz8u4w+dBWyi=E zi&QEI;%deKt;Sit5y4VOloV(D((zA&!mZ6=JO^us@?2}*PE;NZYLT4t`x-JU9giv= zT^yg`@9#a2RgB0bA&m~Y3-^}jXyS~cH4D(C#2Yf|4eTf|cD~mLSPs`G%_$-tQXXi{ z$~AvX9M+MMcSuigc**mk!oRTos8`U|}5tVqdR64rXNCzj)f>{A)?p)Xl$*{(&Hlbom4R*{btu zv)-w~Gdu6j`2Zach-smw=Z^sL*^?EOEtSCk0X=NDGY8KF3fCD6TBiX3GED5k*S|;! z4%P!P@Eb9r9LyF#Rrbs^zfjyM0^5Fsh4RmM$B>kc6YOq@jGfRz^2D5o?5nGy{zR;ftU7osg(AyhyDNM1^ zmYG7vzdhZsIzWwf_|;Cj&^zVQ40*i_$UiJ2=+f*Y#V08l6eUeKLmA*#g94N}mdzoy zWSZWvR}1OLiZRaQYV*ubKx&-!RFt-!Y*^-DeuK7QQpi95eP|V~ggNvf& zcymC3Ul2YOI<<5SL}I+C2WQ3qJt7%WUj&zvF%^hMvC!Ukaj4mY9wLo*32IO_U7g$>Zf7T3OgCzIQK94FVR)AZrjr3 zW&v9&tY4Soo92HQf` z}8T5q6us1pp)xkG+vM166Va%)asj-pLDPgv4|CK zcx%Y;fY<3Q?9y%?c7>ku^|za8D#=79!Yw`r5$Bu&kp7{Jy%ltmQStcg-tCz4K_-svNlCB*^(nS%eIacp@ z$Wi*|ZlS&IRO54VhTk8C>wfLzr#%aeW61TkV=5|>_n0%5$MTMhJe-7Mef^TLd49&^ zr=i06dW3SuL@bkd$|ff$WABl{OZb%pNoI6^7v1V!iu=UIu3~Z{7;<_~j`@^dPvjyt z!P7jv9tGQtHeixp?R~%cYwsQqyser^wtQT^72gzgT?gh(?L^-kJic=Y?iFjf>f_f8 zA;S0-@09q{8&#h%iP2BxekipRKNZFJN;cMs+23h};<=}*Pyz^u&5GLatVZ{~ib1wm zel08&2&op@8H1F1S=vJbRR=VQdq%V+UQ^~;a@N8yB7%=Jo;sz zbHtTpS;?IMi;#CekE39jn9j#OTTU4c)ccH^KC(TNUVp}vL~O)FgyMpeghJYv#L~X) zT(d=O7k(i=vYiwq9fcwO#s42Y#$`QA8bh4r0RN|+LM3K)<9iSE=%GGI{4J?}K$gk* zjN1Z)hs~^VrqAfIX{Y{jw4_)iyKEm%IF7R+e`&ps)rBcd?@&SaWWR4hkL2K&GF8i> z;jt6%9nt#sV-=XxGmqnVYeuDG)@#DVS(Ey>m!GcnMKN-EupuFEy3eR6iA_(6XvJ8R zUvZzyVTwB}39^JaNAtA6tINUzy0Te^l4|rT_J&;YL!N|(EPD6Cr+6t!=K=~D9^k|R zmeQLUyu_tD$u+rz-)t#fe8gyg^}a+Bvf6FBm>ZU^pKDrx)C&w>PsJiK7pC2D*jzn@xmPR?A zS|syYH#BwVkRmz)3G-k9sP>uB(XY>J^3HF68IA#DNDCQ$Be0>WIz<6jcrAaH&Nb_X zF-MAO$2YeJUeMQg9Tk5%UQTpmkL@>ES~d))eo4s4Pb0BQIgC`)bP!$AZJB!+^)a5n zLs!8Y2wqP{uCD9NX|{S0eu?tno0sx))7R$+|E^Cf4P=q3aiOFp=x{JzptP(9!Lmj~ zLWXns&_~(qG$Vx+)paLe)dx6#8}$nF(BK|~ksIAjJCzKA?3XX_&fT(<0VJfNoMyMUzKin}in`2%j+?Es4T z{k+%LZ*yK?CA5EgRZ78pQ6%cYi}44U^j8SUH3*<2?ISa&c6KU4$f@dojt!)Rj5sp1 zeZ+kTC!JCbD~m=Qw!#9bz^l9g@r_WOn*i6zOH|ffdRZb!MB3w)~aAqq|8$ltD-eLKj#= zGC*N+E21I-U|PdTY6R!?1%e7w680jYuwp9;qKSjoK?#;{@g1LZX%p7mi% zY0YhPD0VeeXR7K8{Dg)#xrEF+lZHzRhSqu0{Tg{bn3hXna1!|*8VY2|k~Ic)`oUkkPnLs$llSet?m0cAfBW4B z*djzW)B4?k@FucrcUId=O6!Y8w%XMjSzj#MQ6s#)M`2WK7Qcw+JEmh96`W2 z*mAV0md!SJ4~37$#F?evGz*!7gfA09QHoc*lvtFi8#VD&l0|_nf%gL*BnHQMT|2hP zU3usZ36@qjoMiPvJY{*Tl6)}-ird!%L9z)6^?!`pun&M=T8uR&Kp9w<({G| zb?3<4I#0hqk@65)kqZG$X0kMJR36PLo>qnr%_SKzrElv*JgJhARaC&gNG=79l^E+w zmKM0dwoyPl7`Ltb7mVprnA7l`gFHIEJyrP~Hov8>v&rLNh}v-8slk8Xr>hEtn8Z?h zOI!+5UThuydD?KCGX394hGe7&X4bpI^=x*Bj}&0yV)vTa`D$fC=e8``h`9n@O17$7 zkm1G)Ff1iNdS(&Ev90bmLpxs1hWX8F4za6W)SIypG58(J{{QNdFUZ2bQ+J{^pUhv6&!uf2Nkf47y#z_ZEKNUZM~tyco>e~JOM8&R@fHU! zvrI_`IwsNwTPeC&lw)*MKT(D^5GnWQ!r*|P5lK7=W^qK4q#ywe6cl`bM*iXWWjmc@ zOeyJ2W6Ou)>NTmo&rN}W&=7a0i{$354+!5{r+YQyXV=Rp%;?b8lohASl8mILqezJq zQE|@~J}9xUTacuiX8)7JVC-zPRsx`<<5^#qZY2AtuM-l@FljS!s zyv+tedF!YUXT4k3T-k!ih(VSAvZusX6RnF{pg6afZUB55zECEi&nPt#wkK);h zFcgyjdw-}pQser(CaD{~JoNmGD^>eo8Fc(bkf;~CE=UsV(#3A1M1o?l@|wVgF9#nL zCd0}P4AE?(CIm$ymJ&JHBjhUlb^?j9<3ipALXQwr!prDGry^ptHTU_s7`T@dH-Vul z%%nvFNEEe8SN1LFE%8~2jG4*uGHGx>p;r*L{V1z73MbZvbOfed>|xd_)=$sGL}&E^ z<^g9UF_FtBsdr^Qx|Cm;8Vn7bw{r1u&^xDawI!}Z$EuGtnO&uhlIACojEni=)JoYW z3r128=6VD36Ihn4bnJQ`cK3ZeZ~!y`phZ|bi&GUb#<06$MRrL21QV629TU3g_ZU1z zQeQ__f8cMqq4GQTS=RsxvB6^eX@|Vc!L^l<{yc%PfE5buNg4w9oP#rkDVvu9cUt@` zxY8YlcMXicXPJgm+C5KB)nAIG?C*od9F3{elU+m|^GV8CjHvM&MA>r2IX*m>7T<$} zUnL-~BM~ij+*XUl!pTf$#`#tw7 z-JWIx8xsYNHy=8_t8^`ZlHuGN>(bhLaVy1rm;{hRo+=Jw+DzD?@PW5L&Vj|TmBSi- zE!gnrbXV`G)tE>9S4E~_8ruzQ{=P&u7f*3=40x&)3p5j|*)t-=Y=3_?W&J_=u)w~J zs(XWf-h4WSgd)T>lm{i|Nzqp&3JS%%d`aaC#k9SDZ zBLOQT7Yrnh>p^^pC5c@KnfqC4 z(P>raro`_{tQbXgY)nhI_5H;2Us1h9jMhv`&cQ;2Z2pe0bJ;~?m=fu_=B!I+9KEjn z-7muB<6c#a{(afzSH-HD>Ri2_k9o7v9?lEK8Y@&pE@?bSY>w4TWp>lg=C2_Drx+<^ zC*AaGI&kL+S0gfngM6~t<4;%%aS}3+#c?d0{RsDTc`+jerHLUK)idn{9RRg-5{=eT z>%k8TDHW&%jZni8u zskC+wI#?(6EoSK0yJ2bR{6P%)e~a^sE;#k8ECwQ1by|Dj`KcF)vL;e2P)dZT{ZivT z8nlC)JVCoYT&ekPkIFk?GBOnBEpmJA(p7fLoAo^Yy?#rs&4HLi4@yh8WZ>S1j>m5b$lzYxdL1zkPkWZ=cXYv1 zt2$E%>9Z2~<n6oTQHP_&-zepFJ@HRC!VNfm_^yl8VI+`OTET_S!kTi~@GQj6P$X+Jv79a(xii%c%#kMuC_(<1;Yu(qeM zh`im230&%ndgb&yAUTR0y|M(T$s*oXA_OxX_uoZkr_8}`z+io)2w<0%#zfuVjw?}a z25-DcTPV!`;w4CrV0LJ;u0Mi?37s5ESymYH(b&?ekboPT*_uyq8V$EKSXu5byAViT z52?Uts$KW!Y*W7f{U0b6UkJ*`I_v3A99-=@FI;)Ay{}Gk&TO!#kj|*q?Yro>N35lC zf6gZqubJ9K$fnEl7Z3O7zw}v1#0-i(&X636%h$GLzMy}v@!;+7=33MwSK`yc0MQ|~ zF->`fF8kb3O24MbExr1#cgZ>J9vj)GKbJ=7s{zq@uXi8if4Gnol!bNs8_N zqs@DNijyUvH_FKJ^mrWcbATvHjGb$j2nop2!X0>@t&p?T=Bcvcj=|CuOe*Rhe`oiM zab}M3w##b=2isSjQ72T$vKtmM8g@4#4eJKCba`Tv_&{Diw7V;fvT_{I?P%;U8#G_b zp-CE-w$!+@b`40Y-Z=>)QbI) ze1*@jYPjEWSAW(4ukTHh-2=KWp!;L-Id;khAA+8m-w62%+>>W|n-H^4f2Xj_BBfzU9O%Q&tli`ojlWWcvpcE7 zBZx-l^Rl6w(pYekAw>L-qMQA^E|=$J#xP@j&Y+{1Z1kYf#y5*&D?~Q-CR-L@$`(}-|=ylCan+LlEByGkjktIoAfH6pj zF~l^BnnjGu=--fMQqG?rU^Qt8(`(PB_?9!Zegu~|Y_rGrx^qd^$ z?rgi|QmgT~819)BlJ3t-1WIQe&8-+?1%>S-zeCI7Aq`6tx^R^!t*Cn(zK2e#yrl}N z2loNSW)!%-PSzfwanGV3?h-3wwD?isUA_AZHI+(;HnaCj8EJ)OBH6ndE+Vy%&8rEx zllbc$Owk~y_X+YGIV2}5@i1b?gx7vS1G-tj=Sfi zJ^5=uA`KYRHTDRhN^r2%9pVJm%#gKqA8ed@(Y+XL>c`667t%=t2rfBz#^3^US_YE3HlY#{$!L22ateGa^E+3=LH!l-+xtvdZ;&$i zh01ePBwlly7#NQcb*UZu`MkTH&FIB?)bGx?Gerkh6M#*YR-WUM0v#Nmuj@oNX z(6xs~0=i+1V8GPKA;7G{WNM$So!eLZH6qPvf|=!d;k+~t7*lcMfcBXY&oTEzEk&wPID6r-3fn;y2r~^yI@)l*)f4=iG?uv(cSx-W z#?GYEx7!t6pzsOe?xS|NmP+w`z1-6jsK%tzZ08NtD6d9@5goa*PL|y+UVW=uuL4>s zcG&b3_ITe|yhlPT+Z>EG<&#hc^83a3CN`~H41zH?a|Dm)T+0#+5Vg=HSdk-hn(H^} zf7Yj{n;ysX7trBF?NCe=Yf$352CQKDMIBKD=~FlFS-;T;(88VILsJvGBX6?sKAymP z*DH{04R)k%;be9#Q?{jBhH%_lku~<0^%lU$KJp8ig5%h_=YQyJ~#fqOt8tx}ps(*Vm*ToWuzn%R; zfr$XazS89|qAdG3H&o-lx}NbZpXHp0#R~0jM6N^)g}FRZ|8|q=tV$Ekq4P^@VDSNV z-0#a5-J&gheQ5ufL56SVUA7T9dAblewjy*_&ZEIB0Pd3$X$v-T6OOdgXh4D{049D_ zPf%leHCln$tL87J@7od!b6h6c9m6#VO-w*rIyHB!_xfb=A;;9PEH<*bfyq&L8eW}kIH2cgWnyB>dpP;~0DLYe&8%xK%D0{!T ziwDU%oAnGJv$QdadfO^1Z+`!5kEOQ`CmL@&CB}-)5_P#Bn{s}@oTgBxf~4^vP^4S;l7op==@<4MT*c=BW#Ng;-#VC*nFEXMQ04jhQa5Tp?dV9!HOwN-1w3eB~2>y{hs1Mxg~v( z+ZSVWsIReS0Qva1R3fk_G|ap&1U|4+@PUMgXcj5LLeid{3X7eVhzTbq9D`siY_?{e zsgJ#wHYy|(u79QG#BjOiW5ufa6n0vhn?ZqLP|LYuY`7hT-kxA6#uAPOcL;>y-BRQK z>dBfRu4D}j@oFdA6b`;p2hN?j3M|IP6MwDOwZXv(E|%^4#bOgklKN(bn56`f=zEJzN=-o}sr$V|(ff=uXL89C4V$ec;6+#b7c$Kf&C%$?aD)g@MqnG>)a zt-CPBr9*CSJk+9mRcF}D@-%GB3z0=%(cJyKh4RV3oC05_5-WJ{tO5(^G-FbwQK2Jn zc966+5A z*F%4pA1`b6A9I?*#GfOt+>ib zVhPx+$EXSRA`EYygh&Vp0iCf)uC@19QE5;8@6+Ik1EIbYG#aWuKlY(=Zv-O5pSQEEQrW0MN*a?Pf92c-S_D=H&F|vPh>P2Nl>f2xIs#B z=L8Z2*l|i!CZ-eLuc(%ag-ke5Gy#m6+-X`Ac-{(hGUk2jB`Sd~Dyv4Ca#ya%oMS?( zT3N!#9WsN(7b38(?E4F$_=BW)wPmEFuLhcM;iJ-U09kf;Arbw>`|6)pf=iXvd%@Lh z-jSKu5C^kY-BvIc4;P9!yT)-X-u(1uQhv+17G@Str&MB@%s!!C*C1Q*$XQN!LBcE5 zC|py~X3mrvT+vvr$GN{TiuOfk*y>n(R9ce}%T#)wlC_Y~<#Z-By6h~JIh&ql1z7?+ z@4=6j3 zIn9=6MioSWCg2Pxi@QYQ0b!YG)y&sdbZMf$?IO%EkOnT6C^$jwr0TW$EJJUi;hZt4 z3l^3~rw$|UZ1hjT$dJccvsX=&t~p`UE&^9b2L0FBa>1rr$qhD^11;p%qIA$;f8*b? zF6EhwR7hBk_vUsO=3N6)m`A%=J^UF1bmz{ zOWiv}(2B@?@HY)F>h>FyLNm0I^r8`?9ZcVne%Kne{I-~sSZVZcM#{j!Vpj6?yW+hH zY^&&EKRZcJ3x?iKE58ZT=9mX=#-jQ0$dxBHJqNebG>1=#5QK^m4v z$4!z{_*u>qD99OjUM1ya5Za_8EnU!|^2@R275F(_4BT^R@vG8e**$z#I4m&#)C{2> z=;9ZfR;KPL%H96M2r&)fZ=)bKMnX8am*8Yxz>WBuTj+O|J2_=#&{#;H8dDbPzv651 z(R4?jbr6f@mcAh$`hzd`1sj}2p&~lPh$VJ5Qjt0;?IfC)B+>h$L40%QyO(=zR5x)n zdo6Nl@edEJWR$_&QGyQcG58ZH=XxMJdk}zVxbOGW05xIOG?iL~Zn5On*am~+-VDJC zKcD?l8KdRx;oCA@DrKMUkS>Qn(PsdMAE9R(E@X@UW~wu)15@9MmWWhqmS}4SE_VN| zIt+p=l=l0vip9_{*(j4gg8-Ov_eh!Yj&+25b`;4od<`l3N9$>!20O)9uR(YH@wg>Z zNW(tw<}GIX?oSewS_vfq{U2{wxgVo%en4|?MKGiJBk;so9O7FqlJ(epUQf6(xqAu> zLTPkcJ*(EX##q|YNcrCcYeVg%Nu^MbNJYM7#mA?o=q-|QZZTn2qU?-+I+j!?HI?HX z{Ep7`ll?r|l*X4``9g|wBMvsC$hv_`I|OWTLS;pz6l}89m}vWHc+l6-Aw~;pnNCCW zl+s*Hbb5SIezC4z^l&yjaHrB0jf6t|uy=^B1HDXozxc*1n4P-O!paWzf_4?Ftilkc z@1N&AdmY`sr?w@IB3y?zdI-AHQ!DC~BB7g%%C`@5d$dTi)@7_oeY7ndvonMsNwRKX zjQNKy*F@7yCD=^$WVn_x{YiIyZL*N9HYX4V<{gqZ?qIa)KE_DJY;tCw9Q(%u>d0hvq#tHaJO=l@kA|P!{-z&pK=txryJJ991Zm@+xpTk&%OF z3J4d)|H~0^`v@NrJ5egJIN|2U^X`@v)v$*R$_|^q%90Winr15wdkAY2#M>h^;ZTd; zLF>&B07-r$kA{Jd#=_eW8Nz>MVWDn3%5x58d10dh*={zX=H+8{tWlkuHb&jsW+nx2 zWTgpe4%mLhxX-5&W5#`}O+VKZRfQ1GPpcw*@T3?A=Zp0bAfbol55fVCCMm0B=MWB? z>T?!2592)}DW`>K&*$dOpkT{NvvMjBrtXhgn!X@7Nu{P-dCmZavu@3EiMJ+P1l-wpboBtU#GbnzBX5mqk4hbJb;si&`+IWn4sAu1uF%Ps*07b`6uS7 zbj_;P#y^I)pDUy|pr}o%Y%CN?a}Fx9l6%@+vaNo{zs5Il3;h*bKrhWAGVwI|MssO) zNLBJV7Yp^gL!8E3Ga2_q8wZukt*Cj?H*tGI_Y4VPZB35j>vzX-0jstGO+p$%`N}&s zMXz2KCP@`Y&0X{ffn_=JPdu*wOlAg%iuuAc)r7N`9VuDTi@-mI4cJ< zH0BC(zXIq8m`?3Fk;-^H4tg3j`}5UoSTw013;WA-9g5p12#GG4GU60meI2OU$9nDj|F@#$YZV%`1=&%CXYBl+UFwCxz zo+Zbz;ge!2#N~p$iX9g`E3RmzcZMY6-!W&@M7f<&VA_^!u8WCU7Y0KT-3cyROL{SU z*(CXx#P;YAM*>0uu5dWaQ>!RHt^?dQ3AgLln8XI(8l~M73({r7;j`!mP$0a7WNct)4749)+3{D$w0h6%YKhITb!LL%Sh0xkdPuN z-j*~@hb{R_ysCs7*!4P6*F*v5)ZVR}xR3Cjxx4pSoEQs zL8!r`lZ?(>{L>I48D73r7~}||f)eJr?@t&YpfrGhG@fB;KF^D#yTWE2<;`Y9E@mg( z)YOnG#VpPAN4^0=1m4(6fF&Kt9-FV4{*%HMD9H8)v#pX`^pT!+;uS~o@SD#`jiaJ3 zjyL(a6_rN1I+K>)1w?gvVPdo$JLQ` z@A0NXb)l$mzaXAN5|%;Z-(tKC{UTN~o>e%A3EY{}&V){9z`=gu$YOu&*TQ_xRHfBA)jLjXWh;lOmB@ zToK%om1%{Dyms{f7t#yE`D#`QJSnJ5A!ahHLGjr!Qk@r^#fN2~i^<&L?6k<|Es#mS zaep@htjtZVWX)*$#3-A__7!w_U1@WF8)=Hu)+46Rt!{K4GDvVTkVMTC$z~YJQ8P6m-l=zG_&6LHt(Vk5EUsi7rSDCz7jMcrBU=fT zr=p;&lhz#^Y^k~4WptUI*8N+uFVBFoS^FSzhMLOXQuAPIwjvh`&(CR^X`Ax55#HYM z(eUuU8vJ%Wy73`KQvrX%k*9Hsd%Wf)>|ov#wR2OMSxSZ9=3Dym({X~qWbzb8@+sNa zw3=RvTKqQNv^x+P7*ULihVF{G_$j7-Hb`@TrH`<1Wr7!tm<(vA2Kl}F6aH3L`4%r! z>EbQmU!cuRxQ%m`G3F_L#a0{j&Z^{P6wuF8v}|4SX7o6FTyC5{bQ1XuW{)NK-J2#Np6`10I`f7!(=pR%$!y&~ zijCj6X_`@vKaUkT@MY5vWtF1feFNJdjl8sekZepU(f7=zI`$}!b`S<3zx~eeVZq9z z7TIfG$1gY%556x-p9Ong5~&~jG=u4H`Y6JiKJi3$vhYRtY%MjjGeLUSo06y$2D&~c z!aFllRhs!Low}Qv4h4c{AD>buV9DSE`od30Zpsiy{&3b$8If<(mMKf^^xF{GW2JdXYz;0Qk zsztmGu^EI3UBC7qY6MQx$423ttM*u6jPiqfQ^Q1oO@Z|BUC~Ahf+4lc_f7xRhXF}xsBiBO?#2mDCLE==b9sh1qw=r& zvi-b6GEhzf@1O*!8Zq*R#23*@GDM>&qW^aEAhz3*x3Xr~2Egydu1pTby9A{eZ<8%h z<;RaTU{TdRF^J2_v(Qh9-wv6kjsQF2^@ z6BQ(>XvphDKU;g-ZjSf9P6G~N9F*5Ld*FxbvsI#_dWmpqNuaOmo7j6Qu{Z<2zXiu} z4&{WeK?LTOdhKaB%5sS#eg-zC#at(USj?vO;Faqek(d{eZD-4E$!E(kk-_ILe1cgV z9}v*vXW{aLOIySt@!4%a@!f4Oxjm9&DlrLm9#;Q_xrsL~xuSCvYMsqHXY_j?MZaaD z@47hI29b_%(lDO`CpJl0NYo<@P3a%TXPSEGBA?ZUp|cHN;bd%N%#z@1MN{?QS8^G3 ziU)irBlUpll5ga*yULAY5& zBwR{x7Wy<9k@YkxSu$llly!(|-j~*LT6Hp_6y&5BoQt>QV_5)WG=ymut+^Ahe{c3o zcw|ocA7v$hOgk^5G92o1{lSSW<~ky$dLl;Cl;fwrvm?{Y=RXu08(DMlghmzeZCrAj zKu$om&vlj|C-eqH+TrQO%F$QHK?x~bQ4Ss$K9^)=U#l;G2M-3+VSFpD_>BwsQ+}t> z>6HX5FfWC}g1<7=K&k02tyUIgvf#GK_vQjgIXH+@(U2O)*~ZhYP&tUeVzlDA`hmRARMiNcAxSguU*u&jHZq6v%3uhL@uyNy z0RzMXXECTWM7&f)tChoo3H6P$l{b;e8dbM^?BcC%9DNw~o`z@^XhR%Jv4^Bgv~Ll+ z(XpXlLW-h4wb=NCp(VsCfHT=^QRbpi@TY9(JUrz+CwZp!4h(rgy?IHRQfy#Vj4L?v z58$W44Ixi^4VC{~et9|GUbBp|x0x(6zw&p1?*UqaN>twaE;L84O!jUPZT1Khag zG&S5PlO?V2B=Lpxe+=o?&bEkVrx$7m3-YQRb!?#C0TdKYsGO9zW)tqoGitfEgSaVk zIZPtvD_~^wDovCjA`7-uLgpJ1N5L7feZsE{>)Ku0`BAv5Ph7Q;%l9aFiOYe*4@iN{ zrhCG}6B*dCl@YlF-Yk$A+50B0^b*1|dlyAqXcYO6H9|fOy3qdeze2i6&%JC>d#OHVtl^xI7;^>S zpqUqb7>}B&oO6f!;F;Wm@9`W8iaEzjAdUacZ#KaT zZ&!Z}`io+ToW8B_926#M5@R6(FT&51Tf`1tjx`wD{q6RGZ^)R+b88*FMKLsF0{;2sNP9nh3tPI|Rru=A;5qHSwqjjd(ngW13t*KlqPWGtP3e3E9-HD*hz;j9 z9+@rMSB*;#2)3IYJ{1vrxfKVm;3=2F5Q@E$-?w_U&I5Y&^23w(`(x!KVa zBb2h+hNEQV;2z8gS!e)CL&)u3&uk)f$;g<@idN|oBFR>ujqfeS0{0A7 zlPx|rd8y5I#F3cn05NP7SbzsHD-6x?&?D@+aaPy2%}VhNnHVWv^-{4BjPcnLS&Qsl zp$$6x^P`41ZgIw`&L}kM$ zN9rw*zz5k1v_Wc*7?&NK-dC3esMcx*&05gH(x8SF})*G4psvb++K| zY#F_`XiTj=ha6`m%l`YhPb?KD2CJFk-5h=>0p~H|04TseZLP4{*9o`#=I@y0WD@Jy;Ffq#*({x;vgQr|8`qS;-`rhQ6C{9`LvY@1t zt_HjnIvDQlg_^M3_fs6#3`pCM(_4Rc`fFnVpfzQuztm z2C^A`3n>=+sGx&_=c6mSf7s63)S(cQ-PPT7nP+vEjbVjy!0916%2Laz`I(LX1> z_j_{H#qowH2C4SW57mzHYNn@4p*lnyj==Xkb35=a)j2`$_cEThuaw|{djqK2M^&1f z4;Usz6&NPr{WNi4F0g(;>0N+}GkCU|3j~ zSBY3d8=^^31zJ(zTOc)(UBFvWJeH8Ag(}SdC?eYSrc@lSE90~s$_K9DTb_==KwzK> z7(xYlEYf`z7-X32-Mq5i9AB_C&yD`29KOFX)7kI}NB*-%gGk4Qe8R^hy#&WG966@C zqvM91vPg|*P5aYuDYwjiUANS~)znH4@1qposqiE)k}gdLzrX3mrw3o=>#i)laCG{( z+ingAanZ!o(?ZZ+nT4=nPpx&S9eAk!!iV(b^7*>tseq0oSbiYm3xzP=9LPx>4DUM$ zpYgKv=U1{G4TvXDS3`1pwinr-GF(ITWpk?-@Gir8>@bHG^4QUJ?_*c;J(8L{79$D3 z=1Fy1Kb9eq1HUD?+E{r2XH9(r?}j$d-gJPw2O-owU5k24$(wNuqEr4c0l|-$6m?qHH*s ztaF_4BFB}?7E8{{dKu$pWC!b+aF8JS2X^M#w9PW&FG&>{6lZ5+3NBy~s^I+QtYFR%JcRX*>U9nQo*qX;40 z0G>C0AO0)$5wcTa^QwXj`*Kb$nzYeM-6er+vEht$vfnRK`s zqMo!#ujQbp`b>of!+La+^ybz!+PG{dasBtRe2(n8#cK5?ARD0qbga$Nx0f`IkV?f@ zgyh@HOir~dF3*ff1Sdi?gfglO5Gn|zAO{LWDj%cxR{?M~*BN!xz0JdyK61_)^@VBs zh&+R`aqcIv#yu`_muz*svPFDiZoqON0=`Mu^({|9c)PdU^ai(%b!NlJ8~O$&ANA+y zhkNv>7mK7=4{LwpIqHciNxyVzLHi`N>F4NAS4;;e)kn8D*C7{&Vr(rnzz}jd&`MQ* zDIzJrQ`^H;!<6x-S-?zMvQQgvh`|~9;o`J0WJe_$i!_e9{_fm3#>3z? z%&bj0pniw4o0rv%bJ62Z#yKriPCWi)u0#+|Vd{X1g+N5E+agX?sDkgbGDOblw8PSX zMK0#BG!!QOx8C{tMTWlkB@^Ur^B+q?4?Y!td*?~LJNl!oCiRK8I5pP8BGzwwLk2!N zvsYfA+Fl@CcrQw;V^3+&+@Q$`Xz1zO-T=v-Ztk=wTC@MkudlED$^x&8VUd%QBDxr- zrcWOinjD6v((oaJHkNT;|E?=Zj6GM3M}Tz3o$S{`s^}2%qF``Ojbn1j9>R1Y?yh`( zu+~h&ZiaeS_eS~I_KwcsM{a)V6wmALpxZArE4zVAsYZV6dqKN<}daY&V)dT7j_Ns*F3FR!fXdl%- z%!dAZ*ts!v&^ziV6@~x@*#Mpo+BU9OvhsvJKiVt$KyiTNbA9rpkd~G_E^o=aHS3hM zi8RUi#FZgK{*m5^w!FD*)zE}7)4v$2tk%Q-s*oa3;h*3`EAdQ)xJY>RtWEJVVz~3f zad?LCu3fuQ0^G?#36gI>*Cq921RNSElKtt!i2;2Vqq`YyOMOoE{^z&BR0C`L@O+h> ztq`vg#`FN7h;Z?s}q4cVIvq1{!XA{}KfyU=)dpT}ADlQQ&gFC{9CjwdS$Re4|u96^B}-xj-$-1Ht}x$JDS9~O2<{CL#* zXke7)AqDLhaDNTjC_Z+M$CFl#nwpLU5hd50=RkLhX;Sl)9m6e*r)DL%V;|d#O2OoiuPDE zY_9OYeg>%>g~pQG2uF$ym}VstIK(kQpuPfls_IB);dfUBJ>}O%8iuU$tAm`<%S{kS!)R$l zZtu%f&V#t%Gq)8$%vMJOinI>bl2D*@=RKwKmS-qsrvB{KXIHtRZqgoxpu%f{w3;F* zIm$d0kcHYYWR>94Q}_6x zz!2dv2(72v)RXsT1#EE!NqpE?*eCZLk2;An`Zkc+6YdfvRexC(1vAx>A?n%df@J{6 zLctJ#NP)9xyK*!^IiXQg-_r_pX{i2F>Ym!J5{{$&j=RLo28JO=GUpp1A>&ou{ZUW6 zA!rh$m%WkX@=e(8T@#q7w46gh~%KuuVo`8o;o!;!FImXE$oPw%2_ zQXWoPAN+MoiSAOFeQlrKxY9j<_ms=~gHT4kh1nI$IR!B?|@VlM0q7~+G)nFYy!9u)Y} zVu(7a2_Xg)3`0Pdyi!xtAF~Cy<(n|+RzcN+RJ?c=x={}Kb|;vU1UMcODR9Fx+F_<9 zn1{0JNUuztmNf8xGTUxV6T73Oj%{&29^133_qrWI9v5tg9AAGK@=TRn;*P?4Rk{U} zxcxil$Lx&k0;*qCsOn#XUDp<{8ro9joRgP@h<^O!t)Zzw39G50g}xF3bihvPa_wGc-@kD2M)Zoac@z_64n6W#5L>a ziKg9Plj^03N(NJ_EOc1Es@b?qyXG)Mn$76u)ccAHgQ8X= zJ~WrkI0?z`0~U1w?ELTbp9}t9v!HViZhA)ql^$*RR_Nle5b!uU9JRlXC1m^$^E2B` literal 0 HcmV?d00001 diff --git a/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh b/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh deleted file mode 100755 index 26158570e2..0000000000 --- a/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/bin/bash - -# make sure the following line is added to `~/.bashrc` -source "$HOME/.cargo/env" - -# assign local ip by curl from AWS metadata server: -# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html -AWS_METADATA_IP=`curl http://169.254.169.254/latest/meta-data/local-ipv4` -orchestrator_url=http://"$AWS_METADATA_IP":4444 -cdn_marshal_address="$AWS_METADATA_IP":9000 -keydb_address=redis://"$AWS_METADATA_IP":6379 -current_commit=$(git rev-parse HEAD) -commit_append="" - -# Check if at least two arguments are provided -if [ $# -lt 1 ]; then - echo "Usage: $0 " - exit 1 -fi -REMOTE_USER="$1" - -# this is to prevent "Error: Too many open files (os error 24). Pausing for 500ms" -ulimit -n 65536 -# build to get the bin in advance, uncomment the following if built first time -# just example validator-push-cdn -- http://localhost:4444 & -# # remember to sleep enough time if it's built first time -# sleep 3m -# for pid in $(ps -ef | grep "validator" | awk '{print $2}'); do kill -9 $pid; done - -# docker build and push -docker build . -f ./docker/validator-cdn-local.Dockerfile -t ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio -docker push ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio - -# ecs deploy -ecs deploy --region us-east-2 hotshot hotshot_centralized -i centralized ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio -ecs deploy --region us-east-2 hotshot hotshot_centralized -c centralized ${orchestrator_url} - -# runstart keydb -# docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb & -# server1: marshal -echo -e "\e[35mGoing to start cdn-marshal on local server\e[0m" -just example cdn-marshal -- -d redis://localhost:6379 -b 9000 & -# remember to sleep enough time if it's built first time - -# Function to round up to the nearest integer -round_up() { - echo "scale=0;($1+0.5)/1" | bc -} - -# for a single run -# total_nodes, da_committee_size, transactions_per_round, transaction_size = 100, 10, 1, 4096 -for total_nodes in 10 50 100 200 500 1000 -do - for da_committee_size in 10 - do - if [ $da_committee_size -le $total_nodes ] - then - for transactions_per_round in 1 10 - do - for transaction_size in 100000 1000000 10000000 20000000 - do - for fixed_leader_for_gpuvid in 1 - do - if [ $fixed_leader_for_gpuvid -le $da_committee_size ] - then - for rounds in 100 - do - # server1: broker - echo -e "\e[35mGoing to start cdn-broker on local server\e[0m" - just example cdn-broker -- -d redis://localhost:6379 \ - --public-bind-endpoint 0.0.0.0:1740 \ - --public-advertise-endpoint local_ip:1740 \ - --private-bind-endpoint 0.0.0.0:1741 \ - --private-advertise-endpoint local_ip:1741 & - # server2: broker - # make sure you're able to access the remote host from current host - echo -e "\e[35mGoing to start cdn-broker on remote server\e[0m" - BROKER_COUNTER=0 - for REMOTE_BROKER_HOST in "$@"; do - if [ "$BROKER_COUNTER" -ge 1 ]; then - echo -e "\e[35mstart broker $BROKER_COUNTER on $REMOTE_BROKER_HOST\e[0m" - ssh $REMOTE_USER@$REMOTE_BROKER_HOST << EOF -cd HotShot -nohup bash scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh ${keydb_address} > nohup.out 2>&1 & -exit -EOF - fi - BROKER_COUNTER=$((BROKER_COUNTER + 1)) - done - - # start orchestrator - echo -e "\e[35mGoing to start orchestrator on local server\e[0m" - just example orchestrator -- --config_file ./crates/orchestrator/run-config.toml \ - --orchestrator_url http://0.0.0.0:4444 \ - --total_nodes ${total_nodes} \ - --da_committee_size ${da_committee_size} \ - --transactions_per_round ${transactions_per_round} \ - --transaction_size ${transaction_size} \ - --rounds ${rounds} \ - --fixed_leader_for_gpuvid ${fixed_leader_for_gpuvid} \ - --cdn_marshal_address ${cdn_marshal_address} \ - --commit_sha ${current_commit}${commit_append} & - sleep 30 - - # start validators - echo -e "\e[35mGoing to start validators on remote servers\e[0m" - ecs scale --region us-east-2 hotshot hotshot_centralized ${total_nodes} --timeout -1 - base=100 - mul=$(echo "l($transaction_size * $transactions_per_round)/l($base)" | bc -l) - mul=$(round_up $mul) - sleep_time=$(( ($rounds + $total_nodes / 2 ) * $mul )) - echo -e "\e[35msleep_time: $sleep_time\e[0m" - sleep $sleep_time - - # kill them - echo -e "\e[35mGoing to stop validators on remote servers\e[0m" - ecs scale --region us-east-2 hotshot hotshot_centralized 0 --timeout -1 - for pid in $(ps -ef | grep "orchestrator" | awk '{print $2}'); do kill -9 $pid; done - # shut down brokers - echo -e "\e[35mGoing to stop cdn-broker\e[0m" - killall -9 cdn-broker - BROKER_COUNTER=0 - for REMOTE_BROKER_HOST in "$@"; do - if [ "$BROKER_COUNTER" -ge 1 ]; then - echo -e "\e[35mstop broker $BROKER_COUNTER on $REMOTE_BROKER_HOST\e[0m" - ssh $REMOTE_USER@$REMOTE_BROKER_HOST "killall -9 cdn-broker && exit" - fi - BROKER_COUNTER=$((BROKER_COUNTER + 1)) - done - # remove brokers from keydb - # you'll need to do `echo DEL brokers | keydb-cli -a THE_PASSWORD` and set it to whatever password you set - echo DEL brokers | keydb-cli - # make sure you sleep at least 1 min - sleep $(( $total_nodes + 60)) - done - fi - done - done - done - fi - done -done - -# shut down all related threads -echo -e "\e[35mGoing to stop cdn-marshal\e[0m" -killall -9 cdn-marshal -# for pid in $(ps -ef | grep "keydb-server" | awk '{print $2}'); do sudo kill -9 $pid; done \ No newline at end of file diff --git a/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh b/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh deleted file mode 100755 index 9e486f9cea..0000000000 --- a/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh +++ /dev/null @@ -1,168 +0,0 @@ -#!/bin/bash - -# make sure the following line is added to `~/.bashrc` -source "$HOME/.cargo/env" - -# assign local ip by curl from AWS metadata server: -# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html -AWS_METADATA_IP=`curl http://169.254.169.254/latest/meta-data/local-ipv4` -orchestrator_url=http://"$AWS_METADATA_IP":4444 -cdn_marshal_address="$AWS_METADATA_IP":9000 -keydb_address=redis://"$AWS_METADATA_IP":6379 -current_commit=$(git rev-parse HEAD) -commit_append="_gpu" - -# Check if at least two arguments are provided -if [ $# -lt 3 ]; then - echo "Usage: $0 " - exit 1 -fi -REMOTE_USER="$1" -REMOTE_GPU_HOST="$2" - -# this is to prevent "Error: Too many open files (os error 24). Pausing for 500ms" -ulimit -n 65536 - -# build to get the bin in advance, uncomment the following if built first time -just example_fixed_leader validator-push-cdn -- http://localhost:4444 & -# remember to sleep enough time if it's built first time -sleep 3m -for pid in $(ps -ef | grep "validator" | awk '{print $2}'); do kill -9 $pid; done - -# docker build and push -docker build . -f ./docker/validator-cdn-local.Dockerfile -t ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio -docker push ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio - -# ecs deploy -ecs deploy --region us-east-2 hotshot hotshot_centralized -i centralized ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio -ecs deploy --region us-east-2 hotshot hotshot_centralized -c centralized ${orchestrator_url} - -# runstart keydb -# docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb & -# echo DEL brokers | keydb-cli -# server1: marshal -echo -e "\e[35mGoing to start cdn-marshal on local server\e[0m" -just example cdn-marshal -- -d redis://localhost:6379 -b 9000 & -# remember to sleep enough time if it's built first time - -# Function to round up to the nearest integer -round_up() { - echo "scale=0;($1+0.5)/1" | bc -} - -# for a single run -# total_nodes, da_committee_size, transactions_per_round, transaction_size = 100, 10, 1, 4096 -for total_nodes in 10 50 100 200 500 1000 -do - for da_committee_size in 5 10 50 100 - do - if [ $da_committee_size -le $total_nodes ] - then - for transactions_per_round in 1 10 - do - for transaction_size in 100000 1000000 10000000 20000000 - do - for fixed_leader_for_gpuvid in 1 5 10 - do - if [ $fixed_leader_for_gpuvid -le $da_committee_size ] - then - for rounds in 100 - do - # server1: broker - echo -e "\e[35mGoing to start cdn-broker on local server\e[0m" - just example cdn-broker -- -d redis://localhost:6379 \ - --public-bind-endpoint 0.0.0.0:1740 \ - --public-advertise-endpoint local_ip:1740 \ - --private-bind-endpoint 0.0.0.0:1741 \ - --private-advertise-endpoint local_ip:1741 & - # server2: broker - # make sure you're able to access the remote host from current host - echo -e "\e[35mGoing to start cdn-broker on remote server\e[0m" - BROKER_COUNTER=0 - for REMOTE_BROKER_HOST in "$@"; do - if [ "$BROKER_COUNTER" -ge 2 ]; then - echo -e "\e[35mstart broker $((BROKER_COUNTER - 1)) on $REMOTE_BROKER_HOST\e[0m" - ssh $REMOTE_USER@$REMOTE_BROKER_HOST << EOF -cd HotShot -nohup bash scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh ${keydb_address} > nohup.out 2>&1 & -exit -EOF - fi - BROKER_COUNTER=$((BROKER_COUNTER + 1)) - done - - # start orchestrator - echo -e "\e[35mGoing to start orchestrator on local server\e[0m" - just example_fixed_leader orchestrator -- --config_file ./crates/orchestrator/run-config.toml \ - --orchestrator_url http://0.0.0.0:4444 \ - --total_nodes ${total_nodes} \ - --da_committee_size ${da_committee_size} \ - --transactions_per_round ${transactions_per_round} \ - --transaction_size ${transaction_size} \ - --rounds ${rounds} \ - --fixed_leader_for_gpuvid ${fixed_leader_for_gpuvid} \ - --cdn_marshal_address ${cdn_marshal_address} \ - --commit_sha ${current_commit}${commit_append} & - sleep 30 - - # start leaders need to run on GPU FIRST - # and WAIT for enough time till it registered at orchestrator - # make sure you're able to access the remote nvidia gpu server - echo -e "\e[35mGoing to start leaders on remote gpu server $REMOTE_GPU_HOST\e[0m" - - ssh $REMOTE_USER@$REMOTE_GPU_HOST << EOF -cd HotShot -nohup bash scripts/benchmark_scripts/benchmarks_start_leader_gpu.sh ${fixed_leader_for_gpuvid} ${orchestrator_url} > nohup.out 2>&1 & -exit -EOF - - sleep 1m - - # start validators - echo -e "\e[35mGoing to start validators on remote cpu servers\e[0m" - ecs scale --region us-east-2 hotshot hotshot_centralized $(($total_nodes - $fixed_leader_for_gpuvid)) --timeout -1 - base=100 - mul=$(echo "l($transaction_size * $transactions_per_round)/l($base)" | bc -l) - mul=$(round_up $mul) - sleep_time=$(( ($rounds + $total_nodes / 2) * $mul )) - echo -e "\e[35msleep_time: $sleep_time\e[0m" - sleep $sleep_time - - # kill them - # shut down nodes - echo -e "\e[35mGoing to stop validators on remote cpu servers\e[0m" - ecs scale --region us-east-2 hotshot hotshot_centralized 0 --timeout -1 - # shut down leaders on gpu - echo -e "\e[35mGoing to stop leaders on remote gpu server\e[0m" - ssh $REMOTE_GPU_USER@$REMOTE_GPU_HOST "for pid in $(ps -ef | grep "validator" | awk '{print $2}'); do kill -9 $pid; done && exit" - echo -e "\e[35mGoing to stop orchestrator\e[0m" - for pid in $(ps -ef | grep "orchestrator" | awk '{print $2}'); do kill -9 $pid; done - # shut down brokers - echo -e "\e[35mGoing to stop cdn-broker\e[0m" - killall -9 cdn-broker - BROKER_COUNTER=0 - for REMOTE_BROKER_HOST in "$@"; do - if [ "$BROKER_COUNTER" -ge 2 ]; then - echo -e "\e[35mstop broker $((BROKER_COUNTER - 1)) on $REMOTE_BROKER_HOST\e[0m" - ssh $REMOTE_USER@$REMOTE_BROKER_HOST "killall -9 cdn-broker && exit" - fi - BROKER_COUNTER=$((BROKER_COUNTER + 1)) - done - # remove brokers from keydb - # you'll need to do `echo DEL brokers | keydb-cli -a THE_PASSWORD` and set it to whatever password you set - echo DEL brokers | keydb-cli - # make sure you sleep at least 1 min to wait for DB to forget brokers and marshals - sleep $(( $total_nodes + 60)) - done - fi - done - done - done - fi - done -done - -# shut down all related threads -echo -e "\e[35mGoing to stop cdn-marshal\e[0m" -killall -9 cdn-marshal -# for pid in $(ps -ef | grep "keydb-server" | awk '{print $2}'); do sudo kill -9 $pid; done \ No newline at end of file diff --git a/scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh b/scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh deleted file mode 100755 index e438ce8692..0000000000 --- a/scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# make sure the following line is added to `~/.bashrc` -source "$HOME/.cargo/env" - - -if [ -z "$1" ]; then - echo "No arguments provided. Usage: $0 " - exit 1 -fi - -echo "Argument 1: $1" -keydb_address="$1" - -just example cdn-broker -- -d $keydb_address \ - --public-bind-endpoint 0.0.0.0:1740 \ - --public-advertise-endpoint local_ip:1740 \ - --private-bind-endpoint 0.0.0.0:1741 \ - --private-advertise-endpoint local_ip:1741 & diff --git a/scripts/benchmark_scripts/benchmarks_start_leader_gpu.sh b/scripts/benchmark_scripts/benchmarks_start_leader_gpu.sh deleted file mode 100755 index 353516eb68..0000000000 --- a/scripts/benchmark_scripts/benchmarks_start_leader_gpu.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# make sure the following line is added to `~/.bashrc` -source "$HOME/.cargo/env" - -# Check if at least two arguments are provided -if [ $# -lt 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -echo "Argument 1: $1" -echo "Argument 2: $2" -fixed_leader_for_gpuvid="$1" -orchestrator_url="$2" - -just example_gpuvid_leader multi-validator-push-cdn -- $fixed_leader_for_gpuvid $orchestrator_url & - - \ No newline at end of file diff --git a/scripts/benchmarks_results/README.md b/scripts/benchmarks_results/README.md deleted file mode 100644 index e67ba6095d..0000000000 --- a/scripts/benchmarks_results/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## How to run the benchmarks - -- To run it locally, check out `crates/examples/push-cdn/README.md`. -- To run it in AWS, take a look at `scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh` and change value of parameters as you'd like, make sure you've installed everything needed in the script and have access to needed servers (and have hotshot on those servers), then start `key-db` in one `tmux` session, in another session, enter `HotShot`, run `./scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh [YOUR_NAME] [REMOTE_BROKER_HOST_PUBLIC_IP_1] [REMOTE_BROKER_HOST_PUBLIC_IP_2] ...`. If you want to run leaders on GPU, for the last step, run `./scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh [YOUR_NAME] [REMOTE_GPU_HOST] [REMOTE_BROKER_HOST_PUBLIC_IP_1] [REMOTE_BROKER_HOST_PUBLIC_IP_2] ...` instead (e.g. `./scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh sishan 11.111.223.224 3.111.223.224`). -- When running on a large group of nodes (1000 etc.), it might take too long for all nodes to post "start", you can use `manual_start` when you think there're enough nodes connected: -``` -export ORCHESTRATOR_MANUAL_START_PASSWORD=password -curl -X POST http://172.31.8.82:4444/v0/api/manual_start -d 'password' -``` - -## How to view the results - -- Three ways to gather the results - - (recommended) check out`scripts/benchmarks_results/results.csv` on EC2, where you should have organized overall results. - - check out Datadog under `host:/hotshot/` where you have stats for each individual validator but it's hard to track since they’re distributed. - - wait for the output of orchestrator in local terminal, where the results are not that organized if you do multiple runs, also hard to track. - -- Explanation on confusing arguments - - `partial_results`: Whether the results are partially collected. It's set to "One" when the results are collected for one node; "HalfDA" when the results are collected for the number equals to DA_committee_number / 2; "Half" when the results are collected for half running nodes; "Full" if the results are successfully collected from all nodes. The reason is sometimes we'll get high throughput however not all the nodes can terminate successfully (I suspect the reason is that some of them fall behind when fetching large transactions). \ No newline at end of file diff --git a/scripts/benchmarks_results/results_upload.csv b/scripts/benchmarks_results/results_upload.csv deleted file mode 100644 index 892d93df07..0000000000 --- a/scripts/benchmarks_results/results_upload.csv +++ /dev/null @@ -1,160 +0,0 @@ -commit_sha,total_nodes,da_committee_size,transactions_per_round,transaction_size,rounds,leader_election_type,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -test_large_tx_local,10,5,1,2000016,10,hotshot::traits::election::static_committee::StaticElectionConfig,25,11,44,977785,22,45,9,0 -commit_sha,total_nodes,da_committee_size,transactions_per_round,transaction_size,rounds,leader_election_type,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -nginx_script_test,10,5,1,1000016,20,hotshot::traits::election::static_committee::StaticElectionConfig,36,6,83,1317668,112,85,19,0 -commit_sha,total_nodes,da_committee_size,transactions_per_round,transaction_size,rounds,leader_election_type,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -test50nodes,50,5,1,1000016,20,hotshot::traits::election::static_committee::StaticElectionConfig,26,6,58,2372919,140,60,19,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,10,5,1,1,1000016,50,fixed-leader-election,Partial,19,1,63,4859452,311,64,50,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,10,5,1,1,1000016,50,fixed-leader-election,Complete,14,1,58,4929903,281,57,50,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,100,10,1,1,1000016,50,fixed-leader-election,Half,48,1,132,8137534,1066,131,50,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,100,10,1,1,1000016,100,fixed-leader-election,HalfDA,87,2,254,6949606,2064,297,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,50,10,1,1,1000016,50,fixed-leader-election,Partial,48,3,128,7640747,978,128,50,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,50,10,1,1,1000016,100,fixed-leader-election,Partial,49,1,142,4759732,1109,233,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,100,10,1,10,1000016,100,fixed-leader-election,HalfDA,59,7,161,3139208,970,310,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,100,10,1,1,10000016,100,fixed-leader-election,Half,17,8,33,449102,15,335,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,200,10,1,1,1000016,100,fixed-leader-election,Half,97,4,280,7056362,2258,325,100,2 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,500,10,1,1,1000016,100,fixed-leader-election,HalfDA,95,8,269,3871646,1417,366,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,1000,10,1,1,1000016,100,fixed-leader-election,Half,142,17,423,1908157,1080,626,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,10,5,1,1,100016,100,fixed-leader-election,Full,1,0,7,1843152,516,29,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,10,10,1,10,100016,100,fixed-leader-election,Full,1,0,2,3097269,960,31,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,50,5,1,1,100016,100,fixed-leader-election,Half,17,1,60,6179321,3707,60,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,50,10,1,10,100016,100,fixed-leader-election,HalfDA,38,1,109,3492698,7997,229,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,50,10,1,1,1000016,100,fixed-leader-election,HalfDA,52,1,155,5478535,1271,232,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,1,1000016,100,fixed-leader-election,HalfDA,80,2,236,6540455,1864,285,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,200,10,1,1,1000016,100,fixed-leader-election,HalfDA,99,3,291,7201334,2362,328,102,2 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,500,10,1,1,1000016,100,fixed-leader-election,HalfDA,110,8,319,4476755,1728,386,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,1000,10,1,1,1000016,100,fixed-leader-election,Half,147,13,371,2234382,1249,560,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,1,1000016,100,fixed-leader-election,HalfDA,102,2,284,7881746,2530,321,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,1,100016,100,fixed-leader-election,Half,29,1,84,5523502,4639,84,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,1,10000016,100,fixed-leader-election,Half,16,7,32,425532,16,376,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,10,1000016,100,fixed-leader-election,Half,18,8,36,382506,153,400,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,10,100016,100,fixed-leader-election,Half,47,2,137,3823773,10399,273,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,100,1,10,100016,100,fixed-leader-election,Full,2,2,5,1477159,960,65,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,100,100016,100,fixed-leader-election,Half,9,4,11,170343,700,411,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,500,10,1,1,1000016,100,fixed-leader-election,Half,93,8,257,5577012,1885,338,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,200,10,1,1,1000016,100,fixed-leader-election,Half,101,3,279,9168400,2888,419,100,8 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,100,10,1,1,1000016,100,fixed-leader-election,Half,81,2,240,8896939,2500,283,102,4 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,50,10,1,1,100016,100,fixed-leader-election,Full,12,0,29,6032337,3076,51,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,50,10,1,1,1000016,100,fixed-leader-election,Half,45,1,130,5372813,1182,220,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,10,10,1,10,1000016,100,fixed-leader-election,Full,8,3,14,4481203,950,213,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,10,10,1,10,100016,100,fixed-leader-election,Full,1,0,3,3097269,960,31,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,1000,10,1,1,1000016,100,fixed-leader-election,Half,140,13,363,2636913,1416,537,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,100,10,1,1,10000016,100,fixed-leader-election,Half,27,11,68,585586,26,444,102,2 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,10,5,1,1,1000016,100,fixed-leader-election,Full,1,0,4,3310397,96,29,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_simple_builder_fixed_leader,10,5,1,1,1000016,100,fixed-leader-election,Full,1,1,6,2341500,96,41,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,50,10,1,1,1000016,100,fixed-leader-election,Full,1,1,6,2666709,96,37,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,50,10,1,10,1000016,100,fixed-leader-election,Full,4,1,6,8275994,960,117,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,50,10,5,10,1000016,100,fixed-leader-election,Full,8,4,13,4363706,960,221,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,1,1000016,100,fixed-leader-election,Full,1,1,6,1920030,96,50,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,1,10000016,100,fixed-leader-election,Full,5,5,9,6442963,96,149,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,1,10000016,100,fixed-leader-election,Full,5,5,9,6442963,96,149,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,10,1000016,100,fixed-leader-election,Full,6,5,8,6400102,960,150,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,200,10,1,1,1000016,100,fixed-leader-election,Full,2,2,7,1280020,96,75,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,200,10,1,1,10000016,100,fixed-leader-election,Full,11,8,14,3428576,96,281,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,1,15000016,100,fixed-leader-election,Half,10,9,11,5691705,96,253,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,1,20000016,100,fixed-leader-election,Half,12,12,14,6018813,96,319,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,10,2000016,100,fixed-leader-election,Half,12,8,15,6241092,958,307,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_simple_builder_fixed_leader,100,10,1,1,1000016,100,fixed-leader-election,Half,1,1,7,1882383,96,51,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_simple_builder_fixed_leader,100,10,1,10,2000016,50,fixed-leader-election,Full,13,9,15,5380160,460,171,50,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,10,5,1,1,1000016,100,static-leader-selection,Full,1,1,5,3057191,107,35,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,100,10,1,1,1000016,100,static-leader-selection,Full,1,1,7,1515649,97,64,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,100,10,1,10,1000016,100,static-leader-selection,HalfDA,7,3,43,6770294,1090,161,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,100,10,1,1,20000016,100,static-leader-selection,Half,13,8,21,6815292,107,314,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,100,10,1,1,10000016,100,static-leader-selection,Half,7,5,49,7023820,118,168,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,200,10,1,1,10000016,100,static-leader-selection,Half,12,9,18,4857150,136,280,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,500,10,1,1,1000016,100,static-leader-selection,Full,8,7,18,430500,96,223,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,1000,10,1,1,1000016,100,static-leader-selection,Half,20,18,26,188238,96,510,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,500,10,1,1,5000016,100,static-leader-selection,Half,14,9,63,1839768,124,337,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,1000,10,1,1,5000016,100,static-leader-selection,Half,35,19,90,552660,106,959,117,17 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,100,10,1,10,2000016,100,static-leader-selection,HalfDA,21,7,89,7342677,1318,359,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker,500,10,1,1,10000016,100,static-leader-selection,Half,20,15,77,3225063,139,431,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker,500,10,1,1,5000016,100,static-leader-selection,Half,10,9,15,1946571,102,262,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3brokers_my,500,10,1,1,10000016,100,static-leader-selection,Half,23,12,90,2453187,131,534,104,4 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3brokers,500,10,1,1,10000016,100,static-leader-selection,Half,22,13,86,2351742,115,490,104,4 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker,100,10,1,1,10000016,100,static-leader-selection,HalfDA,10,4,105,5670112,110,194,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker,100,10,1,1,20000016,100,static-leader-selection,Half,16,7,96,6749231,109,323,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker,500,10,1,1,10000016,100,static-leader-selection,Half,15,11,33,3202104,122,381,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -4broker,1000,10,1,1,10000016,100,static-leader-selection,HalfDA,148,19,1059,323529,66,2040,157,57 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker_cdn_improve,500,10,1,1,20000016,100,static-leader-selection,HalfDA,40,14,284,3643619,137,752,112,12 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker_cdn_improve,1000,10,1,1,10000016,100,static-leader-selection,Half,39,19,175,1528928,111,727,104,4 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker_cdn_improve,500,10,1,1,10000016,100,static-leader-selection,Half,13,10,77,3260874,105,322,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -,10,10,1,1,100016,100,static-leader-selection,Full,1,0,30,181510,98,54,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -ag_latest_main,100,10,1,1,10000016,100,static-leader-selection,Full,8,7,39,4292042,97,226,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -549d41f,10,5,1,1,1000016,100,static-leader-selection,Full,2,1,32,1447784,97,68,100,0 \ No newline at end of file diff --git a/scripts/count_fds.sh b/scripts/count_fds.sh deleted file mode 100755 index 0e9e52131e..0000000000 --- a/scripts/count_fds.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -# USAGE: periodically print number of file descriptors in use -# will work with gnu coreutils - -for i in {0..100000} -do - echo NUM FDS: $(lsof | grep -i "udp" | grep -i "libp2p" | tr -s ' ' | cut -d" " -f11 | sort | uniq | wc -l) - sleep 0.2s -done diff --git a/scripts/nix_bump_pr_changes.py b/scripts/nix_bump_pr_changes.py deleted file mode 100755 index 0d9f2296cd..0000000000 --- a/scripts/nix_bump_pr_changes.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 - -# Utility script to show the github commit diff when a `update_flake_lock_action` PR is made. -# -# To run, pipe the contents of the `Flake lock file updates:` into this file -# -# e.g. `cat updates.txt | ./scripts/nix_bump_pr_changes.py` -# -# The output of this script should be pasted as a reply to that PR - -import sys -import re - -name_commit_regex = re.compile(r"'github:([^\/]+\/[^\/]+)\/([^']+)") -prev = '' - -for line in sys.stdin: - line = line.rstrip() - if line.startswith(" 'github:"): - prev = line - if line.startswith(" → 'github:"): - - [_, repo, start_commit, _] = name_commit_regex.split(prev) - [_, _, end_commit, _] = name_commit_regex.split(line) - print("- [ ] " + repo + ": [repo](https://github.com/" + repo + ") | [commits this PR](https://github.com/" + repo + "/compare/" + start_commit + ".." + end_commit + ")") - - diff --git a/scripts/runfail.sh b/scripts/runfail.sh deleted file mode 100755 index 38e882b6d6..0000000000 --- a/scripts/runfail.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Runs a command until it fails. -# Useful for running overnight to see if tests don't fail sporadically. -# -# Usage: -# `./scripts/runfail.sh cargo test --profile=release-lto --features=full-ci --manifest-path testing/Cargo.toml` - -while [ $? -eq 0 ]; do - $@ -done From 8d43ba060d71a306c6ea0db007b0835103f6c588 Mon Sep 17 00:00:00 2001 From: Rob Date: Mon, 9 Dec 2024 17:13:49 -0500 Subject: [PATCH 12/16] clippy --- crates/examples/coordinator.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/crates/examples/coordinator.rs b/crates/examples/coordinator.rs index 1756ba6467..bdff1669d8 100644 --- a/crates/examples/coordinator.rs +++ b/crates/examples/coordinator.rs @@ -22,6 +22,7 @@ use libp2p::{multiaddr::Protocol, Multiaddr}; use parking_lot::RwLock; use warp::Filter; +/// The coordinator service, used to assign unique indices to nodes when running benchmarks #[derive(Parser)] struct Args { /// The address to bind to @@ -45,11 +46,11 @@ async fn main() -> Result<()> { // Create a shared counter let counter = Arc::new(AtomicU32::new(0)); - let counter = warp::any().map(move || counter.clone()); + let counter = warp::any().map(move || Arc::clone(&counter)); // Create a shared set of multiaddrs for Libp2p let libp2p_multiaddrs = Arc::new(RwLock::new(HashSet::new())); - let libp2p_multiaddrs = warp::any().map(move || libp2p_multiaddrs.clone()); + let libp2p_multiaddrs = warp::any().map(move || Arc::clone(&libp2p_multiaddrs)); // `/index` returns the node index we are assigned let index = warp::path!("index") @@ -100,7 +101,7 @@ async fn main() -> Result<()> { // Convert the multiaddrs to a string, separated by newlines multiaddrs .iter() - .map(|m| m.to_string()) + .map(ToString::to_string) .collect::>() .join("\n") }); @@ -118,14 +119,9 @@ async fn main() -> Result<()> { ); // Run the server - warp::serve( - index - .or(reset) - .or(submit_libp2p_info) - .or(get_libp2p_info), - ) - .run(bind_address) - .await; + warp::serve(index.or(reset).or(submit_libp2p_info).or(get_libp2p_info)) + .run(bind_address) + .await; Ok(()) } From 3add972fe926086234399973d00eef9495a9d2f6 Mon Sep 17 00:00:00 2001 From: Rob Date: Wed, 11 Dec 2024 12:50:23 -0500 Subject: [PATCH 13/16] add example for single validator --- Cargo.lock | 1 + crates/examples/Cargo.toml | 8 +- crates/examples/all.rs | 397 ++++++--------------------- crates/examples/common.rs | 243 ++++++++++++++++ crates/examples/coordinator.rs | 4 +- crates/examples/process-compose.yaml | 86 ++++++ crates/examples/single-validator.rs | 358 ++++++++++++++++++++++++ 7 files changed, 774 insertions(+), 323 deletions(-) create mode 100644 crates/examples/process-compose.yaml create mode 100644 crates/examples/single-validator.rs diff --git a/Cargo.lock b/Cargo.lock index 595dca94d9..9dc1b29903 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2934,6 +2934,7 @@ dependencies = [ "hotshot-types", "libp2p", "libp2p-networking", + "local-ip-address", "lru 0.12.5", "parking_lot", "portpicker", diff --git a/crates/examples/Cargo.toml b/crates/examples/Cargo.toml index e75b714e99..28f99f70af 100644 --- a/crates/examples/Cargo.toml +++ b/crates/examples/Cargo.toml @@ -31,6 +31,8 @@ warp = { version = "0.3", default-features = false } reqwest.workspace = true bytes = "1" parking_lot.workspace = true +local-ip-address = "0.6" + [[example]] name = "all" path = "all.rs" @@ -47,9 +49,9 @@ path = "cdn/marshal.rs" name = "coordinator" path = "coordinator.rs" -# [[example]] -# name = "single-validator" -# path = "single-validator.rs" +[[example]] +name = "single-validator" +path = "single-validator.rs" [lints] workspace = true diff --git a/crates/examples/all.rs b/crates/examples/all.rs index 787cf7ee7b..9a9f62b966 100644 --- a/crates/examples/all.rs +++ b/crates/examples/all.rs @@ -12,8 +12,8 @@ use hotshot::{ traits::{ election::static_committee::StaticCommittee, implementations::{ - derive_libp2p_keypair, CdnMetricsValue, CdnTopic, CombinedNetworks, KeyPair, - Libp2pMetricsValue, Libp2pNetwork, PushCdnNetwork, TestingDef, WrappedSignatureKey, + derive_libp2p_keypair, CdnMetricsValue, CdnTopic, KeyPair, Libp2pMetricsValue, + Libp2pNetwork, PushCdnNetwork, TestingDef, WrappedSignatureKey, }, }, types::{BLSPrivKey, BLSPubKey, EventType, SignatureKey}, @@ -30,7 +30,6 @@ use hotshot_example_types::{ use hotshot_testing::block_builder::{SimpleBuilderImplementation, TestBuilderImplementation}; use hotshot_types::{ consensus::ConsensusMetricsValue, - light_client::StateKeyPair, traits::{election::Membership, node_implementation::NodeType}, HotShotConfig, PeerConfig, }; @@ -42,10 +41,13 @@ use libp2p_networking::network::{ }; use lru::LruCache; use rand::Rng; -use tokio::{spawn, sync::OnceCell, task::JoinSet}; -use tracing::{error, info}; +use tokio::{spawn, sync::OnceCell}; +use tracing::info; use url::Url; +// Include some common code +include!("common.rs"); + /// This example runs all necessary HotShot components #[derive(Parser)] struct Args { @@ -61,11 +63,11 @@ struct Args { #[arg(long)] num_views: Option, - /// The number of transactions to submit to the builder per view - #[arg(long, default_value_t = 100)] + /// The number of transactions to submit to each nodes' builder per view + #[arg(long, default_value_t = 1)] num_transactions_per_view: usize, - /// The size of the transactions submitted to the builder per view + /// The size of the transactions submitted to each nodes' builder per view #[arg(long, default_value_t = 1000)] transaction_size: usize, @@ -75,23 +77,6 @@ struct Args { network: String, } -/// This is a testing function which allows us to easily determine if a node should be a DA node -fn is_da_node(index: usize, num_da_nodes: usize) -> bool { - index < num_da_nodes -} - -/// This is a testing function which allows us to easily generate peer configs from indexes -fn peer_info_from_index(index: usize) -> PeerConfig { - // Get the node's public key - let (public_key, _) = BLSPubKey::generated_from_seed_indexed([0u8; 32], index as u64); - - // Generate the peer config - PeerConfig { - stake_table_entry: public_key.stake_table_entry(1), - state_ver_key: StateKeyPair::default().0.ver_key(), - } -} - /// Generate a Libp2p multiaddress from a port fn libp2p_multiaddress_from_index(index: usize) -> Result { // Generate the peer's private key and derive their libp2p keypair @@ -109,19 +94,6 @@ fn libp2p_multiaddress_from_index(index: usize) -> Result { .with_context(|| "Failed to parse multiaddress") } -/// The type of network to use for the example -#[derive(Debug, PartialEq, Eq)] -enum NetworkType { - /// A combined network, which is a combination of a Libp2p and Push CDN network - Combined, - - /// A network solely using the Push CDN - Cdn, - - /// A Libp2p network - LibP2P, -} - /// A helper function to start the CDN /// (2 brokers + 1 marshal) /// @@ -198,236 +170,17 @@ async fn start_cdn() -> Result { Ok(marshal_address) } -/// A helper function to create a Libp2p network -async fn create_libp2p_network( - node_index: usize, - total_num_nodes: usize, - public_key: &BLSPubKey, - private_key: &BLSPrivKey, - known_libp2p_nodes: &[Multiaddr], -) -> Result>> { - // Derive the Libp2p keypair from the private key - let libp2p_keypair = derive_libp2p_keypair::(private_key) - .with_context(|| "Failed to derive libp2p keypair")?; - - // Sign our Libp2p lookup record value - let lookup_record_value = RecordValue::new_signed( - &RecordKey::new(Namespace::Lookup, public_key.to_bytes()), - libp2p_keypair.public().to_peer_id().to_bytes(), - private_key, - ) - .expect("Failed to sign DHT lookup record"); - - // Configure Libp2p - let libp2p_config = Libp2pConfig { - keypair: libp2p_keypair, - bind_address: known_libp2p_nodes[node_index].clone(), - known_peers: known_libp2p_nodes.to_vec(), - quorum_membership: None, // This disables stake-table authentication - auth_message: None, // This disables stake-table authentication - gossip_config: GossipConfig::default(), - request_response_config: RequestResponseConfig::default(), - kademlia_config: KademliaConfig { - replication_factor: total_num_nodes * 2 / 3, - record_ttl: None, - publication_interval: None, - file_path: format!("/tmp/kademlia-{}.db", rand::random::()), - lookup_record_value, - }, - }; - - // Create the network with the config - Ok(Arc::new( - Libp2pNetwork::new( - libp2p_config, - public_key, - Libp2pMetricsValue::default(), - None, - ) - .await - .with_context(|| "Failed to create libp2p network")?, - )) -} - -/// Create a Push CDN network, starting the CDN if it's not already running -async fn create_cdn_network( - is_da_node: bool, - public_key: &BLSPubKey, - private_key: &BLSPrivKey, -) -> Result>> { +/// Start the CDN if it's not already running +/// Returns the address of the marshal +async fn try_start_cdn() -> Result { /// Create a static cell to store the marshal's endpoint - static MARSHAL_ENDPOINT: OnceCell = OnceCell::const_new(); + static MARSHAL_ADDRESS: OnceCell = OnceCell::const_new(); // If the marshal endpoint isn't already set, start the CDN and set the endpoint - let marshal_endpoint = MARSHAL_ENDPOINT + Ok(MARSHAL_ADDRESS .get_or_init(|| async { start_cdn().await.expect("Failed to start CDN") }) .await - .clone(); - - // Subscribe to topics based on whether we're a DA node or not - let mut topics = vec![CdnTopic::Global]; - if is_da_node { - topics.push(CdnTopic::Da); - } - - // Create and return the network - Ok(Arc::new( - PushCdnNetwork::new( - marshal_endpoint, - topics, - KeyPair { - public_key: WrappedSignatureKey(*public_key), - private_key: private_key.clone(), - }, - CdnMetricsValue::default(), - ) - .with_context(|| "Failed to create Push CDN network")?, - )) -} - -/// A helper function to create a Combined network, which is a combination of a Libp2p and Push CDN network -async fn create_combined_network( - node_index: usize, - total_num_nodes: usize, - known_libp2p_nodes: &[Multiaddr], - is_da_node: bool, - public_key: &BLSPubKey, - private_key: &BLSPrivKey, -) -> Result>> { - // Create the CDN network - let cdn_network = - Arc::into_inner(create_cdn_network(is_da_node, public_key, private_key).await?).unwrap(); - - // Create the Libp2p network - let libp2p_network = Arc::into_inner( - create_libp2p_network( - node_index, - total_num_nodes, - public_key, - private_key, - known_libp2p_nodes, - ) - .await?, - ) - .unwrap(); - - // Create and return the combined network - Ok(Arc::new(CombinedNetworks::new( - cdn_network, - libp2p_network, - Some(Duration::from_secs(1)), - ))) -} - -/// A macro to start a node with a given network. We need this because `NodeImplementation` requires we -/// define a bunch of traits, which we can't do in a generic context. -macro_rules! start_node_with_network { - ($network_type:ident, $node_index:expr, $total_num_nodes:expr, $public_key:expr, $private_key:expr, $config:expr, $memberships:expr, $hotshot_initializer:expr, $network:expr, $join_set:expr, $num_transactions_per_view:expr, $transaction_size:expr, $num_views:expr, $builder_url:expr) => { - // Create the marketplace config - let marketplace_config = MarketplaceConfig::<_, $network_type> { - auction_results_provider: TestAuctionResultsProvider::::default().into(), - // TODO: we need to pass a valid fallback builder url here somehow - fallback_builder_url: Url::parse("http://localhost:8080").unwrap(), - }; - - // Initialize the system context - let handle = SystemContext::::init( - $public_key.clone(), - $private_key.clone(), - $config.clone(), - $memberships.clone(), - $network, - $hotshot_initializer, - ConsensusMetricsValue::default(), - TestStorage::::default(), - marketplace_config, - ) - .await - .with_context(|| "Failed to initialize system context")? - .0; - - // If the node index is 0, create and start the builder, which sips off events from HotShot - if $node_index == 0 { - // Create it - let builder_handle = - >::start( - $total_num_nodes, - $builder_url.clone(), - (), - HashMap::new(), - ) - .await; - - // Start it - builder_handle.start(Box::new(handle.event_stream())); - } - - // Start consensus - handle.hotshot.start_consensus().await; - - // Create an LRU cache to store the size of the proposed block if we're DA - let mut proposed_block_size_cache = LruCache::new(NonZero::new(100).unwrap()); - - // Spawn the task into the join set - $join_set.spawn(async move { - // Get the event stream for this particular node - let mut event_stream = handle.event_stream(); - - // Wait for a `Decide` event for the view number we requested - loop { - // Get the next event - let event = event_stream.next().await.unwrap(); - - // DA proposals contain the full list of transactions. We can use this to cache - // the size of the proposed block - if let EventType::DaProposal { proposal, .. } = event.event { - // Insert the size of the proposed block into the cache - proposed_block_size_cache.put( - *proposal.data.view_number, - proposal.data.encoded_transactions.len(), - ); - - // A `Decide` event contains data that HotShot has decided on - } else if let EventType::Decide { qc, .. } = event.event { - // If we cached the size of the proposed block, log it - if let Some(size) = proposed_block_size_cache.get(&*qc.view_number) { - info!( - block_size = size, - "Decided on view {}", *qc.view_number - ); - } else { - info!("Decided on view {}", *qc.view_number); - } - - // If the view number is divisible by the node's index, submit transactions - if $node_index == 0 { - // Generate and submit the requested number of transactions - for _ in 0..$num_transactions_per_view { - // Generate a random transaction - let mut transaction_bytes = vec![0u8; $transaction_size]; - rand::thread_rng().fill(&mut transaction_bytes[..]); - - // Submit the transaction - if let Err(err) = handle - .submit_transaction(TestTransaction::new(transaction_bytes)) - .await - { - error!("Failed to submit transaction: {:?}", err); - }; - } - } - - // If we have a specific view number we want to wait for, check if we've reached it - if let Some(num_views) = $num_views { - if *qc.view_number == num_views as u64 { - // Break when we've decided on the view number we requested - break; - } - } - } - } - }); - }; + .clone()) } #[tokio::main] @@ -483,8 +236,8 @@ async fn main() -> Result<()> { // Create the memberships from the known nodes and known da nodes let memberships = StaticCommittee::new(known_nodes.clone(), known_da_nodes.clone()); - // Create a `JoinSet` composed of all handles - let mut join_set = JoinSet::new(); + // Create a set composed of all handles + let mut join_set = Vec::new(); // Spawn each node for index in 0..args.total_num_nodes { @@ -525,9 +278,13 @@ async fn main() -> Result<()> { // Create a network and start HotShot based on the network type match network_type { NetworkType::Combined => { + // Start the CDN if it's not already running + let marshal_address = try_start_cdn().await?; + // Create the combined network - let network = create_combined_network( - index, + let network = new_combined_network( + Some(marshal_address), + known_libp2p_nodes[index].clone(), args.total_num_nodes, &known_libp2p_nodes, is_da_node(index, args.num_da_nodes), @@ -538,56 +295,60 @@ async fn main() -> Result<()> { .with_context(|| "Failed to create Combined network")?; // Start the node - start_node_with_network!( - CombinedImpl, - index, - args.total_num_nodes, - &public_key, - &private_key, - &config, - memberships, - hotshot_initializer, - network, - join_set, - args.num_transactions_per_view, - args.transaction_size, - args.num_views, - builder_url + join_set.push( + start_consensus::( + public_key, + private_key, + config, + memberships.clone(), + network, + hotshot_initializer, + args.total_num_nodes, + builder_url.clone(), + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?, ); } NetworkType::Cdn => { + // Start the CDN if it's not already running + let marshal_address = try_start_cdn().await?; + // Create the CDN network - let network = create_cdn_network( + let network = new_cdn_network( + Some(marshal_address), is_da_node(index, args.num_da_nodes), &public_key, &private_key, ) - .await .with_context(|| "Failed to create CDN network")?; // Start the node - start_node_with_network!( - PushCdnImpl, - index, - args.total_num_nodes, - &public_key, - &private_key, - &config, - memberships, - hotshot_initializer, - network, - join_set, - args.num_transactions_per_view, - args.transaction_size, - args.num_views, - builder_url + join_set.push( + start_consensus::( + public_key, + private_key, + config, + memberships.clone(), + network, + hotshot_initializer, + args.total_num_nodes, + builder_url.clone(), + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?, ); } NetworkType::LibP2P => { // Create the Libp2p network - let network = create_libp2p_network( - index, + let network = new_libp2p_network( + // Advertise == bind address here + known_libp2p_nodes[index].clone(), args.total_num_nodes, &public_key, &private_key, @@ -597,29 +358,29 @@ async fn main() -> Result<()> { .with_context(|| "Failed to create Libp2p network")?; // Start the node - start_node_with_network!( - Libp2pImpl, - index, - args.total_num_nodes, - &public_key, - &private_key, - &config, - memberships, - hotshot_initializer, - network, - join_set, - args.num_transactions_per_view, - args.transaction_size, - args.num_views, - builder_url + join_set.push( + start_consensus::( + public_key, + private_key, + config, + memberships.clone(), + network, + hotshot_initializer, + args.total_num_nodes, + builder_url.clone(), + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?, ); } }; } // Wait for all the tasks to finish - while let Some(res) = join_set.join_next().await { - res.expect("Failed to join task"); + while let Some(res) = join_set.pop() { + res.await.expect("Failed to join task"); } Ok(()) diff --git a/crates/examples/common.rs b/crates/examples/common.rs index dc8cdbdd33..3b87287a65 100644 --- a/crates/examples/common.rs +++ b/crates/examples/common.rs @@ -30,3 +30,246 @@ fn peer_info_from_index(index: usize) -> hotshot_types::PeerConfig, + is_da_node: bool, + public_key: &BLSPubKey, + private_key: &BLSPrivKey, +) -> Result>> { + // If the marshal endpoint is not provided, we don't need to create a CDN network + let Some(marshal_address) = marshal_address else { + anyhow::bail!("Marshal endpoint is required for CDN networks"); + }; + + // Subscribe to topics based on whether we're a DA node or not + let mut topics = vec![CdnTopic::Global]; + if is_da_node { + topics.push(CdnTopic::Da); + } + + // Create and return the network + Ok(Arc::new( + PushCdnNetwork::new( + marshal_address, + topics, + KeyPair { + public_key: WrappedSignatureKey(*public_key), + private_key: private_key.clone(), + }, + CdnMetricsValue::default(), + ) + .with_context(|| "Failed to create Push CDN network")?, + )) +} + +/// A helper function to create a Libp2p network +async fn new_libp2p_network( + bind_address: Multiaddr, + total_num_nodes: usize, + public_key: &BLSPubKey, + private_key: &BLSPrivKey, + known_libp2p_nodes: &[Multiaddr], +) -> Result>> { + // Derive the Libp2p keypair from the private key + let libp2p_keypair = derive_libp2p_keypair::(private_key) + .with_context(|| "Failed to derive libp2p keypair")?; + + // Sign our Libp2p lookup record value + let lookup_record_value = RecordValue::new_signed( + &RecordKey::new(Namespace::Lookup, public_key.to_bytes()), + libp2p_keypair.public().to_peer_id().to_bytes(), + private_key, + ) + .expect("Failed to sign DHT lookup record"); + + // Configure Libp2p + let libp2p_config = Libp2pConfig { + keypair: libp2p_keypair, + bind_address, + known_peers: known_libp2p_nodes.to_vec(), + quorum_membership: None, // This disables stake-table authentication + auth_message: None, // This disables stake-table authentication + gossip_config: GossipConfig::default(), + request_response_config: RequestResponseConfig::default(), + kademlia_config: KademliaConfig { + replication_factor: total_num_nodes * 2 / 3, + record_ttl: None, + publication_interval: None, + file_path: format!("/tmp/kademlia-{}.db", rand::random::()), + lookup_record_value, + }, + }; + + // Create the network with the config + Ok(Arc::new( + Libp2pNetwork::new( + libp2p_config, + public_key, + Libp2pMetricsValue::default(), + None, + ) + .await + .with_context(|| "Failed to create libp2p network")?, + )) +} + +/// A helper function to create a Combined network, which is a combination of a Libp2p and Push CDN network +async fn new_combined_network( + marshal_address: Option, + libp2p_bind_address: Multiaddr, + total_num_nodes: usize, + known_libp2p_nodes: &[Multiaddr], + is_da_node: bool, + public_key: &BLSPubKey, + private_key: &BLSPrivKey, +) -> Result>> { + // Create the CDN network and launch the CDN + let cdn_network = Arc::into_inner( + new_cdn_network(marshal_address, is_da_node, public_key, private_key)?, + ) + .unwrap(); + + // Create the Libp2p network + let libp2p_network = Arc::into_inner( + new_libp2p_network( + libp2p_bind_address, + total_num_nodes, + public_key, + private_key, + known_libp2p_nodes, + ) + .await?, + ) + .unwrap(); + + // Create and return the combined network + Ok(Arc::new( + hotshot::traits::implementations::CombinedNetworks::new( + cdn_network, + libp2p_network, + Some(Duration::from_secs(1)), + ), + )) +} + +#[allow(clippy::too_many_arguments)] +/// A helper function to start consensus with a builder +async fn start_consensus< + I: hotshot::traits::NodeImplementation< + TestTypes, + Storage = TestStorage, + AuctionResultsProvider = TestAuctionResultsProvider, + >, +>( + public_key: BLSPubKey, + private_key: BLSPrivKey, + config: HotShotConfig, + memberships: hotshot_example_types::node_types::StaticMembership, + network: Arc, + hotshot_initializer: hotshot::HotShotInitializer, + total_num_nodes: usize, + builder_url: Url, + num_transactions_per_view: usize, + transaction_size: usize, + num_views: Option, +) -> Result> { + // Create the marketplace config + let marketplace_config: MarketplaceConfig = MarketplaceConfig { + auction_results_provider: TestAuctionResultsProvider::::default().into(), + // TODO: we need to pass a valid fallback builder url here somehow + fallback_builder_url: Url::parse("http://localhost:8080").unwrap(), + }; + + // Initialize the system context + let handle = SystemContext::::init( + public_key, + private_key, + config, + memberships, + network, + hotshot_initializer, + ConsensusMetricsValue::default(), + TestStorage::::default(), + marketplace_config, + ) + .await + .with_context(|| "Failed to initialize system context")? + .0; + + // Each node has to start the builder since we don't have a sovereign builder in this example + let builder_handle = + >::start( + total_num_nodes, + builder_url.clone(), + (), + HashMap::new(), + ) + .await; + + // Start it + builder_handle.start(Box::new(handle.event_stream())); + + // Start consensus + handle.hotshot.start_consensus().await; + + // Create an LRU cache to store the size of the proposed block if we're DA + let mut proposed_block_size_cache = LruCache::new(NonZero::new(100).unwrap()); + + // Spawn the task to wait for events + let join_handle = tokio::spawn(async move { + // Get the event stream for this particular node + let mut event_stream = handle.event_stream(); + + // Wait for a `Decide` event for the view number we requested + loop { + // Get the next event + let event = event_stream.next().await.unwrap(); + + // DA proposals contain the full list of transactions. We can use this to cache + // the size of the proposed block + if let EventType::DaProposal { proposal, .. } = event.event { + // Insert the size of the proposed block into the cache + proposed_block_size_cache.put( + *proposal.data.view_number, + proposal.data.encoded_transactions.len(), + ); + + // A `Decide` event contains data that HotShot has decided on + } else if let EventType::Decide { qc, .. } = event.event { + // If we cached the size of the proposed block, log it + if let Some(size) = proposed_block_size_cache.get(&*qc.view_number) { + info!(block_size = size, "Decided on view {}", *qc.view_number); + } else { + info!("Decided on view {}", *qc.view_number); + } + + // Generate and submit the requested number of transactions + for _ in 0..num_transactions_per_view { + // Generate a random transaction + let mut transaction_bytes = vec![0u8; transaction_size]; + rand::thread_rng().fill(&mut transaction_bytes[..]); + + // Submit the transaction + if let Err(err) = handle + .submit_transaction(TestTransaction::new(transaction_bytes)) + .await + { + tracing::error!("Failed to submit transaction: {:?}", err); + }; + } + + // If we have a specific view number we want to wait for, check if we've reached it + if let Some(num_views) = num_views { + if *qc.view_number == num_views as u64 { + // Break when we've decided on the view number we requested + break; + } + } + } + } + }); + + Ok(join_handle) +} diff --git a/crates/examples/coordinator.rs b/crates/examples/coordinator.rs index bdff1669d8..dd041832f4 100644 --- a/crates/examples/coordinator.rs +++ b/crates/examples/coordinator.rs @@ -70,12 +70,12 @@ async fn main() -> Result<()> { }; // Attempt to parse the string as a Libp2p Multiaddr - let Ok(mut multiaddr) = Multiaddr::from_str(&string) else { + let Ok(multiaddr) = Multiaddr::from_str(&string) else { return "Failed to parse body as Multiaddr".to_string(); }; // Pop off the last protocol - let Some(last_protocol) = multiaddr.pop() else { + let Some(last_protocol) = multiaddr.clone().pop() else { return "Failed to get last protocol of multiaddr".to_string(); }; diff --git a/crates/examples/process-compose.yaml b/crates/examples/process-compose.yaml new file mode 100644 index 0000000000..1df6494df1 --- /dev/null +++ b/crates/examples/process-compose.yaml @@ -0,0 +1,86 @@ +# This file is used to run all components of the multi-process examples (in combined network mode). +# If you want to just run them all in one process, you can use `cargo run --example all`. + +# To run this, do `process-compose up` + +# NOTE: You will need Docker installed to run this, as we use KeyDB as a database for the CDN for now + +version: "3" + +processes: + # The coordinator is used to assign unique indices to nodes when running benchmarks on multiple machines + # (or in this case, multiple processes). It is also used to share Libp2p addresses with other nodes + # so they can bootstrap to each other. + coordinator: + command: RUST_LOG=info cargo run --example coordinator + readiness_probe: + exec: + command: nc -zv localhost 3030 + period_seconds: 5 + timeout_seconds: 4 + failure_threshold: 20 + + # We use KeyDB (a Redis variant) to maintain consistency between + # different parts of the CDN + # Cheating a bit here too, but KeyDB is not available as a Nix package. + # Could do local (SQLite) discovery, but removes some of the spirit + # from the local demo. + keydb: + command: docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb --requirepass changeme! + readiness_probe: + exec: + command: nc -zv localhost 6379 + period_seconds: 5 + timeout_seconds: 4 + failure_threshold: 20 + + # The marshal is the CDN component that is responsible for authenticating users and + # pointing them to the correct broker. + marshal: + command: RUST_LOG=info cargo run --example cdn-marshal -- --discovery-endpoint "redis://:changeme!@localhost:6379" + depends_on: + keydb: + condition: process_healthy + + # The `broker` is the CDN component that is primarily responsible for routing messages + # between nodes. + broker: + command: RUST_LOG=info,libp2p=off cargo run --example cdn-broker -- --discovery-endpoint "redis://:changeme!@localhost:6379" + depends_on: + keydb: + condition: process_healthy + + # `hotshot1` is a single HotShot node + hotshot1: + command: RUST_LOG=info,libp2p=off cargo run --example single-validator -- --network combined --libp2p-port 3000 --total-num-nodes 5 --num-da-nodes 3 --marshal-address localhost:1737 + depends_on: + coordinator: + condition: process_healthy + + # `hotshot2` is a second HotShot node + hotshot2: + command: RUST_LOG=info,libp2p=off cargo run --example single-validator -- --network combined --libp2p-port 3001 --total-num-nodes 5 --num-da-nodes 3 --marshal-address localhost:1737 + depends_on: + coordinator: + condition: process_healthy + + # `hotshot3` is a third HotShot node + hotshot3: + command: RUST_LOG=info,libp2p=off cargo run --example single-validator -- --network combined --libp2p-port 3002 --total-num-nodes 5 --num-da-nodes 3 --marshal-address localhost:1737 + depends_on: + coordinator: + condition: process_healthy + + # `hotshot4` is a fourth HotShot node + hotshot4: + command: RUST_LOG=info,libp2p=off cargo run --example single-validator -- --network combined --libp2p-port 3003 --total-num-nodes 5 --num-da-nodes 3 --marshal-address localhost:1737 + depends_on: + coordinator: + condition: process_healthy + + # `hotshot5` is a fifth HotShot node + hotshot5: + command: RUST_LOG=info,libp2p=off cargo run --example single-validator -- --network combined --libp2p-port 3004 --total-num-nodes 5 --num-da-nodes 3 --marshal-address localhost:1737 + depends_on: + coordinator: + condition: process_healthy diff --git a/crates/examples/single-validator.rs b/crates/examples/single-validator.rs new file mode 100644 index 0000000000..10a6e285ba --- /dev/null +++ b/crates/examples/single-validator.rs @@ -0,0 +1,358 @@ +//! This is meant to be run externally, e.g. when running benchmarks on the protocol. +//! If you just want to run everything required, you can use the `all` example +//! +//! This example runs a single validator node along with a simple builder (since the +//! real builder is in an upstream repo) + +use std::{ + collections::HashMap, net::IpAddr, num::NonZero, str::FromStr, sync::Arc, time::Duration, +}; + +use anyhow::{Context, Result}; +use clap::Parser; +use futures::StreamExt; +use hotshot::{ + helpers::initialize_logging, + traits::{ + election::static_committee::StaticCommittee, + implementations::{ + derive_libp2p_keypair, CdnMetricsValue, CdnTopic, KeyPair, Libp2pMetricsValue, + Libp2pNetwork, PushCdnNetwork, WrappedSignatureKey, + }, + }, + types::{BLSPrivKey, BLSPubKey, EventType, SignatureKey}, + MarketplaceConfig, SystemContext, +}; +use hotshot_example_types::{ + auction_results_provider_types::TestAuctionResultsProvider, + block_types::TestTransaction, + node_types::{CombinedImpl, Libp2pImpl, PushCdnImpl, TestTypes, TestVersions}, + state_types::TestInstanceState, + storage_types::TestStorage, + testable_delay::DelayConfig, +}; +use hotshot_testing::block_builder::{SimpleBuilderImplementation, TestBuilderImplementation}; +use hotshot_types::{ + consensus::ConsensusMetricsValue, traits::election::Membership, HotShotConfig, +}; +use libp2p::Multiaddr; +use libp2p_networking::network::{ + behaviours::dht::record::{Namespace, RecordKey, RecordValue}, + node::config::{KademliaConfig, Libp2pConfig}, + GossipConfig, RequestResponseConfig, +}; +use lru::LruCache; +use rand::Rng; +use tracing::info; +use url::Url; + +// Include some common code +include!("common.rs"); + +/// This example runs a single validator node +#[derive(Parser)] +struct Args { + /// The coordinator address to connect to. The coordinator is just used to tell other nodes + /// about the libp2p bootstrap addresses and give each one a unique index + #[arg(long, default_value = "http://127.0.0.1:3030")] + coordinator_address: String, + + /// The marshal's endpoint to use. Only required if the network type is "cdn" or "combined" + #[arg(long)] + marshal_address: Option, + + /// The source of the public IP address to use. This is used to generate the libp2p + /// bootstrap addresses. Acceptable values are "ipify", "local", "localhost", "aws-local", or + /// "aws-public" + #[arg(long, default_value = "localhost")] + ip_source: String, + + /// The port to use for Libp2p + #[arg(long, default_value_t = 3000)] + libp2p_port: u16, + + /// The number of nodes in the network. This needs to be the same between all nodes + #[arg(long, default_value_t = 5)] + total_num_nodes: usize, + + /// The number of nodes which are DA. This needs to be the same between all nodes + #[arg(long, default_value_t = 3)] + num_da_nodes: usize, + + /// The number of views to run for. If not specified, it will run indefinitely + #[arg(long)] + num_views: Option, + + /// The number of transactions to submit to each nodes' builder per view + #[arg(long, default_value_t = 1)] + num_transactions_per_view: usize, + + /// The size of the transactions submitted to each nodes' builder per view + #[arg(long, default_value_t = 1000)] + transaction_size: usize, + + /// The type of network to use. Acceptable values are + /// "combined", "cdn", or "libp2p" + #[arg(long, default_value = "combined")] + network: String, +} + +/// Get the IP address to use based on the source +async fn get_public_ip_address(source: &str) -> Result { + // Get the IP to use based on the source + Ok(match source.to_lowercase().as_str() { + "ipify" => reqwest::get("https://api.ipify.org") + .await? + .text() + .await? + .parse::() + .with_context(|| "Failed to parse IP address from IPify")?, + "local" => { + local_ip_address::local_ip().with_context(|| "Failed to get local IP address")? + } + "localhost" => "127.0.0.1".parse::().unwrap(), + "aws-local" => reqwest::get("http://169.254.169.254/latest/meta-data/local-ipv4") + .await? + .text() + .await? + .parse::() + .with_context(|| "Failed to parse IP address from AWS local")?, + "aws-public" => reqwest::get("http://169.254.169.254/latest/meta-data/public-ipv4") + .await? + .text() + .await? + .parse::() + .with_context(|| "Failed to parse IP address from AWS public")?, + _ => { + anyhow::bail!( + "Invalid public IP source. Please use one of 'ipify', 'local', 'localhost', 'aws-local', or 'aws-public'." + ) + } + }) +} + +#[tokio::main] +#[allow(clippy::too_many_lines)] +async fn main() -> Result<()> { + // Initialize logging + initialize_logging(); + + // Parse the command-line arguments + let args = Args::parse(); + + // Match the network type + let network_type = match args.network.to_lowercase().as_str() { + "combined" => NetworkType::Combined, + "cdn" => NetworkType::Cdn, + "libp2p" => NetworkType::LibP2P, + _ => { + anyhow::bail!("Invalid network type. Please use one of 'combined', 'cdn', or 'libp2p'.") + } + }; + + // Get the IP address to use for Libp2p based on the source + let libp2p_ip = get_public_ip_address(&args.ip_source).await?; + info!("Using Libp2p address: {}:{}", libp2p_ip, args.libp2p_port); + + // Create a new instance state + let instance_state = TestInstanceState::new(DelayConfig::default()); + + // Initialize HotShot from genesis + let hotshot_initializer = + hotshot::HotShotInitializer::::from_genesis::(instance_state) + .await + .with_context(|| "Failed to initialize HotShot")?; + + // Get our index from the coordinator + let index = reqwest::get(format!("{}/index", &args.coordinator_address).as_str()) + .await? + .text() + .await? + .parse::() + .with_context(|| "Failed to parse index from coordinator")?; + + // Derive our keypair from the index we got + let (public_key, private_key) = BLSPubKey::generated_from_seed_indexed([0u8; 32], index as u64); + + // Derive our libp2p keypair from the private key + let peer_libp2p_keypair = derive_libp2p_keypair::(&private_key) + .with_context(|| "Failed to derive libp2p keypair")?; + + // Generate our advertise Multiaddr + let advertise_multiaddr = format!( + "/ip4/{}/udp/{}/quic-v1/p2p/{}", + libp2p_ip, + args.libp2p_port, + peer_libp2p_keypair.public().to_peer_id() + ); + + // Generate our bind Multiaddr + let bind_multiaddr = + Multiaddr::from_str(&format!("/ip4/0.0.0.0/udp/{}/quic-v1", args.libp2p_port)) + .with_context(|| "Failed to parse bind Multiaddr")?; + + // Post our advertise libp2p address to the coordinator + reqwest::Client::new() + .post(format!("{}/libp2p-info", &args.coordinator_address).as_str()) + .body(advertise_multiaddr) + .send() + .await?; + + // Get the other libp2p addresses from the coordinator + let known_libp2p_nodes = + reqwest::get(format!("{}/libp2p-info", &args.coordinator_address).as_str()) + .await? + .text() + .await? + .split('\n') + .map(|s| { + s.parse::() + .with_context(|| "Failed to parse Libp2p bootstrap address") + }) + .collect::>>()?; + + // Print the known libp2p nodes + info!("Known libp2p nodes: {:?}", known_libp2p_nodes); + + // Generate the builder URL we plan to use + let builder_url = Url::parse( + format!( + "http://localhost:{}", + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")? + ) + .as_str(), + ) + .with_context(|| "Failed to parse builder URL")?; + + // Create the known nodes up to the total number of nodes + let known_nodes: Vec<_> = (0..args.total_num_nodes) + .map(peer_info_from_index) + .collect(); + let known_da_nodes: Vec<_> = (0..args.num_da_nodes) + .filter(|i| is_da_node(*i, args.num_da_nodes)) + .map(peer_info_from_index) + .collect(); + + // Create the memberships from the known nodes and known da nodes + let memberships = StaticCommittee::new(known_nodes.clone(), known_da_nodes.clone()); + + // Configure HotShot + let config = HotShotConfig:: { + known_nodes: known_nodes.clone(), + known_da_nodes: known_da_nodes.clone(), + next_view_timeout: 5000, + fixed_leader_for_gpuvid: 0, // This means that we don't have a fixed leader for testing GPU VID + view_sync_timeout: Duration::from_secs(5), + builder_timeout: Duration::from_secs(1), + data_request_delay: Duration::from_millis(200), + builder_urls: vec![builder_url.clone()], + start_proposing_view: u64::MAX, // These just mean the upgrade functionality is disabled + stop_proposing_view: u64::MAX, + start_voting_view: u64::MAX, + stop_voting_view: u64::MAX, + start_proposing_time: u64::MAX, + stop_proposing_time: u64::MAX, + start_voting_time: u64::MAX, + stop_voting_time: u64::MAX, + epoch_height: 0, // This just means epochs aren't enabled + }; + + // Create the network and start consensus + match network_type { + NetworkType::Cdn => { + // Create the network + let network = new_cdn_network( + args.marshal_address, + is_da_node(index, args.num_da_nodes), + &public_key, + &private_key, + ) + .with_context(|| "Failed to create CDN network")?; + + // Start consensus + let join_handle = start_consensus::( + public_key, + private_key, + config, + memberships, + network, + hotshot_initializer, + args.total_num_nodes, + builder_url, + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?; + + // Wait for consensus to finish + join_handle.await?; + } + NetworkType::LibP2P => { + // Create the network + let network = new_libp2p_network( + bind_multiaddr, + args.total_num_nodes, + &public_key, + &private_key, + &known_libp2p_nodes, + ) + .await + .with_context(|| "Failed to create libp2p network")?; + + // Start consensus + let join_handle = start_consensus::( + public_key, + private_key, + config, + memberships, + network, + hotshot_initializer, + args.total_num_nodes, + builder_url, + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?; + + // Wait for consensus to finish + join_handle.await?; + } + NetworkType::Combined => { + // Create the network + let network = new_combined_network( + args.marshal_address, + bind_multiaddr, + args.total_num_nodes, + &known_libp2p_nodes, + is_da_node(index, args.num_da_nodes), + &public_key, + &private_key, + ) + .await + .with_context(|| "Failed to create combined network")?; + + // Start consensus + let join_handle = start_consensus::( + public_key, + private_key, + config, + memberships, + network, + hotshot_initializer, + args.total_num_nodes, + builder_url, + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?; + + // Wait for consensus to finish + join_handle.await?; + } + }; + + Ok(()) +} From 60a437684b8136dd87fd3f081e98d226f6801a23 Mon Sep 17 00:00:00 2001 From: Rob Date: Wed, 11 Dec 2024 15:01:53 -0500 Subject: [PATCH 14/16] throughput/latency calcs --- Cargo.lock | 17 ++++ crates/example-types/src/block_types.rs | 32 ++++++ crates/examples/Cargo.toml | 2 + crates/examples/common.rs | 123 +++++++++++++++++++++--- crates/types/src/utils.rs | 14 ++- 5 files changed, 173 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9dc1b29903..466e7d440a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1185,6 +1185,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bytesize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" + [[package]] name = "capnp" version = "0.20.3" @@ -2924,6 +2930,7 @@ version = "0.5.79" dependencies = [ "anyhow", "bytes", + "bytesize", "cdn-broker", "cdn-marshal", "clap", @@ -2941,6 +2948,7 @@ dependencies = [ "rand 0.8.5", "reqwest", "sha2 0.10.8", + "simple_moving_average", "tokio", "tracing", "url", @@ -6616,6 +6624,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_moving_average" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a4b144ad185430cd033299e2c93e465d5a7e65fbb858593dc57181fa13cd310" +dependencies = [ + "num-traits", +] + [[package]] name = "slab" version = "0.4.9" diff --git a/crates/example-types/src/block_types.rs b/crates/example-types/src/block_types.rs index 3d42344bc2..bf2358117b 100644 --- a/crates/example-types/src/block_types.rs +++ b/crates/example-types/src/block_types.rs @@ -6,10 +6,12 @@ use std::{ fmt::{Debug, Display}, + io::{Cursor, Read}, mem::size_of, sync::Arc, }; +use anyhow::Context; use async_trait::async_trait; use committable::{Commitment, Committable, RawCommitmentBuilder}; use hotshot_types::{ @@ -106,6 +108,36 @@ impl TestTransaction { encoded } + + /// Decode a list of individual transactions from the encoded payload + pub fn decode(encoded_transactions: &[u8]) -> anyhow::Result> { + // Create a cursor to read the encoded transactions + let mut cursor = Cursor::new(encoded_transactions); + + // A collection of the transactions to return + let mut transactions = Vec::new(); + + // Process each transaction + let mut transaction_size_bytes = [0; size_of::()]; + while cursor.position() < encoded_transactions.len() as u64 { + // Read the transaction size + cursor + .read_exact(&mut transaction_size_bytes) + .context("Failed to read transaction size")?; + let transaction_size = u32::from_le_bytes(transaction_size_bytes); + + // Read the transaction + let mut transaction_bytes = vec![0; transaction_size as usize]; + cursor + .read_exact(&mut transaction_bytes) + .context("Failed to read transaction")?; + + // Add the transaction to the collection + transactions.push(Self(transaction_bytes)); + } + + Ok(transactions) + } } impl Committable for TestTransaction { diff --git a/crates/examples/Cargo.toml b/crates/examples/Cargo.toml index 28f99f70af..982112fc40 100644 --- a/crates/examples/Cargo.toml +++ b/crates/examples/Cargo.toml @@ -32,6 +32,8 @@ reqwest.workspace = true bytes = "1" parking_lot.workspace = true local-ip-address = "0.6" +simple_moving_average = "1" +bytesize = "1" [[example]] name = "all" diff --git a/crates/examples/common.rs b/crates/examples/common.rs index 3b87287a65..d2693174d8 100644 --- a/crates/examples/common.rs +++ b/crates/examples/common.rs @@ -1,3 +1,11 @@ +use std::time::Instant; + +use hotshot_types::{ + data::EpochNumber, traits::node_implementation::ConsensusTime, utils::non_crypto_hash, +}; +use simple_moving_average::SingleSumSMA; +use simple_moving_average::SMA; + /// The type of network to use for the example #[derive(Debug, PartialEq, Eq)] enum NetworkType { @@ -126,9 +134,12 @@ async fn new_combined_network( private_key: &BLSPrivKey, ) -> Result>> { // Create the CDN network and launch the CDN - let cdn_network = Arc::into_inner( - new_cdn_network(marshal_address, is_da_node, public_key, private_key)?, - ) + let cdn_network = Arc::into_inner(new_cdn_network( + marshal_address, + is_da_node, + public_key, + private_key, + )?) .unwrap(); // Create the Libp2p network @@ -155,6 +166,10 @@ async fn new_combined_network( } #[allow(clippy::too_many_arguments)] +#[allow(clippy::cast_precision_loss)] +#[allow(clippy::cast_sign_loss)] +#[allow(clippy::too_many_lines)] +#[allow(clippy::cast_possible_truncation)] /// A helper function to start consensus with a builder async fn start_consensus< I: hotshot::traits::NodeImplementation< @@ -187,7 +202,7 @@ async fn start_consensus< public_key, private_key, config, - memberships, + memberships.clone(), network, hotshot_initializer, ConsensusMetricsValue::default(), @@ -214,8 +229,22 @@ async fn start_consensus< // Start consensus handle.hotshot.start_consensus().await; - // Create an LRU cache to store the size of the proposed block if we're DA - let mut proposed_block_size_cache = LruCache::new(NonZero::new(100).unwrap()); + // See if we're a DA node or not + let is_da_node = memberships.has_da_stake(&public_key, EpochNumber::new(0)); + + // Create an LRU cache to store block data if we're DA. We populate this cache when we receive + // the DA proposal for a view and print the data when we actually decide on that view + let mut view_cache = LruCache::new(NonZero::new(100).unwrap()); + + // A cache of outstanding transactions (hashes of). Used to calculate latency. Isn't needed for non-DA nodes + let mut outstanding_transactions: LruCache = + LruCache::new(NonZero::new(10000).unwrap()); + + // The simple moving average, used to calculate throughput + let mut throughput: SingleSumSMA = SingleSumSMA::::new(); + + // The last time we decided on a view (for calculating throughput) + let mut last_decide_time = Instant::now(); // Spawn the task to wait for events let join_handle = tokio::spawn(async move { @@ -230,17 +259,83 @@ async fn start_consensus< // DA proposals contain the full list of transactions. We can use this to cache // the size of the proposed block if let EventType::DaProposal { proposal, .. } = event.event { - // Insert the size of the proposed block into the cache - proposed_block_size_cache.put( + // Decode the transactions. We use this to log the size of the proposed block + // when we decide on a view + let transactions = + match TestTransaction::decode(&proposal.data.encoded_transactions) { + Ok(transactions) => transactions, + Err(err) => { + tracing::error!("Failed to decode transactions: {:?}", err); + continue; + } + }; + + // Get the number of transactions in the proposed block + let num_transactions = transactions.len(); + + // Sum the total number of bytes in the proposed block and cache + // the hash so we can calculate latency + let mut sum = 0; + let mut submitted_times = Vec::new(); + for transaction in transactions { + // Add the size of the transaction to the sum + sum += transaction.bytes().len(); + + // If we can find the transaction in the cache, add the hash of the transaction to the cache + if let Some(&instant) = + outstanding_transactions.get(&non_crypto_hash(transaction.bytes())) + { + submitted_times.push(instant); + } + } + + // Insert the size of the proposed block and the number of transactions into the cache. + // We use this to log the size of the proposed block when we decide on a view + view_cache.put( *proposal.data.view_number, - proposal.data.encoded_transactions.len(), + (sum, num_transactions, submitted_times), ); // A `Decide` event contains data that HotShot has decided on } else if let EventType::Decide { qc, .. } = event.event { // If we cached the size of the proposed block, log it - if let Some(size) = proposed_block_size_cache.get(&*qc.view_number) { - info!(block_size = size, "Decided on view {}", *qc.view_number); + if let Some((block_size, num_transactions, submitted_times)) = + view_cache.get(&*qc.view_number) + { + // Calculate the average latency of the transactions + let mut total_latency = Duration::default(); + let mut num_found_transactions = 0; + for submitted_time in submitted_times { + total_latency += submitted_time.elapsed(); + num_found_transactions += 1; + } + let average_latency = total_latency.checked_div(num_found_transactions); + + // Update the throughput SMA + throughput + .add_sample(*block_size as f64 / last_decide_time.elapsed().as_secs_f64()); + + // Update the last decided time + last_decide_time = Instant::now(); + + // If we have a valid average latency, log it + if let Some(average_latency) = average_latency { + info!( + block_size = block_size, + num_txs = num_transactions, + avg_tx_latency =? average_latency, + avg_throughput = format!("{}/s", bytesize::ByteSize::b(throughput.get_average() as u64)), + "Decided on view {}", + *qc.view_number + ); + } else { + info!( + block_size = block_size, + num_txs = num_transactions, + "Decided on view {}", + *qc.view_number + ); + } } else { info!("Decided on view {}", *qc.view_number); } @@ -251,6 +346,12 @@ async fn start_consensus< let mut transaction_bytes = vec![0u8; transaction_size]; rand::thread_rng().fill(&mut transaction_bytes[..]); + // If we're a DA node, cache the transaction so we can calculate latency + if is_da_node { + outstanding_transactions + .put(non_crypto_hash(&transaction_bytes), Instant::now()); + } + // Submit the transaction if let Err(err) = handle .submit_transaction(TestTransaction::new(transaction_bytes)) diff --git a/crates/types/src/utils.rs b/crates/types/src/utils.rs index c4a4dfb927..36d915d3b0 100644 --- a/crates/types/src/utils.rs +++ b/crates/types/src/utils.rs @@ -7,7 +7,7 @@ //! Utility functions, type aliases, helper structs and enum definitions. use std::{ - hash::{Hash, Hasher}, + hash::{DefaultHasher, Hash, Hasher}, ops::Deref, sync::Arc, }; @@ -230,7 +230,13 @@ pub fn epoch_from_block_number(block_number: u64, epoch_height: u64) -> u64 { /// A function for generating a cute little user mnemonic from a hash #[must_use] pub fn mnemonic(bytes: H) -> String { - let mut state = std::collections::hash_map::DefaultHasher::new(); - bytes.hash(&mut state); - mnemonic::to_string(state.finish().to_le_bytes()) + let hash = non_crypto_hash(bytes); + mnemonic::to_string(hash.to_le_bytes()) +} + +/// A helper function to generate a non-cryptographic hash +pub fn non_crypto_hash(val: H) -> u64 { + let mut state = DefaultHasher::new(); + val.hash(&mut state); + state.finish() } From 635cfacf6238f28bcaf904ff9d738884d4356dcf Mon Sep 17 00:00:00 2001 From: Rob Date: Wed, 11 Dec 2024 16:48:14 -0500 Subject: [PATCH 15/16] build and push examples --- .dockerignore | 16 +++---- .github/workflows/build-and-push-examples.yml | 43 +++++++++++++++++++ .../Dockerfiles/cdn-broker.Dockerfile | 7 +++ .../Dockerfiles/cdn-marshal.Dockerfile | 7 +++ .../Dockerfiles/coordinator.Dockerfile | 7 +++ .../Dockerfiles/single-validator.Dockerfile | 7 +++ 6 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/build-and-push-examples.yml create mode 100644 crates/examples/Dockerfiles/cdn-broker.Dockerfile create mode 100644 crates/examples/Dockerfiles/cdn-marshal.Dockerfile create mode 100644 crates/examples/Dockerfiles/coordinator.Dockerfile create mode 100644 crates/examples/Dockerfiles/single-validator.Dockerfile diff --git a/.dockerignore b/.dockerignore index df05904eab..f10af9ed76 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,12 +1,6 @@ .git -/target/tmp -/target/release* -/target/debug -/target/x86_64-unknown-linux-musl/release*/build -/target/x86_64-unknown-linux-musl/release*/deps -/target/x86_64-unknown-linux-musl/release*/incremental -!/target/x86_64-unknown-linux-musl/release*/examples/ -!/target/release*/examples/ -!/target/release-lto/examples/ -!/target/release*/benchmark_client -Dockerfile* +target/* +!target/release/examples/cdn-broker +!target/release/examples/cdn-marshal +!target/release/examples/coordinator +!target/release/examples/single-validator \ No newline at end of file diff --git a/.github/workflows/build-and-push-examples.yml b/.github/workflows/build-and-push-examples.yml new file mode 100644 index 0000000000..74aa0ffefa --- /dev/null +++ b/.github/workflows/build-and-push-examples.yml @@ -0,0 +1,43 @@ +name: Build and Push Examples + +on: + push: + branches: + - "main" + + +jobs: + build-and-push-examples: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + name: Checkout Repository + + - name: Install Rust + uses: mkroening/rust-toolchain-toml@main + + - uses: Swatinem/rust-cache@v2 + name: Enable Rust Caching + with: + shared-key: "build-release" + cache-on-failure: "true" + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Build examples + run: cargo build --examples --release + + - name: Build dockerfiles + run: | + docker build . -f crates/examples/Dockerfiles/cdn-marshal.Dockerfile -t ghcr.io/espressosystems/hotshot/cdn-marshal:main + docker build . -f crates/examples/Dockerfiles/cdn-broker.Dockerfile -t ghcr.io/espressosystems/hotshot/cdn-broker:main + docker build . -f crates/examples/Dockerfiles/coordinator.Dockerfile -t ghcr.io/espressosystems/hotshot/coordinator:main + docker build . -f crates/examples/Dockerfiles/single-validator.Dockerfile -t ghcr.io/espressosystems/hotshot/single-validator:main + + - name: Push dockerfiles + run: | + docker push ghcr.io/espressosystems/hotshot/cdn-marshal:main + docker push ghcr.io/espressosystems/hotshot/cdn-broker:main + docker push ghcr.io/espressosystems/hotshot/coordinator:main + docker push ghcr.io/espressosystems/hotshot/single-validator:main + + diff --git a/crates/examples/Dockerfiles/cdn-broker.Dockerfile b/crates/examples/Dockerfiles/cdn-broker.Dockerfile new file mode 100644 index 0000000000..a5d03571eb --- /dev/null +++ b/crates/examples/Dockerfiles/cdn-broker.Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3 + +# Copy the source files +COPY ./target/release/examples/cdn-broker /cdn-broker + +# Run the broker +ENTRYPOINT ["/cdn-broker"] diff --git a/crates/examples/Dockerfiles/cdn-marshal.Dockerfile b/crates/examples/Dockerfiles/cdn-marshal.Dockerfile new file mode 100644 index 0000000000..639b1a9238 --- /dev/null +++ b/crates/examples/Dockerfiles/cdn-marshal.Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3 + +# Copy the source files +COPY ./target/release/examples/cdn-marshal /cdn-marshal + +# Run the broker +ENTRYPOINT ["/cdn-marshal"] diff --git a/crates/examples/Dockerfiles/coordinator.Dockerfile b/crates/examples/Dockerfiles/coordinator.Dockerfile new file mode 100644 index 0000000000..65f7f8ed43 --- /dev/null +++ b/crates/examples/Dockerfiles/coordinator.Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3 + +# Copy the source files +COPY ./target/release/examples/coordinator /coordinator + +# Run the broker +ENTRYPOINT ["/coordinator"] diff --git a/crates/examples/Dockerfiles/single-validator.Dockerfile b/crates/examples/Dockerfiles/single-validator.Dockerfile new file mode 100644 index 0000000000..55c0d253f8 --- /dev/null +++ b/crates/examples/Dockerfiles/single-validator.Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3 + +# Copy the source files +COPY ./target/release/examples/single-validator /single-validator + +# Run the broker +ENTRYPOINT ["/single-validator"] From 9ee6eda74c1c54357f1ec8bdabe0b6ceb7d440e3 Mon Sep 17 00:00:00 2001 From: Rob Date: Thu, 12 Dec 2024 12:27:36 -0500 Subject: [PATCH 16/16] merge fixes --- crates/hotshot/src/traits/election/static_committee.rs | 3 +-- .../src/traits/election/static_committee_leader_two_views.rs | 4 +--- crates/task-impls/src/quorum_proposal/mod.rs | 3 +-- crates/task-impls/src/quorum_vote/mod.rs | 4 +--- crates/testing/tests/tests_1/test_success.rs | 1 - crates/types/src/traits/election.rs | 5 +---- 6 files changed, 5 insertions(+), 15 deletions(-) diff --git a/crates/hotshot/src/traits/election/static_committee.rs b/crates/hotshot/src/traits/election/static_committee.rs index e423294d81..d2b62f80b7 100644 --- a/crates/hotshot/src/traits/election/static_committee.rs +++ b/crates/hotshot/src/traits/election/static_committee.rs @@ -15,10 +15,9 @@ use hotshot_types::{ PeerConfig, }; use primitive_types::U256; -use serde::{Deserialize, Serialize}; use utils::anytrace::Result; -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Hash)] /// The static committee election pub struct StaticCommittee { /// The nodes eligible for leadership. diff --git a/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs b/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs index eda8c4a263..cd0b826694 100644 --- a/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs +++ b/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs @@ -15,11 +15,9 @@ use hotshot_types::{ PeerConfig, }; use primitive_types::U256; -use serde::{Deserialize, Serialize}; use utils::anytrace::Result; -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] - +#[derive(Clone, Debug, Eq, PartialEq, Hash)] /// The static committee election pub struct StaticCommitteeLeaderForTwoViews { /// The nodes eligible for leadership. diff --git a/crates/task-impls/src/quorum_proposal/mod.rs b/crates/task-impls/src/quorum_proposal/mod.rs index 6c1cc5561a..0ba16ed46f 100644 --- a/crates/task-impls/src/quorum_proposal/mod.rs +++ b/crates/task-impls/src/quorum_proposal/mod.rs @@ -98,7 +98,6 @@ impl, V: Versions> view_number: TYPES::View, event_receiver: Receiver>>, ) -> EventDependency>> { - let id = self.id; EventDependency::new( event_receiver, Box::new(move |event| { @@ -160,7 +159,7 @@ impl, V: Versions> let valid = event_view == view_number; if valid { tracing::debug!( - "Dependency {dependency_type:?} is complete for view {event_view:?}, my id is {id:?}!", + "Dependency {dependency_type:?} is complete for view {event_view:?}", ); } valid diff --git a/crates/task-impls/src/quorum_vote/mod.rs b/crates/task-impls/src/quorum_vote/mod.rs index adb8f80f4a..8cf4698d14 100644 --- a/crates/task-impls/src/quorum_vote/mod.rs +++ b/crates/task-impls/src/quorum_vote/mod.rs @@ -306,7 +306,6 @@ impl, V: Versions> QuorumVoteTaskS view_number: TYPES::View, event_receiver: Receiver>>, ) -> EventDependency>> { - let id = self.id; EventDependency::new( event_receiver.clone(), Box::new(move |event| { @@ -336,10 +335,9 @@ impl, V: Versions> QuorumVoteTaskS }; if event_view == view_number { tracing::trace!( - "Vote dependency {:?} completed for view {:?}, my id is {:?}", + "Vote dependency {:?} completed for view {:?}", dependency_type, view_number, - id, ); return true; } diff --git a/crates/testing/tests/tests_1/test_success.rs b/crates/testing/tests/tests_1/test_success.rs index f380b95f26..97244b1946 100644 --- a/crates/testing/tests/tests_1/test_success.rs +++ b/crates/testing/tests/tests_1/test_success.rs @@ -169,7 +169,6 @@ cross_tests!( epoch_height: 10, num_nodes_with_stake: 10, start_nodes: 10, - num_bootstrap_nodes: 10, da_staked_committee_size: 10, overall_safety_properties: OverallSafetyPropertiesDescription { // Explicitly show that we use normal threshold, i.e. 2 nodes_len / 3 + 1 diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index a01fa444f2..5b72ea4f84 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -7,16 +7,13 @@ //! The election trait, used to decide which node is the leader and determine if a vote is valid. use std::{collections::BTreeSet, fmt::Debug, num::NonZeroU64}; -use serde::{Deserialize, Serialize}; use utils::anytrace::Result; use super::node_implementation::NodeType; use crate::{traits::signature_key::SignatureKey, PeerConfig}; /// A protocol for determining membership in and participating in a committee. -pub trait Membership: - Clone + Debug + Send + Sync + Serialize + for<'de> Deserialize<'de> -{ +pub trait Membership: Clone + Debug + Send + Sync { /// The error type returned by methods like `lookup_leader`. type Error: std::fmt::Display;