diff --git a/.dockerignore b/.dockerignore index df05904eab..f10af9ed76 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,12 +1,6 @@ .git -/target/tmp -/target/release* -/target/debug -/target/x86_64-unknown-linux-musl/release*/build -/target/x86_64-unknown-linux-musl/release*/deps -/target/x86_64-unknown-linux-musl/release*/incremental -!/target/x86_64-unknown-linux-musl/release*/examples/ -!/target/release*/examples/ -!/target/release-lto/examples/ -!/target/release*/benchmark_client -Dockerfile* +target/* +!target/release/examples/cdn-broker +!target/release/examples/cdn-marshal +!target/release/examples/coordinator +!target/release/examples/single-validator \ No newline at end of file diff --git a/.github/workflows/build-and-push-examples.yml b/.github/workflows/build-and-push-examples.yml new file mode 100644 index 0000000000..74aa0ffefa --- /dev/null +++ b/.github/workflows/build-and-push-examples.yml @@ -0,0 +1,43 @@ +name: Build and Push Examples + +on: + push: + branches: + - "main" + + +jobs: + build-and-push-examples: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + name: Checkout Repository + + - name: Install Rust + uses: mkroening/rust-toolchain-toml@main + + - uses: Swatinem/rust-cache@v2 + name: Enable Rust Caching + with: + shared-key: "build-release" + cache-on-failure: "true" + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Build examples + run: cargo build --examples --release + + - name: Build dockerfiles + run: | + docker build . -f crates/examples/Dockerfiles/cdn-marshal.Dockerfile -t ghcr.io/espressosystems/hotshot/cdn-marshal:main + docker build . -f crates/examples/Dockerfiles/cdn-broker.Dockerfile -t ghcr.io/espressosystems/hotshot/cdn-broker:main + docker build . -f crates/examples/Dockerfiles/coordinator.Dockerfile -t ghcr.io/espressosystems/hotshot/coordinator:main + docker build . -f crates/examples/Dockerfiles/single-validator.Dockerfile -t ghcr.io/espressosystems/hotshot/single-validator:main + + - name: Push dockerfiles + run: | + docker push ghcr.io/espressosystems/hotshot/cdn-marshal:main + docker push ghcr.io/espressosystems/hotshot/cdn-broker:main + docker push ghcr.io/espressosystems/hotshot/coordinator:main + docker push ghcr.io/espressosystems/hotshot/single-validator:main + + diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index d8836d871d..cc8924233e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -52,239 +52,4 @@ jobs: env: RUST_BACKTRACE: full - test-examples: - strategy: - fail-fast: false - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - name: Checkout Repository - - - name: Install Rust - uses: mkroening/rust-toolchain-toml@main - - - uses: Swatinem/rust-cache@v2 - name: Enable Rust Caching - with: - shared-key: "examples" - cache-on-failure: "true" - save-if: ${{ github.ref == 'refs/heads/main' }} - - - uses: taiki-e/install-action@just - - - name: Test examples - run: | - just example all-push-cdn -- --config_file ./crates/orchestrator/run-config.toml - timeout-minutes: 20 - - build-release: - strategy: - fail-fast: false - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - name: Checkout Repository - - - name: Install Rust - uses: mkroening/rust-toolchain-toml@main - - - uses: Swatinem/rust-cache@v2 - name: Enable Rust Caching - with: - shared-key: "build-release" - cache-on-failure: "true" - save-if: ${{ github.ref == 'refs/heads/main' }} - - - uses: taiki-e/install-action@just - - - name: Build examples in release mode - run: just build_release --examples --package hotshot-examples --no-default-features - - - name: Upload Binaries - uses: actions/upload-artifact@v4 - with: - name: binaries-amd64 - path: | - target/release/examples/counter - target/release/examples/multi-validator-libp2p - target/release/examples/validator-libp2p - target/release/examples/validator-combined - target/release/examples/validator-push-cdn - target/release/examples/orchestrator - target/release/examples/cdn-broker - target/release/examples/cdn-marshal - - build-arm-release: - strategy: - fail-fast: false - runs-on: buildjet-4vcpu-ubuntu-2204-arm - if: ${{ github.ref == 'refs/heads/main' }} - container: ghcr.io/espressosystems/devops-rust:stable - steps: - - uses: actions/checkout@v4 - name: Checkout Repository - - uses: Swatinem/rust-cache@v2 - name: Enable Rust Caching - with: - shared-key: "build-arm-release" - cache-on-failure: "true" - save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Build examples in release mode - run: just build_release --examples --package hotshot-examples --no-default-features - - - name: Upload Binaries - uses: actions/upload-artifact@v4 - with: - name: binaries-aarch64 - path: | - target/release/examples/counter - target/release/examples/multi-validator-libp2p - target/release/examples/validator-libp2p - target/release/examples/validator-combined - target/release/examples/validator-push-cdn - target/release/examples/orchestrator - target/release/examples/cdn-broker - target/release/examples/cdn-marshal - - build-dockers: - strategy: - fail-fast: false - runs-on: ubuntu-latest - if: ${{ github.ref == 'refs/heads/main' }} - needs: [build-release, build-arm-release, test] - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Setup Docker BuildKit (buildx) - uses: docker/setup-buildx-action@v3 - - - name: Login to Github Container Repo - uses: docker/login-action@v3 - if: github.event_name != 'pull_request' - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Download AMD executables - uses: actions/download-artifact@v4 - with: - name: binaries-amd64 - path: target/amd64/release/examples - - - name: Download ARM executables - uses: actions/download-artifact@v4 - with: - name: binaries-aarch64 - path: target/arm64/release/examples - - - name: Generate validator-libp2p docker metadata - uses: docker/metadata-action@v5 - id: validator-libp2p - with: - images: ghcr.io/espressosystems/hotshot/validator-libp2p - - - name: Generate validator-combined docker metadata - uses: docker/metadata-action@v5 - id: validator-combined - with: - images: ghcr.io/espressosystems/hotshot/validator-combined - - - name: Generate validator-push-cdn docker metadata - uses: docker/metadata-action@v5 - id: validator-push-cdn - with: - images: ghcr.io/espressosystems/hotshot/validator-push-cdn - - - name: Generate orchestrator docker metadata - uses: docker/metadata-action@v5 - id: orchestrator - with: - images: ghcr.io/espressosystems/hotshot/orchestrator - - - name: Generate cdn-broker docker metadata - uses: docker/metadata-action@v5 - id: cdn-broker - with: - images: ghcr.io/espressosystems/hotshot/cdn-broker - - - name: Generate cdn-marshal docker metadata - uses: docker/metadata-action@v5 - id: cdn-marshal - with: - images: ghcr.io/espressosystems/hotshot/cdn-marshal - - - name: Build and push validator-libp2p docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/validator-libp2p.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.validator-libp2p.outputs.tags }} - labels: ${{ steps.validator-libp2p.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push validator-combined docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/validator-combined.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.validator-combined.outputs.tags }} - labels: ${{ steps.validator-combined.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push validator-push-cdn docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/validator-cdn.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.validator-push-cdn.outputs.tags }} - labels: ${{ steps.validator-push-cdn.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push orchestrator docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/orchestrator.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.orchestrator.outputs.tags }} - labels: ${{ steps.orchestrator.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push cdn-broker docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/cdn-broker.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.cdn-broker.outputs.tags }} - labels: ${{ steps.cdn-broker.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push cdn-marshal docker - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./docker/cdn-marshal.Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.cdn-marshal.outputs.tags }} - labels: ${{ steps.cdn-marshal.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/test-sequencer.yml b/.github/workflows/test-sequencer.yml index 2850764bd8..53518911f2 100644 --- a/.github/workflows/test-sequencer.yml +++ b/.github/workflows/test-sequencer.yml @@ -53,7 +53,6 @@ jobs: hotshot-signature-key = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot-signature-key" } hotshot-stake-table = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot-stake-table" } hotshot-state-prover = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot-state-prover" } - hotshot-orchestrator = { path = "${GITHUB_WORKSPACE}/hotshot/crates/orchestrator" } hotshot-web-server = { path = "${GITHUB_WORKSPACE}/hotshot/crates/web_server" } hotshot-task-impls = { path = "${GITHUB_WORKSPACE}/hotshot/crates/task-impls" } hotshot-testing = { path = "${GITHUB_WORKSPACE}/hotshot/crates/testing" } diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 8bcdeb6baf..0000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,45 +0,0 @@ - -## [0.0.4] - 2021-11-01 -### Features -- Downgrade possibly-temporary network faults to warnings -- Improve logging when an invalid transaction is submitted - - - -## [0.0.3] - 2021-10-27 -### Features -- Implement janky catchup - -### BREAKING CHANGE - -Adds new type parameter, corresponding to the state type, to Message - - -## [0.0.2] - 2021-10-19 -### Bug Fixes -- Fix leaders not sending themselves commit votes -- Fix state not getting stored properly - -### Features -- StatefulHandler trait -- Reexport traits from traits module -- State Machine + Node Implementation -- state machine mvp megasquash -- Replace tokio broadcast queue with unbounded equivalent - -### BREAKING CHANGE - -Changes queue type in hotshot methods - - - -## [0.0.1] - 2021-08-20 - - -## 0.0.0 - 2021-07-07 - -[Unreleased]: https://github.com/EspressoSystems/hotshot/compare/0.0.4...HEAD -[0.0.4]: https://github.com/EspressoSystems/hotshot/compare/0.0.3...0.0.4 -[0.0.3]: https://github.com/EspressoSystems/hotshot/compare/0.0.2...0.0.3 -[0.0.2]: https://github.com/EspressoSystems/hotshot/compare/0.0.1...0.0.2 -[0.0.1]: https://github.com/EspressoSystems/hotshot/compare/0.0.0...0.0.1 diff --git a/Cargo.lock b/Cargo.lock index f95a2780c4..208f143dbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -547,10 +547,10 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" dependencies = [ - "event-listener 5.3.1", + "event-listener 5.4.0", "event-listener-strategy", "futures-core", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", ] [[package]] @@ -573,7 +573,7 @@ dependencies = [ "concurrent-queue", "event-listener-strategy", "futures-core", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", ] [[package]] @@ -646,7 +646,7 @@ dependencies = [ "log", "parking", "polling 2.8.0", - "rustix 0.37.27", + "rustix 0.37.28", "slab", "socket2 0.4.10", "waker-fn", @@ -665,7 +665,7 @@ dependencies = [ "futures-lite 2.5.0", "parking", "polling 3.7.4", - "rustix 0.38.42", + "rustix 0.38.43", "slab", "tracing", "windows-sys 0.59.0", @@ -686,9 +686,9 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" dependencies = [ - "event-listener 5.3.1", + "event-listener 5.4.0", "event-listener-strategy", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", ] [[package]] @@ -716,9 +716,9 @@ dependencies = [ "async-task", "blocking", "cfg-if", - "event-listener 5.3.1", + "event-listener 5.4.0", "futures-lite 2.5.0", - "rustix 0.38.42", + "rustix 0.38.43", "tracing", ] @@ -734,7 +734,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 0.38.42", + "rustix 0.38.43", "signal-hook-registry", "slab", "windows-sys 0.59.0", @@ -776,7 +776,7 @@ dependencies = [ "log", "memchr", "once_cell", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "pin-utils", "slab", "wasm-bindgen-futures", @@ -790,7 +790,7 @@ checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", ] [[package]] @@ -846,7 +846,7 @@ dependencies = [ "futures-io", "futures-util", "log", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "tungstenite", ] @@ -860,7 +860,7 @@ dependencies = [ "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", ] [[package]] @@ -919,13 +919,13 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "itoa", "matchit", "memchr", "mime", "percent-encoding", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "rustversion", "serde", "sync_wrapper 0.1.2", @@ -1005,12 +1005,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bimap" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" - [[package]] name = "bincode" version = "1.3.3" @@ -1191,6 +1185,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bytesize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" + [[package]] name = "capnp" version = "0.20.3" @@ -1220,9 +1220,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.3" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" +checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" dependencies = [ "shlex", ] @@ -1300,10 +1300,10 @@ dependencies = [ "prometheus", "quinn", "rand 0.8.5", - "rcgen 0.13.1", + "rcgen 0.13.2", "redis", "rkyv", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", "sqlx", "thiserror 1.0.69", @@ -1432,7 +1432,7 @@ dependencies = [ "bytes", "futures-core", "memchr", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "tokio", "tokio-util", ] @@ -1549,9 +1549,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373e9fafaa20882876db20562275ff58d50e0caa2590077fe7ce7bef90211d0d" +checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" [[package]] name = "constant_time_eq" @@ -1663,18 +1663,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1691,18 +1691,18 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -1741,27 +1741,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "csv" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" -dependencies = [ - "memchr", -] - [[package]] name = "ctr" version = "0.6.0" @@ -2197,7 +2176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2219,13 +2198,13 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "5.3.1" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" dependencies = [ "concurrent-queue", "parking", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", ] [[package]] @@ -2234,8 +2213,8 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" dependencies = [ - "event-listener 5.3.1", - "pin-project-lite 0.2.15", + "event-listener 5.4.0", + "pin-project-lite 0.2.16", ] [[package]] @@ -2300,9 +2279,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" [[package]] name = "foreign-types" @@ -2415,7 +2394,7 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "waker-fn", ] @@ -2429,7 +2408,7 @@ dependencies = [ "futures-core", "futures-io", "parking", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", ] [[package]] @@ -2450,7 +2429,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", ] @@ -2496,7 +2475,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "pin-utils", "slab", ] @@ -2554,9 +2533,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "gloo-timers" @@ -2658,11 +2637,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.2", ] [[package]] @@ -2845,11 +2824,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2871,7 +2850,6 @@ dependencies = [ "async-broadcast", "async-lock 3.4.0", "async-trait", - "bimap", "bincode", "blake3", "cdn-broker", @@ -2895,7 +2873,6 @@ dependencies = [ "primitive-types", "rand 0.8.5", "serde", - "sha2 0.10.8", "time 0.3.37", "tokio", "tracing", @@ -2917,7 +2894,7 @@ dependencies = [ "hotshot-types", "serde", "tagged-base64", - "thiserror 2.0.9", + "thiserror 2.0.10", "tide-disco", "toml", "vbs", @@ -2940,7 +2917,7 @@ dependencies = [ "serde", "sha2 0.10.8", "sha3", - "thiserror 2.0.9", + "thiserror 2.0.10", "time 0.3.37", "tokio", "url", @@ -2953,30 +2930,30 @@ version = "0.5.79" dependencies = [ "anyhow", "async-lock 3.4.0", - "async-trait", + "bytes", + "bytesize", "cdn-broker", "cdn-marshal", - "chrono", "clap", "futures", "hotshot", "hotshot-example-types", - "hotshot-orchestrator", "hotshot-testing", "hotshot-types", + "libp2p", "libp2p-networking", "local-ip-address", + "lru 0.12.5", + "parking_lot", "portpicker", "rand 0.8.5", - "serde", + "reqwest", "sha2 0.10.8", - "surf-disco", - "time 0.3.37", + "simple_moving_average", "tokio", - "toml", "tracing", - "tracing-subscriber 0.3.19", "url", + "warp", ] [[package]] @@ -3006,28 +2983,6 @@ dependencies = [ "syn 2.0.95", ] -[[package]] -name = "hotshot-orchestrator" -version = "0.5.79" -dependencies = [ - "anyhow", - "async-lock 3.4.0", - "blake3", - "clap", - "csv", - "futures", - "hotshot-types", - "libp2p-identity", - "multiaddr", - "serde", - "surf-disco", - "tide-disco", - "tokio", - "toml", - "tracing", - "vbs", -] - [[package]] name = "hotshot-stake-table" version = "0.5.79" @@ -3084,7 +3039,7 @@ dependencies = [ "sha2 0.10.8", "surf-disco", "tagged-base64", - "thiserror 2.0.9", + "thiserror 2.0.10", "time 0.3.37", "tokio", "tracing", @@ -3125,7 +3080,7 @@ dependencies = [ "serde", "sha2 0.10.8", "tagged-base64", - "thiserror 2.0.9", + "thiserror 2.0.10", "tide-disco", "tokio", "tracing", @@ -3150,7 +3105,6 @@ dependencies = [ "bincode", "bitvec", "blake3", - "clap", "committable", "derive_more 1.0.0", "digest 0.10.7", @@ -3163,20 +3117,16 @@ dependencies = [ "jf-utils", "jf-vid", "lazy_static", - "libp2p-identity", "memoize", "mnemonic", - "multiaddr", "primitive-types", "rand 0.8.5", "rand_chacha 0.3.1", "serde", - "serde-inline-default", "serde_bytes", - "serde_json", "sha2 0.10.8", "tagged-base64", - "thiserror 2.0.9", + "thiserror 2.0.10", "time 0.3.37", "tokio", "toml", @@ -3218,7 +3168,7 @@ checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http 0.2.12", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", ] [[package]] @@ -3241,7 +3191,7 @@ dependencies = [ "futures-util", "http 1.2.0", "http-body 1.0.1", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", ] [[package]] @@ -3269,7 +3219,7 @@ dependencies = [ "cookie", "futures-lite 1.13.0", "infer", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "rand 0.7.3", "serde", "serde_json", @@ -3298,9 +3248,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.31" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", @@ -3312,7 +3262,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "socket2 0.5.8", "tokio", "tower-service", @@ -3322,9 +3272,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -3334,7 +3284,7 @@ dependencies = [ "http-body 1.0.1", "httparse", "itoa", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "smallvec", "tokio", "want", @@ -3342,15 +3292,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http 1.2.0", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", "tokio", "tokio-rustls", @@ -3363,8 +3313,8 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.31", - "pin-project-lite 0.2.15", + "hyper 0.14.32", + "pin-project-lite 0.2.16", "tokio", "tokio-io-timeout", ] @@ -3377,7 +3327,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "native-tls", "tokio", @@ -3396,8 +3346,8 @@ dependencies = [ "futures-util", "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.1", - "pin-project-lite 0.2.15", + "hyper 1.5.2", + "pin-project-lite 0.2.16", "socket2 0.5.8", "tokio", "tower-service", @@ -3649,7 +3599,7 @@ dependencies = [ "bytes", "futures", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "log", "rand 0.8.5", "tokio", @@ -4096,9 +4046,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.168" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libloading" @@ -4107,7 +4057,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -4401,8 +4351,6 @@ dependencies = [ "blake3", "cbor4ii", "delegate", - "derive_builder", - "derive_more 1.0.0", "futures", "hotshot-example-types", "hotshot-types", @@ -4435,7 +4383,7 @@ dependencies = [ "quinn", "rand 0.8.5", "ring 0.17.8", - "rustls 0.23.19", + "rustls 0.23.20", "socket2 0.5.8", "thiserror 1.0.69", "tokio", @@ -4529,7 +4477,7 @@ dependencies = [ "libp2p-identity", "rcgen 0.11.3", "ring 0.17.8", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-webpki 0.101.7", "thiserror 1.0.69", "x509-parser", @@ -4635,9 +4583,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" @@ -4833,9 +4781,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -5146,9 +5094,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -5339,7 +5287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.9", + "thiserror 2.0.10", "ucd-trie", ] @@ -5405,9 +5353,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -5471,7 +5419,7 @@ dependencies = [ "concurrent-queue", "libc", "log", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "windows-sys 0.48.0", ] @@ -5484,8 +5432,8 @@ dependencies = [ "cfg-if", "concurrent-queue", "hermit-abi 0.4.0", - "pin-project-lite 0.2.15", - "rustix 0.38.42", + "pin-project-lite 0.2.16", + "rustix 0.38.43", "tracing", "windows-sys 0.59.0", ] @@ -5527,9 +5475,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.25" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +checksum = "483f8c21f64f3ea09fe0f30f5d48c3e8eefe5dac9129f0075f76593b4c1da705" dependencies = [ "proc-macro2", "syn 2.0.95", @@ -5726,13 +5674,13 @@ checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", "futures-io", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.19", + "rustls 0.23.20", "socket2 0.5.8", - "thiserror 2.0.9", + "thiserror 2.0.10", "tokio", "tracing", ] @@ -5748,10 +5696,10 @@ dependencies = [ "rand 0.8.5", "ring 0.17.8", "rustc-hash", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", "slab", - "thiserror 2.0.9", + "thiserror 2.0.10", "tinyvec", "tracing", "web-time", @@ -5759,16 +5707,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2 0.5.8", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5891,9 +5839,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54077e1872c46788540de1ea3d7f4ccb1983d12f9aa909b234468676c1a36779" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ "pem", "ring 0.17.8", @@ -5920,7 +5868,7 @@ dependencies = [ "itoa", "num-bigint", "percent-encoding", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "ryu", "tokio", "tokio-util", @@ -5929,9 +5877,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags 2.6.0", ] @@ -6015,7 +5963,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-rustls", "hyper-tls", "hyper-util", @@ -6026,7 +5974,7 @@ dependencies = [ "native-tls", "once_cell", "percent-encoding", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "rustls-pemfile", "serde", "serde_json", @@ -6222,7 +6170,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.23", + "semver 1.0.24", ] [[package]] @@ -6236,9 +6184,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.27" +version = "0.37.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +checksum = "519165d378b97752ca44bbe15047d5d3409e875f39327546b42ac81d7e18c1b6" dependencies = [ "bitflags 1.3.2", "errno", @@ -6250,15 +6198,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.42" +version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ "bitflags 2.6.0", "errno", "libc", - "linux-raw-sys 0.4.14", - "windows-sys 0.52.0", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", ] [[package]] @@ -6276,9 +6224,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.19" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "log", "once_cell", @@ -6300,9 +6248,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time", ] @@ -6330,9 +6278,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "rw-stream-sink" @@ -6403,9 +6351,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -6422,9 +6370,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" [[package]] name = "semver-parser" @@ -6441,17 +6389,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-inline-default" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fb1bedd774187d304179493b0d3c41fbe97b04b14305363f68d2bdf5e47cb9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.95", -] - [[package]] name = "serde_bytes" version = "0.11.15" @@ -6527,9 +6464,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" dependencies = [ "base64 0.22.1", "chrono", @@ -6545,9 +6482,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" dependencies = [ "darling", "proc-macro2", @@ -6699,6 +6636,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_moving_average" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a4b144ad185430cd033299e2c93e465d5a7e65fbb858593dc57181fa13cd310" +dependencies = [ + "num-traits", +] + [[package]] name = "slab" version = "0.4.9" @@ -6803,21 +6749,11 @@ dependencies = [ "der", ] -[[package]] -name = "sqlformat" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" -dependencies = [ - "nom", - "unicode_categories", -] - [[package]] name = "sqlx" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93334716a037193fac19df402f8571269c84a00852f6a7066b5d2616dcd64d3e" +checksum = "4410e73b3c0d8442c5f99b425d7a435b5ee0ae4167b3196771dd3f7a01be745f" dependencies = [ "sqlx-core", "sqlx-macros", @@ -6828,37 +6764,31 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d8060b456358185f7d50c55d9b5066ad956956fddec42ee2e8567134a8936e" +checksum = "6a007b6936676aa9ab40207cde35daab0a04b823be8ae004368c0793b96a61e0" dependencies = [ - "atoi", - "byteorder", "bytes", "crc", "crossbeam-queue", "either", - "event-listener 5.3.1", - "futures-channel", + "event-listener 5.4.0", "futures-core", "futures-intrusive", "futures-io", "futures-util", - "hashbrown 0.14.5", - "hashlink 0.9.1", - "hex", + "hashbrown 0.15.2", + "hashlink 0.10.0", "indexmap 2.7.0", "log", "memchr", "once_cell", - "paste", "percent-encoding", "serde", "serde_json", "sha2 0.10.8", "smallvec", - "sqlformat", - "thiserror 1.0.69", + "thiserror 2.0.10", "time 0.3.37", "tokio", "tokio-stream", @@ -6868,9 +6798,9 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac0692bcc9de3b073e8d747391827297e075c7710ff6276d9f7a1f3d58c6657" +checksum = "3112e2ad78643fef903618d78cf0aec1cb3134b019730edb039b69eaf531f310" dependencies = [ "proc-macro2", "quote", @@ -6881,9 +6811,9 @@ dependencies = [ [[package]] name = "sqlx-macros-core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1804e8a7c7865599c9c79be146dc8a9fd8cc86935fa641d3ea58e5f0688abaa5" +checksum = "4e9f90acc5ab146a99bf5061a7eb4976b573f560bc898ef3bf8435448dd5e7ad" dependencies = [ "dotenvy", "either", @@ -6907,9 +6837,9 @@ dependencies = [ [[package]] name = "sqlx-mysql" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64bb4714269afa44aef2755150a0fc19d756fb580a67db8885608cf02f47d06a" +checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233" dependencies = [ "atoi", "base64 0.22.1", @@ -6942,7 +6872,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 1.0.69", + "thiserror 2.0.10", "time 0.3.37", "tracing", "whoami", @@ -6950,9 +6880,9 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa91a732d854c5d7726349bb4bb879bb9478993ceb764247660aee25f67c2f8" +checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613" dependencies = [ "atoi", "base64 0.22.1", @@ -6963,7 +6893,6 @@ dependencies = [ "etcetera", "futures-channel", "futures-core", - "futures-io", "futures-util", "hex", "hkdf 0.12.4", @@ -6981,7 +6910,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 1.0.69", + "thiserror 2.0.10", "time 0.3.37", "tracing", "whoami", @@ -6989,9 +6918,9 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5b2cf34a45953bfd3daaf3db0f7a7878ab9b7a6b91b422d24a7a9e4c857b680" +checksum = "f85ca71d3a5b24e64e1d08dd8fe36c6c95c339a896cc33068148906784620540" dependencies = [ "atoi", "flume", @@ -7323,15 +7252,16 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand 2.3.0", + "getrandom 0.2.15", "once_cell", - "rustix 0.38.42", - "windows-sys 0.52.0", + "rustix 0.38.43", + "windows-sys 0.59.0", ] [[package]] @@ -7345,11 +7275,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.9" +version = "2.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" +checksum = "a3ac7f54ca534db81081ef1c1e7f6ea8a3ef428d2fc069097c079443d24124d3" dependencies = [ - "thiserror-impl 2.0.9", + "thiserror-impl 2.0.10", ] [[package]] @@ -7365,9 +7295,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.9" +version = "2.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" +checksum = "9e9465d30713b56a37ede7185763c3492a91be2f5fa68d958c44e41ab9248beb" dependencies = [ "proc-macro2", "quote", @@ -7408,7 +7338,7 @@ dependencies = [ "http-types", "kv-log-macro", "log", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "route-recognizer", "serde", "serde_json", @@ -7446,7 +7376,7 @@ dependencies = [ "rand 0.8.5", "reqwest", "routefinder", - "semver 1.0.23", + "semver 1.0.24", "serde", "serde_json", "serde_with", @@ -7577,9 +7507,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -7592,16 +7522,16 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", "libc", "mio", "parking_lot", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "socket2 0.5.8", "tokio-macros", "tracing", @@ -7614,15 +7544,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "tokio", ] [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", @@ -7645,7 +7575,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.19", + "rustls 0.23.20", "tokio", ] @@ -7656,7 +7586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "tokio", ] @@ -7669,7 +7599,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "tokio", ] @@ -7721,7 +7651,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-timeout", "percent-encoding", "pin-project", @@ -7744,7 +7674,7 @@ dependencies = [ "futures-util", "indexmap 1.9.3", "pin-project", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "rand 0.8.5", "slab", "tokio", @@ -7762,7 +7692,7 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "sync_wrapper 1.0.2", "tokio", "tower-layer", @@ -7788,7 +7718,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", - "pin-project-lite 0.2.15", + "pin-project-lite 0.2.16", "tracing-attributes", "tracing-core", ] @@ -7946,15 +7876,15 @@ dependencies = [ [[package]] name = "unicase" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-bidi" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" @@ -7989,12 +7919,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - [[package]] name = "universal-hash" version = "0.4.0" @@ -8039,7 +7963,7 @@ dependencies = [ "flate2", "log", "once_cell", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", "url", "webpki-roots 0.26.7", @@ -8201,7 +8125,7 @@ dependencies = [ "futures-util", "headers", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "log", "mime", "mime_guess", @@ -8608,9 +8532,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" dependencies = [ "memchr", ] @@ -8666,9 +8590,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432" +checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4" [[package]] name = "xmltree" diff --git a/Cargo.toml b/Cargo.toml index 2ea7c703f1..53780e32fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,6 @@ members = [ "crates/hotshot-stake-table", "crates/libp2p-networking", "crates/macros", - "crates/orchestrator", "crates/task", "crates/task-impls", "crates/testing", @@ -90,6 +89,7 @@ clap = { version = "4", features = ["derive", "env"] } url = { version = "2", features = ["serde"] } vec1 = { version = "1", features = ["serde"] } reqwest = { version = "0.12", features = ["json"] } +parking_lot = "0.12" libp2p = { package = "libp2p", version = "0.54", default-features = false, features = [ "macros", @@ -120,7 +120,6 @@ cdn-client = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.5.6 cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.5.6" } cdn-marshal = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.5.6" } cdn-proto = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.5.6" } - ### Profiles ### ### Note: these only apply to example executables or tests built from within this crate. They have diff --git a/README.md b/README.md index c5e37298af..7094739d3a 100644 --- a/README.md +++ b/README.md @@ -1,229 +1,31 @@ ![GitHub Release](https://img.shields.io/github/v/release/EspressoSystems/HotShot) +# HotShot HotShot Emoji -# License -## Copyright -**(c) 2021-2024 Espresso Systems**. -`HotShot` was developed by Espresso Systems. +HotShot is a Byzantine Fault Tolerant (BFT) consensus protocol that builds upon HotStuff 2. It is modified for proof-of-stake settings and features a linear view-synchronization protocol and a data-availability layer. -# HotShot Consensus Module +## Paper +The HotShot protocol is described in the [Espresso Sequencing Network paper](https://eprint.iacr.org/2024/1189.pdf). -HotShot is a BFT consensus protocol based off of HotStuff, with the addition of proof-of-stake and -VRF committee elections. +## Usage and Examples -## Disclaimer - -**DISCLAIMER:** This software is provided "as is" and its security has not been externally audited. Use at your own risk. - -# Usage - -Please see the rustdoc for API documentation, and the examples directory for usage. - -## Dependencies - -### Unix-like - -#### Nix (macos and linux) - -``` -nix develop -``` - -#### Brew (macos) - -``` -brew install cmake protobuf -``` - -#### Apt-get (linux) - -``` -apt-get install cmake protobuf -``` - -### Windows - -#### Chocolatey - -``` -choco install cmake protoc -``` - -#### Scoop - -``` -scoop bucket add extras -scoop install protobuf cmake -``` - -## Building - -Once dependencies have been installed, to build everything: - -```sh -just build -``` - - - -# Static linking - -HotShot supports static linking for its examples: - -```sh -# Nix-shell is optional but recommended -nix develop .#staticShell - -just build -``` - -# Testing - -To test: - -```sh -RUST_LOG=$ERROR_LOG_LEVEL RUST_LOG_FORMAT=$ERROR_LOG_FORMAT just test -``` - -- `RUST_LOG=$ERROR_LOG_LEVEL`: The basic levels of logging include `warn`, `error`, `info`. -- `RUST_LOG_FORMAT=$ERROR_LOG_FORMAT`: The types of logging include `full`, `json`, and `compact`. -- Internally, the inclusion of the `--nocapture` flag indicates whether or not to output logs. -- Internally, we run at `--test-threads=1` because the tests spawn up a lot of file handles, and unix based systems consistently run out of handles. - -To stress test, run the ignored tests prefixed with `test_stress`: -```sh -RUST_LOG=$ERROR_LOG_LEVEL RUST_LOG_FORMAT=$ERROR_LOG_FORMAT just run_test test_stress -``` - -## Careful - -To double-check for UB: - -```bash -nix develop .#correctnessShell -just careful -``` - -## Testing on CI - -To test as if running on CI, one must limit the number of cores and ram to match github runners (2 core, 7 gig ram). To limit the ram, spin up a virtual machine or container with 7 gigs ram. To limit the core count when running tests: - -``` -ASYNC_STD_THREAD_COUNT=1 RUST_LOG=$ERROR_LOG_LEVEL RUST_LOG_FORMAT=$ERROR_LOG_FORMAT just tokio test -``` - -# Tokio-console - -To use tokio-console, drop into the console shell: - -``` -nix develop .#consoleShell -``` - -Then, run an example. - -On a separate terminal, also drop into the console shell and start tokio-console: -``` -nix develop .#consoleShell -c tokio-console -``` - -This second window should now display task usage. - -# Open Telemetry + Jaeger Integration - -To view distributed logs with just the centralized server and one client, first edit the `centralized_server/orchestrator` file to include have a threshold and num_nodes of 1. - -Then open three separate terminals. +Usage examples are provided in the [examples directory](./crates/examples). +### Running the examples +To run a full example network, use the command ```bash -# Terminal 1 -# Start the jaeger instance to view spans -docker run -d -p6831:6831/udp -p6832:6832/udp -p16686:16686 -p14268:14268 jaegertracing/all-in-one:latest - -# Terminal 2 -# Start the CDN - -# Terminal 3 -# Start the client +RUST_LOG=info cargo run --example all ``` -# Resource Usage Statistics - -To generate usage stats: -- build the test suite -- find the executable containing the test of interest -- run profiling tools - -The executable `cargo` uses is shown in the output of `cargo test`. - -For example, to profile `test_stress_dht_many_round`: - +You can see the list of supported command-line arguments by running ```bash -# bring profiling tooling like flamegraph and heaptrack into scope -nix develop .#perfShell - -# show the executable we need run -# and build all test executables (required for subsequent steps) -cargo test --verbose --release --lib --bins --tests --benches --workspace -- --test-threads=1 -# the output cargo test contains the tests path: -# Running `/home/jrestivo/work/crosscross/target/release/deps/counter-880b1ff53ee21dea test_stress --test-threads=1 --ignored` -# running 7 tests -# test test_stress_dht_many_rounds ... ok -# ... - -# a more detailed alternative to flamegraph -# NOTE: only works on linux -heaptrack $(fd -I "counter*" -t x | rg release) --ignored -- test_stress_dht_many_round --nocapture -# palette provides memory statistics, omission will provide cpu cycle stats as colors -# NOTE: must be run as root on macos -flamegraph --palette=mem $(fd -I "counter*" -t x | rg release) --ignored -- test_stress_dht_one_round -# code coveragte statistics -cargo-llvm-cov llvm-cov --test=test_stress_dht_many_round --workspace --all-targets --release --html --output-path lcov.html -``` - -This will output: -- `heaptrack.counter-$HASH` which is viewable by heaptrack. This provides a plethora of useful statistics about memory and cpu cycles. -- `flamegraph.svg` which is a (moderately) less detailed version of heaptrack. -- `lcov.html` generates a summary of code coverage. - -# Debugging - -A debugging config file is provided for vscode and vscodium in [`.vscode/launch.json`](https://github.com/EspressoSystems/HotShot/blob/main/.vscode/launch.json). This is intended to be used with [vadimcn/vscode-lldb](https://open-vsx.org/extension/vadimcn/vscode-lldb) but may work with other rust debuggers as well. - -To bring `lldb` into scope with nix, run `nix develop .#debugShell`. - -# Git Workflow - -For espresso developers we have written up a description of our workflow [here](./WORKFLOW.md). - -# Extra Editor Configuration - -Choose an async runtime to use before launching a text editor. This may be done by setting the environment RUSTFLAGS. For example: - -``` -nvim # launch text editor of choice. We choose neovim in this example -unset RUSTFLAGS # Unset rustflags so we may continue to use the justfile. The justfile sets these particular config options -``` - -# Debugging - -We support the [CodeLLDB Debugger](https://github.com/vadimcn/vscode-lldb). - -## Neovim - -Install [`dap`](https://github.com/mfussenegger/nvim-dap) and [`rust-tools`](https://github.com/simrat39/rust-tools.nvim). Install the CodeLLDB debugger listed above. -Follow the instructions [here](https://github.com/mfussenegger/nvim-dap/discussions/671#discussioncomment-4286738) to configure the adapter. To add our project-local configurations, run: - -``` -lua require('dap.ext.vscode').load_launchjs(nil, { ["codelldb"] = {"rust"} }) +cargo run --example all -- --help ``` -Finally, place a breakpoint and run `:DapContinue` to begin debugging. +## Audits +The HotShot protocol has been internally audited. The report is available [here](./audits/internal-reviews/EspressoHotshot-2024internal.pdf). -NOTE: Do NOT configure dap at all with rust-tools. Do it manually. - -[Example configuration](https://github.com/DieracDelta/vimconfig/blob/master/modules/lsp.nix#L280). +## Disclaimer -## Vscode +**DISCLAIMER:** This software is provided "as is" and its security has not been externally audited. Use at your own risk. -Install the extension and load the `launch.json` file. Then run the desired test target. diff --git a/config/ValidatorConfigExample b/config/ValidatorConfigExample deleted file mode 100644 index 3170b75f9f..0000000000 --- a/config/ValidatorConfigExample +++ /dev/null @@ -1,32 +0,0 @@ -ValidatorConfig { - public_key: VerKey( - ( - QuadExtField(2264797523581107490935262917175769123227923636811928330606075281145117212394 + 15807017392833049888165434456991157794698032464874424842715555348468160607934 * u), - QuadExtField(7996517616082121122160563552650547601395271017260499735456299700133762512689 + 7504045709281061282278228438613345070383424761478787301859187055302953740948 * u), - QuadExtField(1515973040548822760825076242090160370742046237881440422068330135941139244581 + 20251846261653098602911417004145145971080304248810966341160788194007704966108 * u) - ) - ), - private_key: SignKey( - BigInt( - [3505488234151006356, 6655477166151225138, 3291219027844407676, 2153641080015542578] - ) - ), - stake_value: 1, - state_key_pair: StateKeyPair( - KeyPair { - sk: SignKey( - BigInt( - [2822822805887490846, 6664316196088353173, 4926510007447087464, 116097479308258694] - ) - ), - vk: VerKey( - Projective { - x: BigInt([11315198235793138814, 4744451806709910489, 6921831025042192557, 1125393823825936625]), - y: BigInt([13035879815613524256, 18225673961538637854, 12006860967936477969, 1516668567229692859]), - t: BigInt([13450777528397789701, 12242009376162249168, 12596256366242272750, 3368076418495976469]), - z: BigInt([10465708325245823445, 13967918689717629445, 14943426723808572731, 621075342718756551]) - } - ) - } - ) - } \ No newline at end of file diff --git a/config/ValidatorConfigFile.toml b/config/ValidatorConfigFile.toml deleted file mode 100644 index 1f96cf0eb2..0000000000 --- a/config/ValidatorConfigFile.toml +++ /dev/null @@ -1,3 +0,0 @@ -is_da = true -seed = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -node_id = 0 diff --git a/crates/example-types/src/block_types.rs b/crates/example-types/src/block_types.rs index 3d42344bc2..bf2358117b 100644 --- a/crates/example-types/src/block_types.rs +++ b/crates/example-types/src/block_types.rs @@ -6,10 +6,12 @@ use std::{ fmt::{Debug, Display}, + io::{Cursor, Read}, mem::size_of, sync::Arc, }; +use anyhow::Context; use async_trait::async_trait; use committable::{Commitment, Committable, RawCommitmentBuilder}; use hotshot_types::{ @@ -106,6 +108,36 @@ impl TestTransaction { encoded } + + /// Decode a list of individual transactions from the encoded payload + pub fn decode(encoded_transactions: &[u8]) -> anyhow::Result> { + // Create a cursor to read the encoded transactions + let mut cursor = Cursor::new(encoded_transactions); + + // A collection of the transactions to return + let mut transactions = Vec::new(); + + // Process each transaction + let mut transaction_size_bytes = [0; size_of::()]; + while cursor.position() < encoded_transactions.len() as u64 { + // Read the transaction size + cursor + .read_exact(&mut transaction_size_bytes) + .context("Failed to read transaction size")?; + let transaction_size = u32::from_le_bytes(transaction_size_bytes); + + // Read the transaction + let mut transaction_bytes = vec![0; transaction_size as usize]; + cursor + .read_exact(&mut transaction_bytes) + .context("Failed to read transaction")?; + + // Add the transaction to the collection + transactions.push(Self(transaction_bytes)); + } + + Ok(transactions) + } } impl Committable for TestTransaction { diff --git a/crates/examples/Cargo.toml b/crates/examples/Cargo.toml index 3136acdae4..6cfe139e31 100644 --- a/crates/examples/Cargo.toml +++ b/crates/examples/Cargo.toml @@ -1,114 +1,60 @@ [package] -authors = { workspace = true } -description = "HotShot Examples and binaries" -edition = { workspace = true } name = "hotshot-examples" -readme = "README.md" -version = { workspace = true } -rust-version = { workspace = true } +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true -[features] -default = ["docs", "doc-images", "hotshot-testing"] -gpu-vid = ["hotshot-example-types/gpu-vid"] - -# Build the extended documentation -docs = [] -doc-images = [] -hotshot-testing = ["hotshot/hotshot-testing"] -fixed-leader-election = [] - -# Common -[[example]] -name = "orchestrator" -path = "orchestrator.rs" - -# Libp2p -[[example]] -name = "validator-libp2p" -path = "libp2p/validator.rs" - -[[example]] -name = "multi-validator-libp2p" -path = "libp2p/multi-validator.rs" - -[[example]] -name = "all-libp2p" -path = "libp2p/all.rs" - -# Combined -[[example]] -name = "all-combined" -path = "combined/all.rs" - -[[example]] -name = "multi-validator-combined" -path = "combined/multi-validator.rs" - -[[example]] -name = "validator-combined" -path = "combined/validator.rs" - -[[example]] -name = "orchestrator-combined" -path = "combined/orchestrator.rs" - -# Push CDN -[[example]] -name = "all-push-cdn" -path = "push-cdn/all.rs" - -[[example]] -name = "validator-push-cdn" -path = "push-cdn/validator.rs" +[dependencies] +async-lock.workspace = true +tokio.workspace = true +clap.workspace = true +hotshot = { path = "../hotshot" } +hotshot-types = { path = "../types" } +hotshot-example-types.path = "../example-types" +url.workspace = true +libp2p.workspace = true +portpicker.workspace = true +libp2p-networking.workspace = true +anyhow.workspace = true +rand.workspace = true +futures.workspace = true +tracing.workspace = true +hotshot-testing.path = "../testing" +lru.workspace = true +cdn-broker.workspace = true +cdn-marshal.workspace = true +sha2.workspace = true +warp = { version = "0.3", default-features = false } +reqwest.workspace = true +bytes = "1" +parking_lot.workspace = true +local-ip-address = "0.6" +simple_moving_average = "1" +bytesize = "1" [[example]] -name = "multi-validator-push-cdn" -path = "push-cdn/multi-validator.rs" +name = "all" +path = "all.rs" [[example]] name = "cdn-broker" -path = "push-cdn/broker.rs" +path = "cdn/broker.rs" [[example]] name = "cdn-marshal" -path = "push-cdn/marshal.rs" +path = "cdn/marshal.rs" [[example]] -name = "whitelist-push-cdn" -path = "push-cdn/whitelist-adapter.rs" +name = "coordinator" +path = "coordinator.rs" -[dependencies] -async-lock = { workspace = true } -async-trait = { workspace = true } - -cdn-broker = { workspace = true, features = ["global-permits"] } -cdn-marshal = { workspace = true } -chrono = { workspace = true } -clap = { workspace = true, optional = true } -futures = { workspace = true } -hotshot = { path = "../hotshot" } -hotshot-example-types = { path = "../example-types" } -hotshot-orchestrator = { version = "0.5.36", path = "../orchestrator", default-features = false } -hotshot-testing = { path = "../testing" } -hotshot-types = { path = "../types" } -libp2p-networking = { workspace = true } -local-ip-address = "0.6" -portpicker = { workspace = true } -rand = { workspace = true } -serde = { workspace = true, features = ["rc"] } -sha2 = { workspace = true } -surf-disco = { workspace = true } -time = { workspace = true } -tokio = { workspace = true } - -tracing = { workspace = true } -url = { workspace = true } - -[dev-dependencies] -anyhow = { workspace = true } -clap = { workspace = true } -toml = { workspace = true } -tracing-subscriber = "0.3" +[[example]] +name = "single-validator" +path = "single-validator.rs" [lints] workspace = true diff --git a/crates/examples/Dockerfiles/cdn-broker.Dockerfile b/crates/examples/Dockerfiles/cdn-broker.Dockerfile new file mode 100644 index 0000000000..a5d03571eb --- /dev/null +++ b/crates/examples/Dockerfiles/cdn-broker.Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3 + +# Copy the source files +COPY ./target/release/examples/cdn-broker /cdn-broker + +# Run the broker +ENTRYPOINT ["/cdn-broker"] diff --git a/crates/examples/Dockerfiles/cdn-marshal.Dockerfile b/crates/examples/Dockerfiles/cdn-marshal.Dockerfile new file mode 100644 index 0000000000..639b1a9238 --- /dev/null +++ b/crates/examples/Dockerfiles/cdn-marshal.Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3 + +# Copy the source files +COPY ./target/release/examples/cdn-marshal /cdn-marshal + +# Run the broker +ENTRYPOINT ["/cdn-marshal"] diff --git a/crates/examples/Dockerfiles/coordinator.Dockerfile b/crates/examples/Dockerfiles/coordinator.Dockerfile new file mode 100644 index 0000000000..65f7f8ed43 --- /dev/null +++ b/crates/examples/Dockerfiles/coordinator.Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3 + +# Copy the source files +COPY ./target/release/examples/coordinator /coordinator + +# Run the broker +ENTRYPOINT ["/coordinator"] diff --git a/crates/examples/Dockerfiles/single-validator.Dockerfile b/crates/examples/Dockerfiles/single-validator.Dockerfile new file mode 100644 index 0000000000..55c0d253f8 --- /dev/null +++ b/crates/examples/Dockerfiles/single-validator.Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3 + +# Copy the source files +COPY ./target/release/examples/single-validator /single-validator + +# Run the broker +ENTRYPOINT ["/single-validator"] diff --git a/crates/examples/all.rs b/crates/examples/all.rs new file mode 100644 index 0000000000..9a9f62b966 --- /dev/null +++ b/crates/examples/all.rs @@ -0,0 +1,387 @@ +//! This file contains an example of running a full HotShot network, comprised of +//! a CDN, Libp2p, a builder, and multiple validators. +use std::{collections::HashMap, num::NonZero, sync::Arc, time::Duration}; + +use anyhow::{Context, Result}; +use cdn_broker::{reexports::def::hook::NoMessageHook, Broker, Config as BrokerConfig}; +use cdn_marshal::{Config as MarshalConfig, Marshal}; +use clap::Parser; +use futures::StreamExt; +use hotshot::{ + helpers::initialize_logging, + traits::{ + election::static_committee::StaticCommittee, + implementations::{ + derive_libp2p_keypair, CdnMetricsValue, CdnTopic, KeyPair, Libp2pMetricsValue, + Libp2pNetwork, PushCdnNetwork, TestingDef, WrappedSignatureKey, + }, + }, + types::{BLSPrivKey, BLSPubKey, EventType, SignatureKey}, + MarketplaceConfig, SystemContext, +}; +use hotshot_example_types::{ + auction_results_provider_types::TestAuctionResultsProvider, + block_types::TestTransaction, + node_types::{CombinedImpl, Libp2pImpl, PushCdnImpl, TestTypes, TestVersions}, + state_types::TestInstanceState, + storage_types::TestStorage, + testable_delay::DelayConfig, +}; +use hotshot_testing::block_builder::{SimpleBuilderImplementation, TestBuilderImplementation}; +use hotshot_types::{ + consensus::ConsensusMetricsValue, + traits::{election::Membership, node_implementation::NodeType}, + HotShotConfig, PeerConfig, +}; +use libp2p::Multiaddr; +use libp2p_networking::network::{ + behaviours::dht::record::{Namespace, RecordKey, RecordValue}, + node::config::{KademliaConfig, Libp2pConfig}, + GossipConfig, RequestResponseConfig, +}; +use lru::LruCache; +use rand::Rng; +use tokio::{spawn, sync::OnceCell}; +use tracing::info; +use url::Url; + +// Include some common code +include!("common.rs"); + +/// This example runs all necessary HotShot components +#[derive(Parser)] +struct Args { + /// The number of nodes to start + #[arg(long, default_value_t = 5)] + total_num_nodes: usize, + + /// The number of nodes which are DA nodes + #[arg(long, default_value_t = 3)] + num_da_nodes: usize, + + /// The number of views to run for. If not specified, it will run indefinitely + #[arg(long)] + num_views: Option, + + /// The number of transactions to submit to each nodes' builder per view + #[arg(long, default_value_t = 1)] + num_transactions_per_view: usize, + + /// The size of the transactions submitted to each nodes' builder per view + #[arg(long, default_value_t = 1000)] + transaction_size: usize, + + /// The type of network to use. Acceptable values are + /// "combined", "cdn", or "libp2p" + #[arg(long, default_value = "combined")] + network: String, +} + +/// Generate a Libp2p multiaddress from a port +fn libp2p_multiaddress_from_index(index: usize) -> Result { + // Generate the peer's private key and derive their libp2p keypair + let (_, peer_private_key) = BLSPubKey::generated_from_seed_indexed([0u8; 32], index as u64); + let peer_libp2p_keypair = derive_libp2p_keypair::(&peer_private_key) + .with_context(|| "Failed to derive libp2p keypair")?; + + // Generate the multiaddress from the peer's port and libp2p keypair + format!( + "/ip4/127.0.0.1/udp/{}/quic-v1/p2p/{}", + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")?, + peer_libp2p_keypair.public().to_peer_id() + ) + .parse() + .with_context(|| "Failed to parse multiaddress") +} + +/// A helper function to start the CDN +/// (2 brokers + 1 marshal) +/// +/// Returns the address of the marshal +async fn start_cdn() -> Result { + // Figure out where we're going to spawn the marshal + let marshal_port = + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")?; + let marshal_address = format!("127.0.0.1:{marshal_port}"); + + // Generate a random file path for the SQLite database + let db_path = format!("/tmp/marshal-{}.db", rand::random::()); + + // Configure the marshal + let marshal_config = MarshalConfig { + bind_endpoint: marshal_address.clone(), + discovery_endpoint: db_path.clone(), + ca_cert_path: None, + ca_key_path: None, + metrics_bind_endpoint: None, + global_memory_pool_size: Some(1024 * 1024 * 1024), + }; + + // Create and start the marshal + let marshal: Marshal> = Marshal::new(marshal_config) + .await + .with_context(|| "Failed to create marshal")?; + spawn(marshal.start()); + + // This keypair is shared between brokers + let (broker_public_key, broker_private_key) = + ::SignatureKey::generated_from_seed_indexed([0u8; 32], 1337); + + for _ in 0..2 { + // Generate one random port for the "public" endpoint and one for the "private" endpoint + let public_port = + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")?; + let private_port = + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")?; + + // Throw into address format + let private_address = format!("127.0.0.1:{private_port}"); + let public_address = format!("127.0.0.1:{public_port}"); + + // Configure the broker + let broker_config: BrokerConfig::SignatureKey>> = + BrokerConfig { + public_advertise_endpoint: public_address.clone(), + public_bind_endpoint: public_address, + private_advertise_endpoint: private_address.clone(), + private_bind_endpoint: private_address, + discovery_endpoint: db_path.clone(), + keypair: KeyPair { + public_key: WrappedSignatureKey(broker_public_key), + private_key: broker_private_key.clone(), + }, + + user_message_hook: NoMessageHook, // Don't do any message processing + broker_message_hook: NoMessageHook, + + metrics_bind_endpoint: None, + ca_cert_path: None, + ca_key_path: None, + global_memory_pool_size: Some(1024 * 1024 * 1024), + }; + + // Create and start it + let broker = Broker::new(broker_config) + .await + .with_context(|| "Failed to create broker")?; + spawn(broker.start()); + } + + Ok(marshal_address) +} + +/// Start the CDN if it's not already running +/// Returns the address of the marshal +async fn try_start_cdn() -> Result { + /// Create a static cell to store the marshal's endpoint + static MARSHAL_ADDRESS: OnceCell = OnceCell::const_new(); + + // If the marshal endpoint isn't already set, start the CDN and set the endpoint + Ok(MARSHAL_ADDRESS + .get_or_init(|| async { start_cdn().await.expect("Failed to start CDN") }) + .await + .clone()) +} + +#[tokio::main] +#[allow(clippy::too_many_lines)] +async fn main() -> Result<()> { + // Initialize logging + initialize_logging(); + + // Parse the command line arguments + let args = Args::parse(); + + // Match the network type + let network_type = match args.network.to_lowercase().as_str() { + "combined" => NetworkType::Combined, + "cdn" => NetworkType::Cdn, + "libp2p" => NetworkType::LibP2P, + _ => { + anyhow::bail!("Invalid network type. Please use one of 'combined', 'cdn', or 'libp2p'.") + } + }; + + // Generate the builder URL we plan to use + let builder_url = Url::parse( + format!( + "http://localhost:{}", + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")? + ) + .as_str(), + ) + .with_context(|| "Failed to parse builder URL")?; + + // Create the `known_nodes` and `known_da_nodes` + let known_nodes: Vec> = (0..args.total_num_nodes) + .map(peer_info_from_index) + .collect(); + let known_da_nodes: Vec> = (0..args.num_da_nodes) + .filter(|index| is_da_node(*index, args.num_da_nodes)) + .map(peer_info_from_index) + .collect(); + + // If the network type is "Libp2p" or "Combined", we need to also assign the list of + // Libp2p addresses to be used + let mut known_libp2p_nodes = Vec::new(); + if network_type == NetworkType::LibP2P || network_type == NetworkType::Combined { + for index in 0..args.total_num_nodes { + // Generate a Libp2p multiaddress from a random, unused port + let addr = libp2p_multiaddress_from_index(index) + .with_context(|| "Failed to generate multiaddress")?; + known_libp2p_nodes.push(addr); + } + } + + // Create the memberships from the known nodes and known da nodes + let memberships = StaticCommittee::new(known_nodes.clone(), known_da_nodes.clone()); + + // Create a set composed of all handles + let mut join_set = Vec::new(); + + // Spawn each node + for index in 0..args.total_num_nodes { + // Create a new instance state + let instance_state = TestInstanceState::new(DelayConfig::default()); + + // Initialize HotShot from genesis + let hotshot_initializer = + hotshot::HotShotInitializer::::from_genesis::(instance_state) + .await + .with_context(|| "Failed to initialize HotShot")?; + + // Create our own keypair + let (public_key, private_key) = + BLSPubKey::generated_from_seed_indexed([0u8; 32], index as u64); + + // Configure HotShot + let config = HotShotConfig:: { + known_nodes: known_nodes.clone(), + known_da_nodes: known_da_nodes.clone(), + next_view_timeout: 5000, + fixed_leader_for_gpuvid: 0, // This means that we don't have a fixed leader for testing GPU VID + view_sync_timeout: Duration::from_secs(5), + builder_timeout: Duration::from_secs(1), + data_request_delay: Duration::from_millis(200), + builder_urls: vec![builder_url.clone()], + start_proposing_view: u64::MAX, // These just mean the upgrade functionality is disabled + stop_proposing_view: u64::MAX, + start_voting_view: u64::MAX, + stop_voting_view: u64::MAX, + start_proposing_time: u64::MAX, + stop_proposing_time: u64::MAX, + start_voting_time: u64::MAX, + stop_voting_time: u64::MAX, + epoch_height: 0, // This just means epochs aren't enabled + }; + + // Create a network and start HotShot based on the network type + match network_type { + NetworkType::Combined => { + // Start the CDN if it's not already running + let marshal_address = try_start_cdn().await?; + + // Create the combined network + let network = new_combined_network( + Some(marshal_address), + known_libp2p_nodes[index].clone(), + args.total_num_nodes, + &known_libp2p_nodes, + is_da_node(index, args.num_da_nodes), + &public_key, + &private_key, + ) + .await + .with_context(|| "Failed to create Combined network")?; + + // Start the node + join_set.push( + start_consensus::( + public_key, + private_key, + config, + memberships.clone(), + network, + hotshot_initializer, + args.total_num_nodes, + builder_url.clone(), + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?, + ); + } + NetworkType::Cdn => { + // Start the CDN if it's not already running + let marshal_address = try_start_cdn().await?; + + // Create the CDN network + let network = new_cdn_network( + Some(marshal_address), + is_da_node(index, args.num_da_nodes), + &public_key, + &private_key, + ) + .with_context(|| "Failed to create CDN network")?; + + // Start the node + join_set.push( + start_consensus::( + public_key, + private_key, + config, + memberships.clone(), + network, + hotshot_initializer, + args.total_num_nodes, + builder_url.clone(), + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?, + ); + } + + NetworkType::LibP2P => { + // Create the Libp2p network + let network = new_libp2p_network( + // Advertise == bind address here + known_libp2p_nodes[index].clone(), + args.total_num_nodes, + &public_key, + &private_key, + &known_libp2p_nodes, + ) + .await + .with_context(|| "Failed to create Libp2p network")?; + + // Start the node + join_set.push( + start_consensus::( + public_key, + private_key, + config, + memberships.clone(), + network, + hotshot_initializer, + args.total_num_nodes, + builder_url.clone(), + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?, + ); + } + }; + } + + // Wait for all the tasks to finish + while let Some(res) = join_set.pop() { + res.await.expect("Failed to join task"); + } + + Ok(()) +} diff --git a/crates/examples/push-cdn/broker.rs b/crates/examples/cdn/broker.rs similarity index 61% rename from crates/examples/push-cdn/broker.rs rename to crates/examples/cdn/broker.rs index 8e03999c33..4e4669983c 100644 --- a/crates/examples/push-cdn/broker.rs +++ b/crates/examples/cdn/broker.rs @@ -1,27 +1,26 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. +//! The broker is the message-routing component of the CDN +//! +//! This is meant to be run externally, e.g. when running benchmarks on the protocol. +//! If you just want to run everything required, you can use the `all` example -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! The following is the main `Broker` binary, which just instantiates and runs -//! a `Broker` object. -use anyhow::Result; +use anyhow::{Context, Result}; use cdn_broker::{reexports::def::hook::NoMessageHook, Broker, Config}; use clap::Parser; -use hotshot::traits::implementations::{KeyPair, ProductionDef, WrappedSignatureKey}; +use hotshot::{ + helpers::initialize_logging, + traits::implementations::{KeyPair, ProductionDef, WrappedSignatureKey}, +}; use hotshot_example_types::node_types::TestTypes; -use hotshot_types::traits::{node_implementation::NodeType, signature_key::SignatureKey}; +use hotshot_types::traits::node_implementation::NodeType; +use hotshot_types::traits::signature_key::BuilderSignatureKey; use sha2::Digest; -use tracing_subscriber::EnvFilter; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] /// The main component of the push CDN. struct Args { /// The discovery client endpoint (including scheme) to connect to. - /// With the local discovery feature, this is a file path. - /// With the remote (redis) discovery feature, this is a redis URL (e.g. `redis://127.0.0.1:6789`). + /// This is a URL pointing to a `KeyDB` database (e.g. `redis://127.0.0.1:6789`). #[arg(short, long)] discovery_endpoint: String, @@ -60,31 +59,15 @@ struct Args { /// The seed for broker key generation #[arg(short, long, default_value_t = 0)] key_seed: u64, - - /// The size of the global memory pool (in bytes). This is the maximum number of bytes that - /// can be allocated at once for all connections. A connection will block if it - /// tries to allocate more than this amount until some memory is freed. - /// Default is 1GB. - #[arg(long, default_value_t = 1_073_741_824)] - global_memory_pool_size: usize, } #[tokio::main] async fn main() -> Result<()> { - // Parse command line arguments - let args = Args::parse(); + // Initialize logging + initialize_logging(); - // Initialize tracing - if std::env::var("RUST_LOG_FORMAT") == Ok("json".to_string()) { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .json() - .init(); - } else { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .init(); - } + // Parse the command line arguments + let args = Args::parse(); // Generate the broker key from the supplied seed let key_hash = sha2::Sha256::digest(args.key_seed.to_le_bytes()); @@ -110,15 +93,13 @@ async fn main() -> Result<()> { public_advertise_endpoint: args.public_advertise_endpoint, private_bind_endpoint: args.private_bind_endpoint, private_advertise_endpoint: args.private_advertise_endpoint, - global_memory_pool_size: Some(args.global_memory_pool_size), + // Use a 1GB memory pool size + global_memory_pool_size: Some(1_073_741_824), }; - // Create new `Broker` - // Uses TCP from broker connections and TCP+TLS for user connections. + // Create the new `Broker` let broker = Broker::new(broker_config).await?; - // Start the main loop, consuming it - broker.start().await?; - - Ok(()) + // Run the broker until it is terminated + broker.start().await.with_context(|| "Broker exited") } diff --git a/crates/examples/push-cdn/marshal.rs b/crates/examples/cdn/marshal.rs similarity index 55% rename from crates/examples/push-cdn/marshal.rs rename to crates/examples/cdn/marshal.rs index 569cb0dc33..f4a5d0d969 100644 --- a/crates/examples/push-cdn/marshal.rs +++ b/crates/examples/cdn/marshal.rs @@ -1,27 +1,22 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. +//! The marshal is the component of the CDN that authenticates users and routes +//! them to the appropriate broker +//! +//! This is meant to be run externally, e.g. when running benchmarks on the protocol. +//! If you just want to run everything required, you can use the `all` example -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! The following is the main `Marshal` binary, which just instantiates and runs -//! a `Marshal` object. - -use anyhow::Result; +use anyhow::{Context, Result}; use cdn_marshal::{Config, Marshal}; use clap::Parser; -use hotshot::traits::implementations::ProductionDef; +use hotshot::{helpers::initialize_logging, traits::implementations::ProductionDef}; use hotshot_example_types::node_types::TestTypes; use hotshot_types::traits::node_implementation::NodeType; -use tracing_subscriber::EnvFilter; - -// TODO: forall, add logging where we need it #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] /// The main component of the push CDN. struct Args { - /// The discovery client endpoint (including scheme) to connect to + /// The discovery client endpoint (including scheme) to connect to. + /// This is a URL pointing to a `KeyDB` database (e.g. `redis://127.0.0.1:6789`). #[arg(short, long)] discovery_endpoint: String, @@ -43,13 +38,6 @@ struct Args { /// If not provided, a local, pinned CA is used #[arg(long)] ca_key_path: Option, - - /// The size of the global memory pool (in bytes). This is the maximum number of bytes that - /// can be allocated at once for all connections. A connection will block if it - /// tries to allocate more than this amount until some memory is freed. - /// Default is 1GB. - #[arg(long, default_value_t = 1_073_741_824)] - global_memory_pool_size: usize, } #[tokio::main] @@ -58,16 +46,7 @@ async fn main() -> Result<()> { let args = Args::parse(); // Initialize tracing - if std::env::var("RUST_LOG_FORMAT") == Ok("json".to_string()) { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .json() - .init(); - } else { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .init(); - } + initialize_logging(); // Create a new `Config` let config = Config { @@ -76,7 +55,8 @@ async fn main() -> Result<()> { metrics_bind_endpoint: args.metrics_bind_endpoint, ca_cert_path: args.ca_cert_path, ca_key_path: args.ca_key_path, - global_memory_pool_size: Some(args.global_memory_pool_size), + // Use a 1GB memory pool + global_memory_pool_size: Some(1_073_741_824), }; // Create new `Marshal` from the config @@ -84,7 +64,5 @@ async fn main() -> Result<()> { Marshal::::SignatureKey>>::new(config).await?; // Start the main loop, consuming it - marshal.start().await?; - - Ok(()) + marshal.start().await.with_context(|| "Marshal exited") } diff --git a/crates/examples/combined/all.rs b/crates/examples/combined/all.rs deleted file mode 100644 index 89864af1d1..0000000000 --- a/crates/examples/combined/all.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! An example program using both the web server and libp2p -/// types used for this example -pub mod types; - -use std::path::Path; - -use cdn_broker::{reexports::def::hook::NoMessageHook, Broker}; -use cdn_marshal::Marshal; -use hotshot::{ - helpers::initialize_logging, - traits::implementations::{KeyPair, TestingDef, WrappedSignatureKey}, - types::SignatureKey, -}; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use hotshot_types::traits::node_implementation::NodeType; -use infra::{gen_local_address, BUILDER_BASE_PORT, VALIDATOR_BASE_PORT}; -use rand::{rngs::StdRng, RngCore, SeedableRng}; -use tokio::spawn; -use tracing::{error, instrument}; - -use crate::{ - infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}, - types::{Network, NodeImpl, ThisRun}, -}; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let (config, orchestrator_url) = read_orchestrator_init_config::(); - - // The configuration we are using for testing is 2 brokers & 1 marshal - // A keypair shared between brokers - let (broker_public_key, broker_private_key) = - ::SignatureKey::generated_from_seed_indexed([0u8; 32], 1337); - - // Get the OS temporary directory - let temp_dir = std::env::temp_dir(); - - // Create an SQLite file inside of the temporary directory - let discovery_endpoint = temp_dir - .join(Path::new(&format!( - "test-{}.sqlite", - StdRng::from_entropy().next_u64() - ))) - .to_string_lossy() - .into_owned(); - - // 2 brokers - for _ in 0..2 { - // Get the ports to bind to - let private_port = portpicker::pick_unused_port().expect("could not find an open port"); - let public_port = portpicker::pick_unused_port().expect("could not find an open port"); - - // Extrapolate addresses - let private_address = format!("127.0.0.1:{private_port}"); - let public_address = format!("127.0.0.1:{public_port}"); - - let config: cdn_broker::Config::SignatureKey>> = - cdn_broker::Config { - discovery_endpoint: discovery_endpoint.clone(), - public_advertise_endpoint: public_address.clone(), - public_bind_endpoint: public_address, - private_advertise_endpoint: private_address.clone(), - private_bind_endpoint: private_address, - - keypair: KeyPair { - public_key: WrappedSignatureKey(broker_public_key), - private_key: broker_private_key.clone(), - }, - - user_message_hook: NoMessageHook, - broker_message_hook: NoMessageHook, - - metrics_bind_endpoint: None, - ca_cert_path: None, - ca_key_path: None, - global_memory_pool_size: Some(1024 * 1024 * 1024), - }; - - // Create and spawn the broker - spawn(async move { - let broker: Broker::SignatureKey>> = - Broker::new(config).await.expect("broker failed to start"); - - // Error if we stopped unexpectedly - if let Err(err) = broker.start().await { - error!("broker stopped: {err}"); - } - }); - } - - // Get the port to use for the marshal - let marshal_endpoint = config - .cdn_marshal_address - .clone() - .expect("CDN marshal address must be specified"); - - // Configure the marshal - let marshal_config = cdn_marshal::Config { - bind_endpoint: marshal_endpoint.clone(), - discovery_endpoint, - metrics_bind_endpoint: None, - ca_cert_path: None, - ca_key_path: None, - global_memory_pool_size: Some(1024 * 1024 * 1024), - }; - - // Spawn the marshal - spawn(async move { - let marshal: Marshal::SignatureKey>> = - Marshal::new(marshal_config) - .await - .expect("failed to spawn marshal"); - - // Error if we stopped unexpectedly - if let Err(err) = marshal.start().await { - error!("broker stopped: {err}"); - } - }); - - // orchestrator - spawn(run_orchestrator::(OrchestratorArgs { - url: orchestrator_url.clone(), - config: config.clone(), - })); - - // nodes - let mut nodes = Vec::new(); - for i in 0..config.config.num_nodes_with_stake.into() { - // Calculate our libp2p advertise address, which we will later derive the - // bind address from for example purposes. - let advertise_address = gen_local_address::(i); - let orchestrator_url = orchestrator_url.clone(); - let builder_address = gen_local_address::(i); - - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs { - url: orchestrator_url, - advertise_address: Some(advertise_address.to_string()), - builder_address: Some(builder_address), - network_config_file: None, - }, - ) - .await; - }); - nodes.push(node); - } - futures::future::join_all(nodes).await; -} diff --git a/crates/examples/combined/multi-validator.rs b/crates/examples/combined/multi-validator.rs deleted file mode 100644 index b721cb5c4f..0000000000 --- a/crates/examples/combined/multi-validator.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A multi-validator using both the web server libp2p -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::{MultiValidatorArgs, ValidatorArgs}; -use tokio::spawn; -use tracing::instrument; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let args = MultiValidatorArgs::parse(); - tracing::debug!("connecting to orchestrator at {:?}", args.url); - let mut nodes = Vec::new(); - for node_index in 0..args.num_nodes { - let args = args.clone(); - - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs::from_multi_args(args, node_index), - ) - .await; - }); - nodes.push(node); - } - let _result = futures::future::join_all(nodes).await; -} diff --git a/crates/examples/combined/orchestrator.rs b/crates/examples/combined/orchestrator.rs deleted file mode 100644 index c3d399f489..0000000000 --- a/crates/examples/combined/orchestrator.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! Orchestrator using the web server -/// types used for this example -pub mod types; - -use hotshot::helpers::initialize_logging; -use hotshot_example_types::state_types::TestTypes; -use tracing::instrument; - -use crate::infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}; -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let (config, orchestrator_url) = read_orchestrator_init_config::(); - run_orchestrator::(OrchestratorArgs:: { - url: orchestrator_url.clone(), - config: config.clone(), - }) - .await; -} diff --git a/crates/examples/combined/types.rs b/crates/examples/combined/types.rs deleted file mode 100644 index 1209891b71..0000000000 --- a/crates/examples/combined/types.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -use std::fmt::Debug; - -use hotshot::traits::implementations::CombinedNetworks; -use hotshot_example_types::{ - auction_results_provider_types::TestAuctionResultsProvider, state_types::TestTypes, - storage_types::TestStorage, -}; -use hotshot_types::traits::node_implementation::NodeImplementation; -use serde::{Deserialize, Serialize}; - -use crate::infra::CombinedDaRun; - -/// dummy struct so we can choose types -#[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] -pub struct NodeImpl {} - -/// Convenience type alias -pub type Network = CombinedNetworks; - -impl NodeImplementation for NodeImpl { - type Network = Network; - type Storage = TestStorage; - type AuctionResultsProvider = TestAuctionResultsProvider; -} -/// convenience type alias -pub type ThisRun = CombinedDaRun; diff --git a/crates/examples/combined/validator.rs b/crates/examples/combined/validator.rs deleted file mode 100644 index fd6ff83957..0000000000 --- a/crates/examples/combined/validator.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A validator using both the web server and libp2p - -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use local_ip_address::local_ip; -use tracing::{debug, instrument}; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let mut args = ValidatorArgs::parse(); - - // If we did not set the advertise address, use our local IP and port 8000 - let local_ip = local_ip().expect("failed to get local IP"); - args.advertise_address = Some(args.advertise_address.unwrap_or(format!("{local_ip}:8000"))); - - debug!("connecting to orchestrator at {:?}", args.url); - infra::main_entry_point::(args).await; -} diff --git a/crates/examples/common.rs b/crates/examples/common.rs new file mode 100644 index 0000000000..88a1885ddf --- /dev/null +++ b/crates/examples/common.rs @@ -0,0 +1,377 @@ +use std::time::Instant; + +use async_lock::RwLock; +use hotshot_types::{ + data::EpochNumber, traits::node_implementation::ConsensusTime, utils::non_crypto_hash, +}; +use simple_moving_average::SingleSumSMA; +use simple_moving_average::SMA; + +/// The type of network to use for the example +#[derive(Debug, PartialEq, Eq)] +enum NetworkType { + /// A combined network, which is a combination of a Libp2p and Push CDN network + Combined, + + /// A network solely using the Push CDN + Cdn, + + /// A Libp2p network + LibP2P, +} + +/// This is a testing function which allows us to easily determine if a node should be a DA node +fn is_da_node(index: usize, num_da_nodes: usize) -> bool { + index < num_da_nodes +} + +/// This is a testing function which allows us to easily generate peer configs from indexes +fn peer_info_from_index(index: usize) -> hotshot_types::PeerConfig { + // Get the node's public key + let (public_key, _) = + hotshot::types::BLSPubKey::generated_from_seed_indexed([0u8; 32], index as u64); + + // Generate the peer config + hotshot_types::PeerConfig { + stake_table_entry: public_key.stake_table_entry(1), + state_ver_key: hotshot_types::light_client::StateKeyPair::default() + .0 + .ver_key(), + } +} + +/// Create a new Push CDN network +fn new_cdn_network( + marshal_address: Option, + is_da_node: bool, + public_key: &BLSPubKey, + private_key: &BLSPrivKey, +) -> Result>> { + // If the marshal endpoint is not provided, we don't need to create a CDN network + let Some(marshal_address) = marshal_address else { + anyhow::bail!("Marshal endpoint is required for CDN networks"); + }; + + // Subscribe to topics based on whether we're a DA node or not + let mut topics = vec![CdnTopic::Global]; + if is_da_node { + topics.push(CdnTopic::Da); + } + + // Create and return the network + Ok(Arc::new( + PushCdnNetwork::new( + marshal_address, + topics, + KeyPair { + public_key: WrappedSignatureKey(*public_key), + private_key: private_key.clone(), + }, + CdnMetricsValue::default(), + ) + .with_context(|| "Failed to create Push CDN network")?, + )) +} + +/// A helper function to create a Libp2p network +async fn new_libp2p_network( + bind_address: Multiaddr, + total_num_nodes: usize, + public_key: &BLSPubKey, + private_key: &BLSPrivKey, + known_libp2p_nodes: &[Multiaddr], +) -> Result>> { + // Derive the Libp2p keypair from the private key + let libp2p_keypair = derive_libp2p_keypair::(private_key) + .with_context(|| "Failed to derive libp2p keypair")?; + + // Sign our Libp2p lookup record value + let lookup_record_value = RecordValue::new_signed( + &RecordKey::new(Namespace::Lookup, public_key.to_bytes()), + libp2p_keypair.public().to_peer_id().to_bytes(), + private_key, + ) + .expect("Failed to sign DHT lookup record"); + + // Configure Libp2p + let libp2p_config = Libp2pConfig { + keypair: libp2p_keypair, + bind_address, + known_peers: known_libp2p_nodes.to_vec(), + quorum_membership: None, // This disables stake-table authentication + auth_message: None, // This disables stake-table authentication + gossip_config: GossipConfig::default(), + request_response_config: RequestResponseConfig::default(), + kademlia_config: KademliaConfig { + replication_factor: total_num_nodes * 2 / 3, + record_ttl: None, + publication_interval: None, + file_path: format!("/tmp/kademlia-{}.db", rand::random::()), + lookup_record_value, + }, + }; + + // Create the network with the config + Ok(Arc::new( + Libp2pNetwork::new( + libp2p_config, + public_key, + Libp2pMetricsValue::default(), + None, + ) + .await + .with_context(|| "Failed to create libp2p network")?, + )) +} + +/// A helper function to create a Combined network, which is a combination of a Libp2p and Push CDN network +async fn new_combined_network( + marshal_address: Option, + libp2p_bind_address: Multiaddr, + total_num_nodes: usize, + known_libp2p_nodes: &[Multiaddr], + is_da_node: bool, + public_key: &BLSPubKey, + private_key: &BLSPrivKey, +) -> Result>> { + // Create the CDN network and launch the CDN + let cdn_network = Arc::into_inner(new_cdn_network( + marshal_address, + is_da_node, + public_key, + private_key, + )?) + .unwrap(); + + // Create the Libp2p network + let libp2p_network = Arc::into_inner( + new_libp2p_network( + libp2p_bind_address, + total_num_nodes, + public_key, + private_key, + known_libp2p_nodes, + ) + .await?, + ) + .unwrap(); + + // Create and return the combined network + Ok(Arc::new( + hotshot::traits::implementations::CombinedNetworks::new( + cdn_network, + libp2p_network, + Some(Duration::from_secs(1)), + ), + )) +} + +#[allow(clippy::too_many_arguments)] +#[allow(clippy::cast_precision_loss)] +#[allow(clippy::cast_sign_loss)] +#[allow(clippy::too_many_lines)] +#[allow(clippy::cast_possible_truncation)] +/// A helper function to start consensus with a builder +async fn start_consensus< + I: hotshot::traits::NodeImplementation< + TestTypes, + Storage = TestStorage, + AuctionResultsProvider = TestAuctionResultsProvider, + >, +>( + public_key: BLSPubKey, + private_key: BLSPrivKey, + config: HotShotConfig, + memberships: hotshot_example_types::node_types::StaticMembership, + network: Arc, + hotshot_initializer: hotshot::HotShotInitializer, + total_num_nodes: usize, + builder_url: Url, + num_transactions_per_view: usize, + transaction_size: usize, + num_views: Option, +) -> Result> { + // Create the marketplace config + let marketplace_config: MarketplaceConfig = MarketplaceConfig { + auction_results_provider: TestAuctionResultsProvider::::default().into(), + // TODO: we need to pass a valid fallback builder url here somehow + fallback_builder_url: Url::parse("http://localhost:8080").unwrap(), + }; + + // Initialize the system context + let handle = SystemContext::::init( + public_key, + private_key, + config, + Arc::new(RwLock::new(memberships.clone())), + network, + hotshot_initializer, + ConsensusMetricsValue::default(), + TestStorage::::default(), + marketplace_config, + ) + .await + .with_context(|| "Failed to initialize system context")? + .0; + + // Each node has to start the builder since we don't have a sovereign builder in this example + let builder_handle = + >::start( + total_num_nodes, + builder_url.clone(), + (), + HashMap::new(), + ) + .await; + + // Start it + builder_handle.start(Box::new(handle.event_stream())); + + // Start consensus + handle.hotshot.start_consensus().await; + + // See if we're a DA node or not + let is_da_node = memberships.has_da_stake(&public_key, EpochNumber::new(0)); + + // Create an LRU cache to store block data if we're DA. We populate this cache when we receive + // the DA proposal for a view and print the data when we actually decide on that view + let mut view_cache = LruCache::new(NonZero::new(100).unwrap()); + + // A cache of outstanding transactions (hashes of). Used to calculate latency. Isn't needed for non-DA nodes + let mut outstanding_transactions: LruCache = + LruCache::new(NonZero::new(10000).unwrap()); + + // The simple moving average, used to calculate throughput + let mut throughput: SingleSumSMA = SingleSumSMA::::new(); + + // The last time we decided on a view (for calculating throughput) + let mut last_decide_time = Instant::now(); + + // Spawn the task to wait for events + let join_handle = tokio::spawn(async move { + // Get the event stream for this particular node + let mut event_stream = handle.event_stream(); + + // Wait for a `Decide` event for the view number we requested + loop { + // Get the next event + let event = event_stream.next().await.unwrap(); + + // DA proposals contain the full list of transactions. We can use this to cache + // the size of the proposed block + if let EventType::DaProposal { proposal, .. } = event.event { + // Decode the transactions. We use this to log the size of the proposed block + // when we decide on a view + let transactions = + match TestTransaction::decode(&proposal.data.encoded_transactions) { + Ok(transactions) => transactions, + Err(err) => { + tracing::error!("Failed to decode transactions: {:?}", err); + continue; + } + }; + + // Get the number of transactions in the proposed block + let num_transactions = transactions.len(); + + // Sum the total number of bytes in the proposed block and cache + // the hash so we can calculate latency + let mut sum = 0; + let mut submitted_times = Vec::new(); + for transaction in transactions { + // Add the size of the transaction to the sum + sum += transaction.bytes().len(); + + // If we can find the transaction in the cache, add the hash of the transaction to the cache + if let Some(&instant) = + outstanding_transactions.get(&non_crypto_hash(transaction.bytes())) + { + submitted_times.push(instant); + } + } + + // Insert the size of the proposed block and the number of transactions into the cache. + // We use this to log the size of the proposed block when we decide on a view + view_cache.put( + *proposal.data.view_number, + (sum, num_transactions, submitted_times), + ); + + // A `Decide` event contains data that HotShot has decided on + } else if let EventType::Decide { qc, .. } = event.event { + // If we cached the size of the proposed block, log it + if let Some((block_size, num_transactions, submitted_times)) = + view_cache.get(&*qc.view_number) + { + // Calculate the average latency of the transactions + let mut total_latency = Duration::default(); + let mut num_found_transactions = 0; + for submitted_time in submitted_times { + total_latency += submitted_time.elapsed(); + num_found_transactions += 1; + } + let average_latency = total_latency.checked_div(num_found_transactions); + + // Update the throughput SMA + throughput + .add_sample(*block_size as f64 / last_decide_time.elapsed().as_secs_f64()); + + // Update the last decided time + last_decide_time = Instant::now(); + + // If we have a valid average latency, log it + if let Some(average_latency) = average_latency { + info!( + block_size = block_size, + num_txs = num_transactions, + avg_tx_latency =? average_latency, + avg_throughput = format!("{}/s", bytesize::ByteSize::b(throughput.get_average() as u64)), + "Decided on view {}", + *qc.view_number + ); + } else { + info!( + block_size = block_size, + num_txs = num_transactions, + "Decided on view {}", + *qc.view_number + ); + } + } else { + info!("Decided on view {}", *qc.view_number); + } + + // Generate and submit the requested number of transactions + for _ in 0..num_transactions_per_view { + // Generate a random transaction + let mut transaction_bytes = vec![0u8; transaction_size]; + rand::thread_rng().fill(&mut transaction_bytes[..]); + + // If we're a DA node, cache the transaction so we can calculate latency + if is_da_node { + outstanding_transactions + .put(non_crypto_hash(&transaction_bytes), Instant::now()); + } + + // Submit the transaction + if let Err(err) = handle + .submit_transaction(TestTransaction::new(transaction_bytes)) + .await + { + tracing::error!("Failed to submit transaction: {:?}", err); + }; + } + + // If we have a specific view number we want to wait for, check if we've reached it + if let Some(num_views) = num_views { + if *qc.view_number == num_views as u64 { + // Break when we've decided on the view number we requested + break; + } + } + } + } + }); + + Ok(join_handle) +} diff --git a/crates/examples/coordinator.rs b/crates/examples/coordinator.rs new file mode 100644 index 0000000000..dd041832f4 --- /dev/null +++ b/crates/examples/coordinator.rs @@ -0,0 +1,127 @@ +//! This service helps coordinate running multiple nodes where each needs +//! to be assigned a unique index +//! +//! This is meant to be run externally, e.g. when running benchmarks on the protocol. +//! If you just want to run everything required, you can use the `all` example + +use std::{ + collections::HashSet, + net::SocketAddr, + str::FromStr, + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, +}; + +use anyhow::{Context, Result}; +use bytes::Bytes; +use clap::Parser; +use hotshot::helpers::initialize_logging; +use libp2p::{multiaddr::Protocol, Multiaddr}; +use parking_lot::RwLock; +use warp::Filter; + +/// The coordinator service, used to assign unique indices to nodes when running benchmarks +#[derive(Parser)] +struct Args { + /// The address to bind to + #[arg(long, default_value = "127.0.0.1:3030")] + bind_address: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + // Initialize logging + initialize_logging(); + + // Parse the command-line arguments + let args = Args::parse(); + + // Parse the bind address + let bind_address = args + .bind_address + .parse::() + .with_context(|| "Failed to parse bind address")?; + + // Create a shared counter + let counter = Arc::new(AtomicU32::new(0)); + let counter = warp::any().map(move || Arc::clone(&counter)); + + // Create a shared set of multiaddrs for Libp2p + let libp2p_multiaddrs = Arc::new(RwLock::new(HashSet::new())); + let libp2p_multiaddrs = warp::any().map(move || Arc::clone(&libp2p_multiaddrs)); + + // `/index` returns the node index we are assigned + let index = warp::path!("index") + .and(counter.clone()) + .map(|counter: Arc| counter.fetch_add(1, Ordering::SeqCst).to_string()); + + // POST `/libp2p_info` submits libp2p information to the coordinator + let submit_libp2p_info = warp::path!("libp2p-info") + .and(warp::post()) + .and(warp::body::bytes()) + .and(libp2p_multiaddrs.clone()) + .map( + |body: Bytes, libp2p_multiaddrs: Arc>>| { + // Attempt to process as a string + let Ok(string) = String::from_utf8(body.to_vec()) else { + return "Failed to parse body as string".to_string(); + }; + + // Attempt to parse the string as a Libp2p Multiaddr + let Ok(multiaddr) = Multiaddr::from_str(&string) else { + return "Failed to parse body as Multiaddr".to_string(); + }; + + // Pop off the last protocol + let Some(last_protocol) = multiaddr.clone().pop() else { + return "Failed to get last protocol of multiaddr".to_string(); + }; + + // Make sure it is the P2p protocol + let Protocol::P2p(_) = last_protocol else { + return "Failed to get P2p protocol of multiaddr".to_string(); + }; + + // Add it to the set + libp2p_multiaddrs.write().insert(multiaddr); + + "Ok".to_string() + }, + ); + + // GET `/libp2p_info` returns the list of libp2p multiaddrs + let get_libp2p_info = warp::path!("libp2p-info") + .and(libp2p_multiaddrs.clone()) + .map(|libp2p_multiaddrs: Arc>>| { + // Get the list of multiaddrs + let multiaddrs = libp2p_multiaddrs.read().clone(); + + // Convert the multiaddrs to a string, separated by newlines + multiaddrs + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") + }); + + // `/reset` resets the state of the coordinator + let reset = warp::path!("reset") + .and(counter) + .and(libp2p_multiaddrs) + .map( + |counter: Arc, libp2p_multiaddrs: Arc>>| { + counter.store(0, Ordering::SeqCst); + libp2p_multiaddrs.write().clear(); + "Ok" + }, + ); + + // Run the server + warp::serve(index.or(reset).or(submit_libp2p_info).or(get_libp2p_info)) + .run(bind_address) + .await; + + Ok(()) +} diff --git a/crates/examples/infra/mod.rs b/crates/examples/infra/mod.rs deleted file mode 100755 index 63dd75d203..0000000000 --- a/crates/examples/infra/mod.rs +++ /dev/null @@ -1,1139 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -#![allow(clippy::panic)] -use std::{ - collections::HashMap, - fmt::Debug, - fs, - net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}, - num::NonZeroUsize, - sync::Arc, - time::Instant, -}; - -use async_lock::RwLock; -use async_trait::async_trait; -use cdn_broker::reexports::crypto::signature::KeyPair; -use chrono::Utc; -use clap::{value_parser, Arg, Command, Parser}; -use futures::StreamExt; -use hotshot::{ - traits::{ - implementations::{ - derive_libp2p_multiaddr, derive_libp2p_peer_id, CdnMetricsValue, CdnTopic, - CombinedNetworks, Libp2pMetricsValue, Libp2pNetwork, PushCdnNetwork, - WrappedSignatureKey, - }, - BlockPayload, NodeImplementation, - }, - types::SystemContextHandle, - MarketplaceConfig, SystemContext, -}; -use hotshot_example_types::{ - auction_results_provider_types::TestAuctionResultsProvider, - block_types::{TestBlockHeader, TestBlockPayload, TestTransaction}, - node_types::{Libp2pImpl, PushCdnImpl}, - state_types::TestInstanceState, - storage_types::TestStorage, -}; -use hotshot_orchestrator::{ - self, - client::{get_complete_config, BenchResults, OrchestratorClient, ValidatorArgs}, -}; -use hotshot_testing::block_builder::{ - BuilderTask, RandomBuilderImplementation, SimpleBuilderImplementation, - TestBuilderImplementation, -}; -use hotshot_types::{ - consensus::ConsensusMetricsValue, - data::{Leaf, TestableLeaf}, - event::{Event, EventType}, - network::{BuilderType, NetworkConfig, NetworkConfigFile, NetworkConfigSource}, - traits::{ - block_contents::{BlockHeader, TestableBlock}, - election::Membership, - network::ConnectedNetwork, - node_implementation::{ConsensusTime, NodeType, Versions}, - states::TestableState, - }, - HotShotConfig, PeerConfig, ValidatorConfig, -}; -use libp2p_networking::network::{GossipConfig, RequestResponseConfig}; -use rand::{rngs::StdRng, SeedableRng}; -use surf_disco::Url; -use tracing::{debug, error, info, warn}; - -#[derive(Debug, Clone)] -/// Arguments passed to the orchestrator -pub struct OrchestratorArgs { - /// The url the orchestrator runs on; this should be in the form of `http://localhost:5555` or `http://0.0.0.0:5555` - pub url: Url, - /// The configuration file to be used for this run - pub config: NetworkConfig, -} - -#[derive(Parser, Debug, Clone)] -#[command( - name = "Multi-machine consensus", - about = "Simulates consensus among multiple machines" -)] -/// The configuration file to be used for this run -pub struct ConfigArgs { - /// The configuration file to be used for this run - pub config_file: String, -} - -impl Default for ConfigArgs { - fn default() -> Self { - Self { - config_file: "./crates/orchestrator/run-config.toml".to_string(), - } - } -} - -/// Reads the orchestrator initialization config from the command line -/// # Panics -/// If unable to read the config file from the command line -#[allow(clippy::too_many_lines)] -pub fn read_orchestrator_init_config() -> (NetworkConfig, Url) -{ - // assign default setting - let mut orchestrator_url = Url::parse("http://localhost:4444").unwrap(); - let mut args = ConfigArgs::default(); - // start reading from the command line - let matches = Command::new("orchestrator") - .arg( - Arg::new("config_file") - .short('c') - .long("config_file") - .value_name("FILE") - .help("Sets a custom config file with default values, some might be changed if they are set manually in the command line") - .required(true), - ) - .arg( - Arg::new("total_nodes") - .short('n') - .long("total_nodes") - .value_name("NUM") - .help("Sets the total number of nodes") - .required(false), - ) - .arg( - Arg::new("da_committee_size") - .short('d') - .long("da_committee_size") - .value_name("NUM") - .help("Sets the size of the data availability committee") - .required(false), - ) - .arg( - Arg::new("transactions_per_round") - .short('t') - .long("transactions_per_round") - .value_name("NUM") - .help("Sets the number of transactions per round") - .required(false), - ) - .arg( - Arg::new("transaction_size") - .short('s') - .long("transaction_size") - .value_name("NUM") - .help("Sets the size of each transaction in bytes") - .required(false), - ) - .arg( - Arg::new("rounds") - .short('r') - .long("rounds") - .value_name("NUM") - .help("Sets the number of rounds to run") - .required(false), - ) - .arg( - Arg::new("commit_sha") - .short('o') - .long("commit_sha") - .value_name("SHA") - .help("Sets the commit sha to output in the results") - .required(false), - ) - .arg( - Arg::new("orchestrator_url") - .short('u') - .long("orchestrator_url") - .value_name("URL") - .help("Sets the url of the orchestrator") - .required(false), - ) - .arg( - Arg::new("fixed_leader_for_gpuvid") - .short('f') - .long("fixed_leader_for_gpuvid") - .value_name("BOOL") - .help("Sets the number of fixed leader for gpu vid, only be used when leaders running on gpu") - .required(false), - ) - .arg( - Arg::new("builder") - .short('b') - .long("builder") - .value_name("BUILDER_TYPE") - .value_parser(value_parser!(BuilderType)) - .help("Sets type of builder. `simple` or `random` to run corresponding integrated builder, `external` to use the one specified by `[config.builder_url]` in config") - .required(false), - ) - .arg( - Arg::new("cdn_marshal_address") - .short('m') - .long("cdn_marshal_address") - .value_name("URL") - .help("Sets the url for cdn_broker_marshal_endpoint") - .required(false), - ) - .get_matches(); - - if let Some(config_file_string) = matches.get_one::("config_file") { - args = ConfigArgs { - config_file: config_file_string.clone(), - }; - } else { - error!("No config file provided, we'll use the default one."); - } - let mut config: NetworkConfig = - load_config_from_file::(&args.config_file); - - if let Some(total_nodes_string) = matches.get_one::("total_nodes") { - config.config.num_nodes_with_stake = total_nodes_string.parse::().unwrap(); - config.config.known_nodes_with_stake = - vec![PeerConfig::default(); config.config.num_nodes_with_stake.get() as usize]; - error!( - "config.config.total_nodes: {:?}", - config.config.num_nodes_with_stake - ); - } - if let Some(da_committee_size_string) = matches.get_one::("da_committee_size") { - config.config.da_staked_committee_size = da_committee_size_string.parse::().unwrap(); - } - if let Some(fixed_leader_for_gpuvid_string) = - matches.get_one::("fixed_leader_for_gpuvid") - { - config.config.fixed_leader_for_gpuvid = - fixed_leader_for_gpuvid_string.parse::().unwrap(); - } - if let Some(transactions_per_round_string) = matches.get_one::("transactions_per_round") - { - config.transactions_per_round = transactions_per_round_string.parse::().unwrap(); - } - if let Some(transaction_size_string) = matches.get_one::("transaction_size") { - config.transaction_size = transaction_size_string.parse::().unwrap(); - } - if let Some(rounds_string) = matches.get_one::("rounds") { - config.rounds = rounds_string.parse::().unwrap(); - } - if let Some(commit_sha_string) = matches.get_one::("commit_sha") { - config.commit_sha = commit_sha_string.to_string(); - } - if let Some(orchestrator_url_string) = matches.get_one::("orchestrator_url") { - orchestrator_url = Url::parse(orchestrator_url_string).unwrap(); - } - if let Some(builder_type) = matches.get_one::("builder") { - config.builder = *builder_type; - } - if let Some(cdn_marshal_address_string) = matches.get_one::("cdn_marshal_address") { - config.cdn_marshal_address = Some(cdn_marshal_address_string.to_string()); - } - - (config, orchestrator_url) -} - -/// Reads a network configuration from a given filepath -/// # Panics -/// if unable to convert the config file into toml -/// # Note -/// This derived config is used for initialization of orchestrator, -/// therefore `known_nodes_with_stake` will be an initialized -/// vector full of the node's own config. -#[must_use] -pub fn load_config_from_file( - config_file: &str, -) -> NetworkConfig { - let config_file_as_string: String = fs::read_to_string(config_file) - .unwrap_or_else(|_| panic!("Could not read config file located at {config_file}")); - let config_toml: NetworkConfigFile = - toml::from_str::>(&config_file_as_string) - .expect("Unable to convert config file to TOML"); - - let mut config: NetworkConfig = config_toml.into(); - - // initialize it with size for better assignment of peers' config - config.config.known_nodes_with_stake = - vec![PeerConfig::default(); config.config.num_nodes_with_stake.get() as usize]; - - config -} - -/// Runs the orchestrator -pub async fn run_orchestrator( - OrchestratorArgs { url, config }: OrchestratorArgs, -) { - println!("Starting orchestrator",); - let _ = hotshot_orchestrator::run_orchestrator::(config, url).await; -} - -/// Helper function to calculate the number of transactions to send per node per round -#[allow(clippy::cast_possible_truncation)] -fn calculate_num_tx_per_round( - node_index: u64, - total_num_nodes: usize, - transactions_per_round: usize, -) -> usize { - transactions_per_round / total_num_nodes - + usize::from( - (total_num_nodes) - < (transactions_per_round % total_num_nodes) + 1 + (node_index as usize), - ) -} - -/// Helper function to generate transactions a given node should send -fn generate_transactions>( - node_index: u64, - rounds: usize, - transactions_to_send_per_round: usize, - transaction_size: usize, -) -> Vec -where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, -{ - let mut txn_rng = StdRng::seed_from_u64(node_index); - let mut transactions = Vec::new(); - - for _ in 0..rounds { - for _ in 0..transactions_to_send_per_round { - let txn = ::create_random_transaction( - None, - &mut txn_rng, - transaction_size as u64, - ); - - transactions.push(txn); - } - } - transactions -} - -/// Defines the behavior of a "run" of the network with a given configuration -#[async_trait] -pub trait RunDa< - TYPES: NodeType, - NETWORK: ConnectedNetwork, - NODE: NodeImplementation< - TYPES, - Network = NETWORK, - Storage = TestStorage, - AuctionResultsProvider = TestAuctionResultsProvider, - >, - V: Versions, -> where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - TYPES: NodeType, - Leaf: TestableLeaf, - Self: Sync, -{ - /// Initializes networking, returns self - async fn initialize_networking( - config: NetworkConfig, - validator_config: ValidatorConfig, - libp2p_advertise_address: Option, - membership: &Arc::Membership>>, - ) -> Self; - - /// Initializes the genesis state and HotShot instance; does not start HotShot consensus - /// # Panics if it cannot generate a genesis block, fails to initialize HotShot, or cannot - /// get the anchored view - /// Note: sequencing leaf does not have state, so does not return state - async fn initialize_state_and_hotshot( - &self, - membership: Arc::Membership>>, - ) -> SystemContextHandle { - let initializer = - hotshot::HotShotInitializer::::from_genesis::(TestInstanceState::default()) - .await - .expect("Couldn't generate genesis block"); - - let config = self.config(); - let validator_config = self.validator_config(); - - // Get KeyPair for certificate Aggregation - let pk = validator_config.public_key.clone(); - let sk = validator_config.private_key.clone(); - - let network = self.network(); - - let marketplace_config = MarketplaceConfig { - auction_results_provider: TestAuctionResultsProvider::::default().into(), - // TODO: we need to pass a valid fallback builder url here somehow - fallback_builder_url: config.config.builder_urls.first().clone(), - }; - - SystemContext::init( - pk, - sk, - config.node_index, - config.config, - membership, - Arc::from(network), - initializer, - ConsensusMetricsValue::default(), - TestStorage::::default(), - marketplace_config, - ) - .await - .expect("Could not init hotshot") - .0 - } - - /// Starts HotShot consensus, returns when consensus has finished - #[allow(clippy::too_many_lines)] - async fn run_hotshot( - &self, - context: SystemContextHandle, - transactions: &mut Vec, - transactions_to_send_per_round: u64, - transaction_size_in_bytes: u64, - ) -> BenchResults { - let NetworkConfig { - rounds, node_index, .. - } = self.config(); - - let mut total_transactions_committed = 0; - let mut total_transactions_sent = 0; - let mut minimum_latency = 1000; - let mut maximum_latency = 0; - let mut total_latency = 0; - let mut num_latency = 0; - - info!("Starting HotShot example!"); - let start = Instant::now(); - - let mut event_stream = context.event_stream(); - let mut anchor_view: TYPES::View = ::genesis(); - let mut num_successful_commits = 0; - - context.hotshot.start_consensus().await; - - loop { - match event_stream.next().await { - None => { - panic!("Error! Event stream completed before consensus ended."); - } - Some(Event { event, .. }) => { - match event { - EventType::Error { error } => { - error!("Error in consensus: {:?}", error); - // TODO what to do here - } - EventType::Decide { - leaf_chain, - qc: _, - block_size, - } => { - let current_timestamp = Utc::now().timestamp(); - // this might be a obob - if let Some(leaf_info) = leaf_chain.first() { - let leaf = &leaf_info.leaf; - info!("Decide event for leaf: {}", *leaf.view_number()); - - // iterate all the decided transactions to calculate latency - if let Some(block_payload) = &leaf.block_payload() { - for tx in - block_payload.transactions(leaf.block_header().metadata()) - { - let restored_timestamp_vec = - tx.bytes()[tx.bytes().len() - 8..].to_vec(); - let restored_timestamp = i64::from_be_bytes( - restored_timestamp_vec.as_slice().try_into().unwrap(), - ); - let cur_latency = current_timestamp - restored_timestamp; - total_latency += cur_latency; - num_latency += 1; - minimum_latency = - std::cmp::min(minimum_latency, cur_latency); - maximum_latency = - std::cmp::max(maximum_latency, cur_latency); - } - } - - let new_anchor = leaf.view_number(); - if new_anchor >= anchor_view { - anchor_view = leaf.view_number(); - } - - // send transactions - for _ in 0..transactions_to_send_per_round { - // append current timestamp to the tx to calc latency - let timestamp = Utc::now().timestamp(); - let mut tx = transactions.remove(0).into_bytes(); - let mut timestamp_vec = timestamp.to_be_bytes().to_vec(); - tx.append(&mut timestamp_vec); - - () = context - .submit_transaction(TestTransaction::new(tx)) - .await - .unwrap(); - total_transactions_sent += 1; - } - } - - if let Some(size) = block_size { - total_transactions_committed += size; - debug!("[{node_index}] got block with size: {:?}", size); - } - - num_successful_commits += leaf_chain.len(); - if num_successful_commits >= rounds { - break; - } - - if leaf_chain.len() > 1 { - warn!("Leaf chain is greater than 1 with len {}", leaf_chain.len()); - } - // when we make progress, submit new events - } - EventType::ReplicaViewTimeout { view_number } => { - warn!("Timed out as a replicas in view {:?}", view_number); - } - EventType::ViewTimeout { view_number } => { - warn!("Timed out in view {:?}", view_number); - } - _ => {} // mostly DA proposal - } - } - } - } - let num_eligible_leaders = context - .hotshot - .memberships - .read() - .await - .committee_leaders(TYPES::View::genesis(), TYPES::Epoch::genesis()) - .len(); - let consensus_lock = context.hotshot.consensus(); - let consensus = consensus_lock.read().await; - let total_num_views = usize::try_from(consensus.locked_view().u64()).unwrap(); - // `failed_num_views` could include uncommitted views - let failed_num_views = total_num_views - num_successful_commits; - // When posting to the orchestrator, note that the total number of views also include un-finalized views. - println!("[{node_index}]: Total views: {total_num_views}, Failed views: {failed_num_views}, num_successful_commits: {num_successful_commits}"); - // Output run results - let total_time_elapsed = start.elapsed(); // in seconds - println!("[{node_index}]: {rounds} rounds completed in {total_time_elapsed:?} - Total transactions sent: {total_transactions_sent} - Total transactions committed: {total_transactions_committed} - Total commitments: {num_successful_commits}"); - if total_transactions_committed != 0 { - // prevent division by 0 - let total_time_elapsed_sec = std::cmp::max(total_time_elapsed.as_secs(), 1u64); - // extra 8 bytes for timestamp - let throughput_bytes_per_sec = total_transactions_committed - * (transaction_size_in_bytes + 8) - / total_time_elapsed_sec; - let avg_latency_in_sec = total_latency / num_latency; - println!("[{node_index}]: throughput: {throughput_bytes_per_sec} bytes/sec, avg_latency: {avg_latency_in_sec} sec."); - - BenchResults { - partial_results: "Unset".to_string(), - avg_latency_in_sec, - num_latency, - minimum_latency_in_sec: minimum_latency, - maximum_latency_in_sec: maximum_latency, - throughput_bytes_per_sec, - total_transactions_committed, - transaction_size_in_bytes: transaction_size_in_bytes + 8, // extra 8 bytes for timestamp - total_time_elapsed_in_sec: total_time_elapsed.as_secs(), - total_num_views, - failed_num_views, - committee_type: format!( - "{} with {num_eligible_leaders} eligible leaders", - std::any::type_name::() - ), - } - } else { - // all values with zero - BenchResults::default() - } - } - - /// Returns the underlying network for this run - fn network(&self) -> NETWORK; - - /// Returns the config for this run - fn config(&self) -> NetworkConfig; - - /// Returns the validator config with private signature keys for this run. - fn validator_config(&self) -> ValidatorConfig; -} - -// Push CDN - -/// Represents a Push CDN-based run -pub struct PushCdnDaRun { - /// The underlying configuration - config: NetworkConfig, - /// The private validator config - validator_config: ValidatorConfig, - /// The underlying network - network: PushCdnNetwork, -} - -#[async_trait] -impl< - TYPES: NodeType< - Transaction = TestTransaction, - BlockPayload = TestBlockPayload, - BlockHeader = TestBlockHeader, - InstanceState = TestInstanceState, - >, - NODE: NodeImplementation< - TYPES, - Network = PushCdnNetwork, - Storage = TestStorage, - AuctionResultsProvider = TestAuctionResultsProvider, - >, - V: Versions, - > RunDa, NODE, V> for PushCdnDaRun -where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - Leaf: TestableLeaf, - Self: Sync, -{ - async fn initialize_networking( - config: NetworkConfig, - validator_config: ValidatorConfig, - _libp2p_advertise_address: Option, - _membership: &Arc::Membership>>, - ) -> PushCdnDaRun { - // Convert to the Push-CDN-compatible type - let keypair = KeyPair { - public_key: WrappedSignatureKey(validator_config.public_key.clone()), - private_key: validator_config.private_key.clone(), - }; - - // See if we should be DA, subscribe to the DA topic if so - let mut topics = vec![CdnTopic::Global]; - if validator_config.is_da { - topics.push(CdnTopic::Da); - } - - // Create the network and await the initial connection - let network = PushCdnNetwork::new( - config - .cdn_marshal_address - .clone() - .expect("`cdn_marshal_address` needs to be supplied for a push CDN run"), - topics, - keypair, - CdnMetricsValue::default(), - ) - .expect("failed to create network"); - - // Wait for the network to be ready - network.wait_for_ready().await; - - PushCdnDaRun { - config, - validator_config, - network, - } - } - - fn network(&self) -> PushCdnNetwork { - self.network.clone() - } - - fn config(&self) -> NetworkConfig { - self.config.clone() - } - - fn validator_config(&self) -> ValidatorConfig { - self.validator_config.clone() - } -} - -// Libp2p - -/// Represents a libp2p-based run -pub struct Libp2pDaRun { - /// The underlying network configuration - config: NetworkConfig, - /// The private validator config - validator_config: ValidatorConfig, - /// The underlying network - network: Libp2pNetwork, -} - -#[async_trait] -impl< - TYPES: NodeType< - Transaction = TestTransaction, - BlockPayload = TestBlockPayload, - BlockHeader = TestBlockHeader, - InstanceState = TestInstanceState, - >, - NODE: NodeImplementation< - TYPES, - Network = Libp2pNetwork, - Storage = TestStorage, - AuctionResultsProvider = TestAuctionResultsProvider, - >, - V: Versions, - > RunDa, NODE, V> for Libp2pDaRun -where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - Leaf: TestableLeaf, - Self: Sync, -{ - async fn initialize_networking( - config: NetworkConfig, - validator_config: ValidatorConfig, - libp2p_advertise_address: Option, - membership: &Arc::Membership>>, - ) -> Libp2pDaRun { - // Extrapolate keys for ease of use - let public_key = &validator_config.public_key; - let private_key = &validator_config.private_key; - - // In an example, we can calculate the libp2p bind address as a function - // of the advertise address. - let bind_address = if let Some(libp2p_advertise_address) = libp2p_advertise_address { - let libp2p_advertise_address: SocketAddrV4 = libp2p_advertise_address - .parse() - .expect("failed to parse advertise address"); - - // If we have supplied one, use it - SocketAddr::new( - IpAddr::V4(Ipv4Addr::UNSPECIFIED), - libp2p_advertise_address.port(), - ) - .to_string() - } else { - // If not, index a base port with our node index - SocketAddr::new( - IpAddr::V4(Ipv4Addr::UNSPECIFIED), - 8000 + (u16::try_from(config.node_index) - .expect("failed to create advertise address")), - ) - .to_string() - }; - - // Derive the bind address - let bind_address = - derive_libp2p_multiaddr(&bind_address).expect("failed to derive bind address"); - - // Create the Libp2p network - let libp2p_network = Libp2pNetwork::from_config( - config.clone(), - Arc::clone(membership), - GossipConfig::default(), - RequestResponseConfig::default(), - bind_address, - public_key, - private_key, - Libp2pMetricsValue::default(), - ) - .await - .expect("failed to create libp2p network"); - - // Wait for the network to be ready - libp2p_network.wait_for_ready().await; - - Libp2pDaRun { - config, - validator_config, - network: libp2p_network, - } - } - - fn network(&self) -> Libp2pNetwork { - self.network.clone() - } - - fn config(&self) -> NetworkConfig { - self.config.clone() - } - - fn validator_config(&self) -> ValidatorConfig { - self.validator_config.clone() - } -} - -// Combined network - -/// Represents a combined-network-based run -pub struct CombinedDaRun { - /// The underlying network configuration - config: NetworkConfig, - /// The private validator config - validator_config: ValidatorConfig, - /// The underlying network - network: CombinedNetworks, -} - -#[async_trait] -impl< - TYPES: NodeType< - Transaction = TestTransaction, - BlockPayload = TestBlockPayload, - BlockHeader = TestBlockHeader, - InstanceState = TestInstanceState, - >, - NODE: NodeImplementation< - TYPES, - Network = CombinedNetworks, - Storage = TestStorage, - AuctionResultsProvider = TestAuctionResultsProvider, - >, - V: Versions, - > RunDa, NODE, V> for CombinedDaRun -where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - Leaf: TestableLeaf, - Self: Sync, -{ - async fn initialize_networking( - config: NetworkConfig, - validator_config: ValidatorConfig, - libp2p_advertise_address: Option, - membership: &Arc::Membership>>, - ) -> CombinedDaRun { - // Initialize our Libp2p network - let libp2p_network: Libp2pDaRun = as RunDa< - TYPES, - Libp2pNetwork, - Libp2pImpl, - V, - >>::initialize_networking( - config.clone(), - validator_config.clone(), - libp2p_advertise_address.clone(), - membership, - ) - .await; - - // Initialize our CDN network - let cdn_network: PushCdnDaRun = as RunDa< - TYPES, - PushCdnNetwork, - PushCdnImpl, - V, - >>::initialize_networking( - config.clone(), - validator_config.clone(), - libp2p_advertise_address, - membership, - ) - .await; - - // Create our combined network config - let delay_duration = config - .combined_network_config - .as_ref() - .map(|config| config.delay_duration); - - // Create our combined network - let network = - CombinedNetworks::new(cdn_network.network, libp2p_network.network, delay_duration); - - // Return the run configuration - CombinedDaRun { - config, - validator_config, - network, - } - } - - fn network(&self) -> CombinedNetworks { - self.network.clone() - } - - fn config(&self) -> NetworkConfig { - self.config.clone() - } - - fn validator_config(&self) -> ValidatorConfig { - self.validator_config.clone() - } -} - -#[allow(clippy::too_many_lines)] -/// Main entry point for validators -/// # Panics -/// if unable to get the local ip address -pub async fn main_entry_point< - TYPES: NodeType< - Transaction = TestTransaction, - BlockHeader = TestBlockHeader, - InstanceState = TestInstanceState, - >, - NETWORK: ConnectedNetwork, - NODE: NodeImplementation< - TYPES, - Network = NETWORK, - Storage = TestStorage, - AuctionResultsProvider = TestAuctionResultsProvider, - >, - V: Versions, - RUNDA: RunDa, ->( - args: ValidatorArgs, -) where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - Leaf: TestableLeaf, -{ - // Initialize logging - hotshot::helpers::initialize_logging(); - - info!("Starting validator"); - - let orchestrator_client: OrchestratorClient = OrchestratorClient::new(args.url.clone()); - - // We assume one node will not call this twice to generate two validator_config-s with same identity. - let validator_config = NetworkConfig::::generate_init_validator_config( - orchestrator_client - .get_node_index_for_init_validator_config() - .await, - // we assign nodes to the DA committee by default - true, - ); - - // Derives our Libp2p private key from our private key, and then returns the public key of that key - let libp2p_public_key = - derive_libp2p_peer_id::(&validator_config.private_key) - .expect("failed to derive Libp2p keypair"); - - // We need this to be able to register our node - let peer_config = - PeerConfig::::to_bytes(&validator_config.public_config()).clone(); - - // Derive the advertise multiaddress from the supplied string - let advertise_multiaddress = args.advertise_address.clone().map(|advertise_address| { - derive_libp2p_multiaddr(&advertise_address).expect("failed to derive Libp2p multiaddr") - }); - - // conditionally save/load config from file or orchestrator - // This is a function that will return correct complete config from orchestrator. - // It takes in a valid args.network_config_file when loading from file, or valid validator_config when loading from orchestrator, the invalid one will be ignored. - // It returns the complete config which also includes peer's public key and public config. - // This function will be taken solely by sequencer right after OrchestratorClient::new, - // which means the previous `generate_validator_config_when_init` will not be taken by sequencer, it's only for key pair generation for testing in hotshot. - - let (mut run_config, validator_config, source) = get_complete_config( - &orchestrator_client, - validator_config, - advertise_multiaddress, - Some(libp2p_public_key), - ) - .await - .expect("failed to get config"); - - let builder_task = initialize_builder( - &mut run_config, - &validator_config, - &args, - &orchestrator_client, - ) - .await; - - run_config.config.builder_urls = orchestrator_client - .get_builder_addresses() - .await - .try_into() - .expect("Orchestrator didn't provide any builder addresses"); - - debug!( - "Assigned urls from orchestrator: {}", - run_config - .config - .builder_urls - .iter() - .map(ToString::to_string) - .collect::>() - .join(",") - ); - - let all_nodes = if cfg!(feature = "fixed-leader-election") { - let mut vec = run_config.config.known_nodes_with_stake.clone(); - vec.truncate(run_config.config.fixed_leader_for_gpuvid); - vec - } else { - run_config.config.known_nodes_with_stake.clone() - }; - let membership = Arc::new(RwLock::new(::Membership::new( - all_nodes, - run_config.config.known_da_nodes.clone(), - ))); - - info!("Initializing networking"); - let run = RUNDA::initialize_networking( - run_config.clone(), - validator_config, - args.advertise_address, - &membership, - ) - .await; - let hotshot = run.initialize_state_and_hotshot(membership).await; - - if let Some(task) = builder_task { - task.start(Box::new(hotshot.event_stream())); - } - - // pre-generate transactions - let NetworkConfig { - transaction_size, - rounds, - transactions_per_round, - node_index, - config: HotShotConfig { - num_nodes_with_stake, - .. - }, - .. - } = run_config; - - let transactions_to_send_per_round = calculate_num_tx_per_round( - node_index, - num_nodes_with_stake.get(), - transactions_per_round, - ); - let mut transactions: Vec = generate_transactions::( - node_index, - rounds, - transactions_to_send_per_round, - transaction_size, - ); - - if let NetworkConfigSource::Orchestrator = source { - info!("Waiting for the start command from orchestrator"); - orchestrator_client - .wait_for_all_nodes_ready(peer_config) - .await; - } - - info!("Starting HotShot"); - let bench_results = run - .run_hotshot( - hotshot, - &mut transactions, - transactions_to_send_per_round as u64, - (transaction_size + 8) as u64, // extra 8 bytes for transaction base, see `create_random_transaction`. - ) - .await; - orchestrator_client.post_bench_results(bench_results).await; -} - -/// Sets correct builder_url and registers a builder with orchestrator if this node is running one. -/// Returns a `BuilderTask` if this node is going to be running a builder. -async fn initialize_builder< - TYPES: NodeType< - Transaction = TestTransaction, - BlockHeader = TestBlockHeader, - InstanceState = TestInstanceState, - >, ->( - run_config: &mut NetworkConfig<::SignatureKey>, - validator_config: &ValidatorConfig<::SignatureKey>, - args: &ValidatorArgs, - orchestrator_client: &OrchestratorClient, -) -> Option>> -where - ::ValidatedState: TestableState, - ::BlockPayload: TestableBlock, - Leaf: TestableLeaf, -{ - if !validator_config.is_da { - return None; - } - - let advertise_urls: Vec; - let bind_address: Url; - - match args.builder_address { - None => { - let port = portpicker::pick_unused_port().expect("Failed to pick an unused port"); - advertise_urls = local_ip_address::list_afinet_netifas() - .expect("Couldn't get list of local IP addresses") - .into_iter() - .map(|(_name, ip)| ip) - .filter(|ip| !ip.is_loopback()) - .map(|ip| match ip { - IpAddr::V4(addr) => Url::parse(&format!("http://{addr}:{port}")).unwrap(), - IpAddr::V6(addr) => Url::parse(&format!("http://[{addr}]:{port}")).unwrap(), - }) - .collect(); - bind_address = Url::parse(&format!("http://0.0.0.0:{port}")).unwrap(); - } - Some(ref addr) => { - bind_address = Url::parse(&format!("http://{addr}")).expect("Valid URL"); - advertise_urls = vec![bind_address.clone()]; - } - } - - match run_config.builder { - BuilderType::External => None, - BuilderType::Random => { - let builder_task = - >::start( - run_config.config.num_nodes_with_stake.into(), - bind_address, - run_config.random_builder.clone().unwrap_or_default(), - HashMap::new(), - ) - .await; - - orchestrator_client - .post_builder_addresses(advertise_urls) - .await; - - Some(builder_task) - } - BuilderType::Simple => { - let builder_task = - >::start( - run_config.config.num_nodes_with_stake.into(), - bind_address, - (), - HashMap::new(), - ) - .await; - - orchestrator_client - .post_builder_addresses(advertise_urls) - .await; - - Some(builder_task) - } - } -} - -/// Base port for validator -pub const VALIDATOR_BASE_PORT: u16 = 8000; -/// Base port for builder -pub const BUILDER_BASE_PORT: u16 = 9000; - -/// Generate a local address for node with index `node_index`, offsetting from port `BASE_PORT`. -/// # Panics -/// If `node_index` is too large to fit in a `u16` -#[must_use] -pub fn gen_local_address(node_index: usize) -> SocketAddr { - SocketAddr::new( - IpAddr::V4(Ipv4Addr::LOCALHOST), - BASE_PORT + (u16::try_from(node_index).expect("node index too large")), - ) -} diff --git a/crates/examples/libp2p/all.rs b/crates/examples/libp2p/all.rs deleted file mode 100644 index 4fd99cd0e8..0000000000 --- a/crates/examples/libp2p/all.rs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! An example program using libp2p -/// types used for this example -pub mod types; - -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use infra::{gen_local_address, BUILDER_BASE_PORT, VALIDATOR_BASE_PORT}; -use tokio::spawn; -use tracing::instrument; - -use crate::{ - infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}, - types::{Network, NodeImpl, ThisRun}, -}; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - // use configfile args - let (config, orchestrator_url) = read_orchestrator_init_config::(); - - // orchestrator - spawn(run_orchestrator::(OrchestratorArgs { - url: orchestrator_url.clone(), - config: config.clone(), - })); - - // nodes - let mut nodes = Vec::new(); - for i in 0..config.config.num_nodes_with_stake.into() { - // Calculate our libp2p advertise address, which we will later derive the - // bind address from for example purposes. - let advertise_address = gen_local_address::(i); - let builder_address = gen_local_address::(i); - let orchestrator_url = orchestrator_url.clone(); - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs { - url: orchestrator_url, - advertise_address: Some(advertise_address.to_string()), - builder_address: Some(builder_address), - network_config_file: None, - }, - ) - .await; - }); - nodes.push(node); - } - futures::future::join_all(nodes).await; -} diff --git a/crates/examples/libp2p/multi-validator.rs b/crates/examples/libp2p/multi-validator.rs deleted file mode 100644 index 0767245c3b..0000000000 --- a/crates/examples/libp2p/multi-validator.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A multi-validator using libp2p -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::{MultiValidatorArgs, ValidatorArgs}; -use tokio::spawn; -use tracing::instrument; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let args = MultiValidatorArgs::parse(); - tracing::debug!("connecting to orchestrator at {:?}", args.url); - let mut nodes = Vec::new(); - for node_index in 0..args.num_nodes { - let args = args.clone(); - - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs::from_multi_args(args, node_index), - ) - .await; - }); - nodes.push(node); - } - let _result = futures::future::join_all(nodes).await; -} diff --git a/crates/examples/libp2p/types.rs b/crates/examples/libp2p/types.rs deleted file mode 100644 index ed8fbcda6f..0000000000 --- a/crates/examples/libp2p/types.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -use std::fmt::Debug; - -use hotshot::traits::implementations::Libp2pNetwork; -use hotshot_example_types::{ - auction_results_provider_types::TestAuctionResultsProvider, state_types::TestTypes, - storage_types::TestStorage, -}; -use hotshot_types::traits::node_implementation::NodeImplementation; -use serde::{Deserialize, Serialize}; - -use crate::infra::Libp2pDaRun; - -/// dummy struct so we can choose types -#[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] -pub struct NodeImpl {} - -/// Convenience type alias -pub type Network = Libp2pNetwork; - -impl NodeImplementation for NodeImpl { - type Network = Network; - type Storage = TestStorage; - type AuctionResultsProvider = TestAuctionResultsProvider; -} -/// convenience type alias -pub type ThisRun = Libp2pDaRun; diff --git a/crates/examples/libp2p/validator.rs b/crates/examples/libp2p/validator.rs deleted file mode 100644 index c85e52688e..0000000000 --- a/crates/examples/libp2p/validator.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A validator using libp2p - -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use local_ip_address::local_ip; -use tracing::{debug, instrument}; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let mut args = ValidatorArgs::parse(); - - // If we did not set the advertise address, use our local IP and port 8000 - let local_ip = local_ip().expect("failed to get local IP"); - args.advertise_address = Some(args.advertise_address.unwrap_or(format!("{local_ip}:8000"))); - - debug!("connecting to orchestrator at {:?}", args.url); - infra::main_entry_point::(args).await; -} diff --git a/crates/examples/orchestrator.rs b/crates/examples/orchestrator.rs deleted file mode 100644 index 42ea1d9014..0000000000 --- a/crates/examples/orchestrator.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! An orchestrator - -use hotshot::helpers::initialize_logging; -use hotshot_example_types::state_types::TestTypes; -use tracing::instrument; - -use crate::infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}; - -/// general infra used for this example -#[path = "./infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let (config, orchestrator_url) = read_orchestrator_init_config::(); - run_orchestrator::(OrchestratorArgs:: { - url: orchestrator_url.clone(), - config: config.clone(), - }) - .await; -} diff --git a/crates/examples/process-compose.yaml b/crates/examples/process-compose.yaml new file mode 100644 index 0000000000..1df6494df1 --- /dev/null +++ b/crates/examples/process-compose.yaml @@ -0,0 +1,86 @@ +# This file is used to run all components of the multi-process examples (in combined network mode). +# If you want to just run them all in one process, you can use `cargo run --example all`. + +# To run this, do `process-compose up` + +# NOTE: You will need Docker installed to run this, as we use KeyDB as a database for the CDN for now + +version: "3" + +processes: + # The coordinator is used to assign unique indices to nodes when running benchmarks on multiple machines + # (or in this case, multiple processes). It is also used to share Libp2p addresses with other nodes + # so they can bootstrap to each other. + coordinator: + command: RUST_LOG=info cargo run --example coordinator + readiness_probe: + exec: + command: nc -zv localhost 3030 + period_seconds: 5 + timeout_seconds: 4 + failure_threshold: 20 + + # We use KeyDB (a Redis variant) to maintain consistency between + # different parts of the CDN + # Cheating a bit here too, but KeyDB is not available as a Nix package. + # Could do local (SQLite) discovery, but removes some of the spirit + # from the local demo. + keydb: + command: docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb --requirepass changeme! + readiness_probe: + exec: + command: nc -zv localhost 6379 + period_seconds: 5 + timeout_seconds: 4 + failure_threshold: 20 + + # The marshal is the CDN component that is responsible for authenticating users and + # pointing them to the correct broker. + marshal: + command: RUST_LOG=info cargo run --example cdn-marshal -- --discovery-endpoint "redis://:changeme!@localhost:6379" + depends_on: + keydb: + condition: process_healthy + + # The `broker` is the CDN component that is primarily responsible for routing messages + # between nodes. + broker: + command: RUST_LOG=info,libp2p=off cargo run --example cdn-broker -- --discovery-endpoint "redis://:changeme!@localhost:6379" + depends_on: + keydb: + condition: process_healthy + + # `hotshot1` is a single HotShot node + hotshot1: + command: RUST_LOG=info,libp2p=off cargo run --example single-validator -- --network combined --libp2p-port 3000 --total-num-nodes 5 --num-da-nodes 3 --marshal-address localhost:1737 + depends_on: + coordinator: + condition: process_healthy + + # `hotshot2` is a second HotShot node + hotshot2: + command: RUST_LOG=info,libp2p=off cargo run --example single-validator -- --network combined --libp2p-port 3001 --total-num-nodes 5 --num-da-nodes 3 --marshal-address localhost:1737 + depends_on: + coordinator: + condition: process_healthy + + # `hotshot3` is a third HotShot node + hotshot3: + command: RUST_LOG=info,libp2p=off cargo run --example single-validator -- --network combined --libp2p-port 3002 --total-num-nodes 5 --num-da-nodes 3 --marshal-address localhost:1737 + depends_on: + coordinator: + condition: process_healthy + + # `hotshot4` is a fourth HotShot node + hotshot4: + command: RUST_LOG=info,libp2p=off cargo run --example single-validator -- --network combined --libp2p-port 3003 --total-num-nodes 5 --num-da-nodes 3 --marshal-address localhost:1737 + depends_on: + coordinator: + condition: process_healthy + + # `hotshot5` is a fifth HotShot node + hotshot5: + command: RUST_LOG=info,libp2p=off cargo run --example single-validator -- --network combined --libp2p-port 3004 --total-num-nodes 5 --num-da-nodes 3 --marshal-address localhost:1737 + depends_on: + coordinator: + condition: process_healthy diff --git a/crates/examples/push-cdn/README.md b/crates/examples/push-cdn/README.md deleted file mode 100644 index a20e46f4ce..0000000000 --- a/crates/examples/push-cdn/README.md +++ /dev/null @@ -1,68 +0,0 @@ -Steps ---------------- - -KeyDB is the ephemeral database, it's like Redis but with extra features. The only thing we run it with is with `--requirepass` to set a password. - -**Marshals:** -The marshal is the entry point of the push CDN, all users connect there first. It tells users which broker to connect to. - -- `-d` is the "discovery endpoint", which in this case is the URL of KeyDB. -- `-b` is the bind port. This is what you would set in run_config.toml for cdn_broker_marshal_endpoint -- `-m` is metrics stuff. You shouldn't have to use that - - -**Brokers:** -In a run with multiple machines, we want two brokers. With one machine, it's probably fine to do one broker. These are what route the messages. Here are the relevant command line arguments: - -- `-d` is the "discovery endpoint", which in this case is the URL of KeyDB. -- `--public-bind-endpoint`: the endpoint which we bind to locally for users to connect to (e.g. 0.0.0.0:1740) -- `--public-advertise-endpoint`: the endpoint which we advertise to users (e.g. my.public.ip:1740) -- `--private-bind-endpoint`: the endpoint which we bind to locally for brokers to connect to (e.g. 0.0.0.0:1741) -- `--private-advertise-endpoint`: the endpoint which we advertise to brokers (e.g. my.public.ip:1741) -- `-m` is metrics stuff. You shouldn't have to use that -For brokers, there is a magic value called `local_ip`. This resolves to the local IP address, which skips the need for talking to the AWS metadata server. For in-AWS uses, the following configuration is probably fine: -`cdn-broker --public-bind-endpoint 0.0.0.0:1740 --public-advertise-endpoint local_ip:1740 --private-bind-endpoint 0.0.0.0:1741 --private-advertise-endpoint local_ip:1741`. You won't need to put this port or values anywhere, as the marshal does everything for you. - -Examples: ---------------- - -**Run Locally** - -`just example all-push-cdn -- --config_file ./crates/orchestrator/run-config.toml` - -OR - -``` -docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb -just example cdn-marshal -- -d redis://localhost:6379 -b 9000 -just example cdn-broker -- -d redis://localhost:6379 --public-bind-endpoint 0.0.0.0:1740 --public-advertise-endpoint local_ip:1740 --private-bind-endpoint 0.0.0.0:1741 --private-advertise-endpoint local_ip:1741 -just example orchestrator -- --config_file ./crates/orchestrator/run-config.toml --orchestrator_url http://0.0.0.0:4444 -just example multi-validator-push-cdn -- 10 http://127.0.0.1:4444 -``` - -**Run with GPU-VID** -``` -docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb -just example cdn-marshal -- -d redis://localhost:6379 -b 9000 -just example cdn-broker -- -d redis://localhost:6379 --public-bind-endpoint 0.0.0.0:1740 --public-advertise-endpoint local_ip:1740 --private-bind-endpoint 0.0.0.0:1741 --private-advertise-endpoint local_ip:1741 -just example_fixed_leader orchestrator -- --config_file ./crates/orchestrator/run-config.toml --orchestrator_url http://0.0.0.0:4444 --fixed_leader_for_gpuvid 1 -just example_gpuvid_leader multi-validator-push-cdn -- 1 http://127.0.0.1:4444 -sleep 1m -just example_fixed_leader multi-validator-push-cdn -- 9 http://127.0.0.1:4444 -``` - -Where ones using `example_gpuvid_leader` could be the leader and should be running on an nvidia GPU, and other validators using `example_fixed_leader` will never be a leader. In practice, these url should be changed to the corresponding ip and port. - - -If you don't have a gpu but want to test out fixed leader, you can run: -``` -docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb -just example cdn-marshal -- -d redis://localhost:6379 -b 9000 -just example cdn-broker -- -d redis://localhost:6379 --public-bind-endpoint 0.0.0.0:1740 --public-advertise-endpoint local_ip:1740 --private-bind-endpoint 0.0.0.0:1741 --private-advertise-endpoint local_ip:1741 -just example_fixed_leader orchestrator -- --config_file ./crates/orchestrator/run-config.toml --orchestrator_url http://0.0.0.0:4444 --fixed_leader_for_gpuvid 1 -just example_fixed_leader multi-validator-push-cdn -- 1 http://127.0.0.1:4444 -sleep 1m -just example_fixed_leader multi-validator-push-cdn -- 9 http://127.0.0.1:4444 -``` - -Remember, you have to run leaders first, then other validators, so that leaders will have lower index. diff --git a/crates/examples/push-cdn/all.rs b/crates/examples/push-cdn/all.rs deleted file mode 100644 index 444edc60fe..0000000000 --- a/crates/examples/push-cdn/all.rs +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! An example program using the Push CDN -/// The types we're importing -pub mod types; - -use std::path::Path; - -use cdn_broker::{ - reexports::{crypto::signature::KeyPair, def::hook::NoMessageHook}, - Broker, -}; -use cdn_marshal::Marshal; -use hotshot::{ - helpers::initialize_logging, - traits::implementations::{TestingDef, WrappedSignatureKey}, - types::SignatureKey, -}; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use hotshot_types::traits::node_implementation::NodeType; -use infra::{gen_local_address, BUILDER_BASE_PORT}; -use rand::{rngs::StdRng, RngCore, SeedableRng}; -use tokio::spawn; - -use crate::{ - infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}, - types::{Network, NodeImpl, ThisRun}, -}; - -/// The infra implementation -#[path = "../infra/mod.rs"] -pub mod infra; - -use tracing::error; - -#[tokio::main] -async fn main() { - // Initialize logging - initialize_logging(); - - // use configfile args - let (config, orchestrator_url) = read_orchestrator_init_config::(); - - // Start the orhcestrator - spawn(run_orchestrator::(OrchestratorArgs { - url: orchestrator_url.clone(), - config: config.clone(), - })); - - // The configuration we are using for this example is 2 brokers & 1 marshal - - // A keypair shared between brokers - let (broker_public_key, broker_private_key) = - ::SignatureKey::generated_from_seed_indexed([0u8; 32], 1337); - - // Get the OS temporary directory - let temp_dir = std::env::temp_dir(); - - // Create an SQLite file inside of the temporary directory - let discovery_endpoint = temp_dir - .join(Path::new(&format!( - "test-{}.sqlite", - StdRng::from_entropy().next_u64() - ))) - .to_string_lossy() - .into_owned(); - - // 2 brokers - for _ in 0..2 { - // Get the ports to bind to - let private_port = portpicker::pick_unused_port().expect("could not find an open port"); - let public_port = portpicker::pick_unused_port().expect("could not find an open port"); - - // Extrapolate addresses - let private_address = format!("127.0.0.1:{private_port}"); - let public_address = format!("127.0.0.1:{public_port}"); - - let config: cdn_broker::Config::SignatureKey>> = - cdn_broker::Config { - discovery_endpoint: discovery_endpoint.clone(), - public_advertise_endpoint: public_address.clone(), - public_bind_endpoint: public_address, - private_advertise_endpoint: private_address.clone(), - private_bind_endpoint: private_address, - - keypair: KeyPair { - public_key: WrappedSignatureKey(broker_public_key), - private_key: broker_private_key.clone(), - }, - - user_message_hook: NoMessageHook, - broker_message_hook: NoMessageHook, - - metrics_bind_endpoint: None, - ca_cert_path: None, - ca_key_path: None, - global_memory_pool_size: Some(1024 * 1024 * 1024), - }; - - // Create and spawn the broker - spawn(async move { - let broker: Broker::SignatureKey>> = - Broker::new(config).await.expect("broker failed to start"); - - // Error if we stopped unexpectedly - if let Err(err) = broker.start().await { - error!("broker stopped: {err}"); - } - }); - } - - // Get the port to use for the marshal - let marshal_endpoint = config - .cdn_marshal_address - .clone() - .expect("CDN marshal address must be specified"); - - // Configure the marshal - let marshal_config = cdn_marshal::Config { - bind_endpoint: marshal_endpoint.clone(), - discovery_endpoint, - metrics_bind_endpoint: None, - ca_cert_path: None, - ca_key_path: None, - global_memory_pool_size: Some(1024 * 1024 * 1024), - }; - - // Spawn the marshal - spawn(async move { - let marshal: Marshal::SignatureKey>> = - Marshal::new(marshal_config) - .await - .expect("failed to spawn marshal"); - - // Error if we stopped unexpectedly - if let Err(err) = marshal.start().await { - error!("broker stopped: {err}"); - } - }); - - // Start the proper number of nodes - let mut nodes = Vec::new(); - for i in 0..(config.config.num_nodes_with_stake.get()) { - let orchestrator_url = orchestrator_url.clone(); - let builder_address = gen_local_address::(i); - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs { - url: orchestrator_url, - advertise_address: None, - builder_address: Some(builder_address), - network_config_file: None, - }, - ) - .await; - }); - nodes.push(node); - } - let _result = futures::future::join_all(nodes).await; -} diff --git a/crates/examples/push-cdn/multi-validator.rs b/crates/examples/push-cdn/multi-validator.rs deleted file mode 100644 index 54718468b3..0000000000 --- a/crates/examples/push-cdn/multi-validator.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A multi validator -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::{MultiValidatorArgs, ValidatorArgs}; -use tokio::spawn; -use tracing::instrument; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let args = MultiValidatorArgs::parse(); - tracing::debug!("connecting to orchestrator at {:?}", args.url); - let mut nodes = Vec::new(); - for node_index in 0..args.num_nodes { - let args = args.clone(); - - let node = spawn(async move { - infra::main_entry_point::( - ValidatorArgs::from_multi_args(args, node_index), - ) - .await; - }); - nodes.push(node); - } - let _result = futures::future::join_all(nodes).await; -} diff --git a/crates/examples/push-cdn/types.rs b/crates/examples/push-cdn/types.rs deleted file mode 100644 index 8803c72152..0000000000 --- a/crates/examples/push-cdn/types.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -use hotshot::traits::{implementations::PushCdnNetwork, NodeImplementation}; -use hotshot_example_types::{ - auction_results_provider_types::TestAuctionResultsProvider, state_types::TestTypes, - storage_types::TestStorage, -}; -use hotshot_types::traits::node_implementation::NodeType; -use serde::{Deserialize, Serialize}; - -use crate::infra::PushCdnDaRun; - -#[derive(Clone, Deserialize, Serialize, Hash, PartialEq, Eq)] -/// Convenience type alias -pub struct NodeImpl {} - -/// Convenience type alias -pub type Network = PushCdnNetwork<::SignatureKey>; - -impl NodeImplementation for NodeImpl { - type Network = Network; - type Storage = TestStorage; - type AuctionResultsProvider = TestAuctionResultsProvider; -} - -/// Convenience type alias -pub type ThisRun = PushCdnDaRun; diff --git a/crates/examples/push-cdn/validator.rs b/crates/examples/push-cdn/validator.rs deleted file mode 100644 index 7b546dfabe..0000000000 --- a/crates/examples/push-cdn/validator.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! A validator -use clap::Parser; -use hotshot::helpers::initialize_logging; -use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; -use hotshot_orchestrator::client::ValidatorArgs; -use tracing::{debug, instrument}; - -use crate::types::{Network, NodeImpl, ThisRun}; - -/// types used for this example -pub mod types; - -/// general infra used for this example -#[path = "../infra/mod.rs"] -pub mod infra; - -#[tokio::main] -#[instrument] -async fn main() { - // Initialize logging - initialize_logging(); - - let args = ValidatorArgs::parse(); - debug!("connecting to orchestrator at {:?}", args.url); - infra::main_entry_point::(args).await; -} diff --git a/crates/examples/push-cdn/whitelist-adapter.rs b/crates/examples/push-cdn/whitelist-adapter.rs deleted file mode 100644 index e855a41aba..0000000000 --- a/crates/examples/push-cdn/whitelist-adapter.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! The whitelist is an adaptor that is able to update the allowed public keys for -//! all brokers. Right now, we do this by asking the orchestrator for the list of -//! allowed public keys. In the future, we will pull the stake table from the L1. - -use std::{str::FromStr, sync::Arc}; - -use anyhow::{Context, Result}; -use cdn_broker::reexports::discovery::{DiscoveryClient, Embedded, Redis}; -use clap::Parser; -use hotshot_example_types::node_types::TestTypes; -use hotshot_orchestrator::client::OrchestratorClient; -use hotshot_types::{ - network::NetworkConfig, - traits::{node_implementation::NodeType, signature_key::SignatureKey}, -}; -use surf_disco::Url; - -#[derive(Parser, Debug)] -#[command(author, version, about, long_about = None)] -/// The main component of the push CDN. -struct Args { - /// The discovery client endpoint (including scheme) to connect to. - /// With the local discovery feature, this is a file path. - /// With the remote (redis) discovery feature, this is a redis URL (e.g. `redis://127.0.0.1:6789`). - #[arg(short, long)] - discovery_endpoint: String, - - /// The URL the orchestrator is running on. This should be something like `http://localhost:5555` - #[arg(short, long)] - orchestrator_url: String, - - /// Whether or not to use the local discovery client - #[arg(short, long)] - local_discovery: bool, -} - -#[tokio::main] -async fn main() -> Result<()> { - // Parse the command line arguments - let args = Args::parse(); - - // Initialize tracing - tracing_subscriber::fmt::init(); - - // Create a new `OrchestratorClient` from the supplied URL - let orchestrator_client = OrchestratorClient::new( - Url::from_str(&args.orchestrator_url).with_context(|| "Invalid URL")?, - ); - - // Attempt to get the config from the orchestrator. - // Loops internally until the config is received. - let config: NetworkConfig<::SignatureKey> = - orchestrator_client.get_config_after_collection().await; - - tracing::info!("Received config from orchestrator"); - - // Extrapolate the state_ver_keys from the config and convert them to a compatible format - let whitelist = config - .config - .known_nodes_with_stake - .iter() - .map(|k| Arc::from(k.stake_table_entry.stake_key.to_bytes())) - .collect(); - - if args.local_discovery { - ::new(args.discovery_endpoint, None) - .await? - .set_whitelist(whitelist) - .await?; - } else { - ::new(args.discovery_endpoint, None) - .await? - .set_whitelist(whitelist) - .await?; - } - - tracing::info!("Posted config to discovery endpoint"); - - Ok(()) -} diff --git a/crates/examples/single-validator.rs b/crates/examples/single-validator.rs new file mode 100644 index 0000000000..10a6e285ba --- /dev/null +++ b/crates/examples/single-validator.rs @@ -0,0 +1,358 @@ +//! This is meant to be run externally, e.g. when running benchmarks on the protocol. +//! If you just want to run everything required, you can use the `all` example +//! +//! This example runs a single validator node along with a simple builder (since the +//! real builder is in an upstream repo) + +use std::{ + collections::HashMap, net::IpAddr, num::NonZero, str::FromStr, sync::Arc, time::Duration, +}; + +use anyhow::{Context, Result}; +use clap::Parser; +use futures::StreamExt; +use hotshot::{ + helpers::initialize_logging, + traits::{ + election::static_committee::StaticCommittee, + implementations::{ + derive_libp2p_keypair, CdnMetricsValue, CdnTopic, KeyPair, Libp2pMetricsValue, + Libp2pNetwork, PushCdnNetwork, WrappedSignatureKey, + }, + }, + types::{BLSPrivKey, BLSPubKey, EventType, SignatureKey}, + MarketplaceConfig, SystemContext, +}; +use hotshot_example_types::{ + auction_results_provider_types::TestAuctionResultsProvider, + block_types::TestTransaction, + node_types::{CombinedImpl, Libp2pImpl, PushCdnImpl, TestTypes, TestVersions}, + state_types::TestInstanceState, + storage_types::TestStorage, + testable_delay::DelayConfig, +}; +use hotshot_testing::block_builder::{SimpleBuilderImplementation, TestBuilderImplementation}; +use hotshot_types::{ + consensus::ConsensusMetricsValue, traits::election::Membership, HotShotConfig, +}; +use libp2p::Multiaddr; +use libp2p_networking::network::{ + behaviours::dht::record::{Namespace, RecordKey, RecordValue}, + node::config::{KademliaConfig, Libp2pConfig}, + GossipConfig, RequestResponseConfig, +}; +use lru::LruCache; +use rand::Rng; +use tracing::info; +use url::Url; + +// Include some common code +include!("common.rs"); + +/// This example runs a single validator node +#[derive(Parser)] +struct Args { + /// The coordinator address to connect to. The coordinator is just used to tell other nodes + /// about the libp2p bootstrap addresses and give each one a unique index + #[arg(long, default_value = "http://127.0.0.1:3030")] + coordinator_address: String, + + /// The marshal's endpoint to use. Only required if the network type is "cdn" or "combined" + #[arg(long)] + marshal_address: Option, + + /// The source of the public IP address to use. This is used to generate the libp2p + /// bootstrap addresses. Acceptable values are "ipify", "local", "localhost", "aws-local", or + /// "aws-public" + #[arg(long, default_value = "localhost")] + ip_source: String, + + /// The port to use for Libp2p + #[arg(long, default_value_t = 3000)] + libp2p_port: u16, + + /// The number of nodes in the network. This needs to be the same between all nodes + #[arg(long, default_value_t = 5)] + total_num_nodes: usize, + + /// The number of nodes which are DA. This needs to be the same between all nodes + #[arg(long, default_value_t = 3)] + num_da_nodes: usize, + + /// The number of views to run for. If not specified, it will run indefinitely + #[arg(long)] + num_views: Option, + + /// The number of transactions to submit to each nodes' builder per view + #[arg(long, default_value_t = 1)] + num_transactions_per_view: usize, + + /// The size of the transactions submitted to each nodes' builder per view + #[arg(long, default_value_t = 1000)] + transaction_size: usize, + + /// The type of network to use. Acceptable values are + /// "combined", "cdn", or "libp2p" + #[arg(long, default_value = "combined")] + network: String, +} + +/// Get the IP address to use based on the source +async fn get_public_ip_address(source: &str) -> Result { + // Get the IP to use based on the source + Ok(match source.to_lowercase().as_str() { + "ipify" => reqwest::get("https://api.ipify.org") + .await? + .text() + .await? + .parse::() + .with_context(|| "Failed to parse IP address from IPify")?, + "local" => { + local_ip_address::local_ip().with_context(|| "Failed to get local IP address")? + } + "localhost" => "127.0.0.1".parse::().unwrap(), + "aws-local" => reqwest::get("http://169.254.169.254/latest/meta-data/local-ipv4") + .await? + .text() + .await? + .parse::() + .with_context(|| "Failed to parse IP address from AWS local")?, + "aws-public" => reqwest::get("http://169.254.169.254/latest/meta-data/public-ipv4") + .await? + .text() + .await? + .parse::() + .with_context(|| "Failed to parse IP address from AWS public")?, + _ => { + anyhow::bail!( + "Invalid public IP source. Please use one of 'ipify', 'local', 'localhost', 'aws-local', or 'aws-public'." + ) + } + }) +} + +#[tokio::main] +#[allow(clippy::too_many_lines)] +async fn main() -> Result<()> { + // Initialize logging + initialize_logging(); + + // Parse the command-line arguments + let args = Args::parse(); + + // Match the network type + let network_type = match args.network.to_lowercase().as_str() { + "combined" => NetworkType::Combined, + "cdn" => NetworkType::Cdn, + "libp2p" => NetworkType::LibP2P, + _ => { + anyhow::bail!("Invalid network type. Please use one of 'combined', 'cdn', or 'libp2p'.") + } + }; + + // Get the IP address to use for Libp2p based on the source + let libp2p_ip = get_public_ip_address(&args.ip_source).await?; + info!("Using Libp2p address: {}:{}", libp2p_ip, args.libp2p_port); + + // Create a new instance state + let instance_state = TestInstanceState::new(DelayConfig::default()); + + // Initialize HotShot from genesis + let hotshot_initializer = + hotshot::HotShotInitializer::::from_genesis::(instance_state) + .await + .with_context(|| "Failed to initialize HotShot")?; + + // Get our index from the coordinator + let index = reqwest::get(format!("{}/index", &args.coordinator_address).as_str()) + .await? + .text() + .await? + .parse::() + .with_context(|| "Failed to parse index from coordinator")?; + + // Derive our keypair from the index we got + let (public_key, private_key) = BLSPubKey::generated_from_seed_indexed([0u8; 32], index as u64); + + // Derive our libp2p keypair from the private key + let peer_libp2p_keypair = derive_libp2p_keypair::(&private_key) + .with_context(|| "Failed to derive libp2p keypair")?; + + // Generate our advertise Multiaddr + let advertise_multiaddr = format!( + "/ip4/{}/udp/{}/quic-v1/p2p/{}", + libp2p_ip, + args.libp2p_port, + peer_libp2p_keypair.public().to_peer_id() + ); + + // Generate our bind Multiaddr + let bind_multiaddr = + Multiaddr::from_str(&format!("/ip4/0.0.0.0/udp/{}/quic-v1", args.libp2p_port)) + .with_context(|| "Failed to parse bind Multiaddr")?; + + // Post our advertise libp2p address to the coordinator + reqwest::Client::new() + .post(format!("{}/libp2p-info", &args.coordinator_address).as_str()) + .body(advertise_multiaddr) + .send() + .await?; + + // Get the other libp2p addresses from the coordinator + let known_libp2p_nodes = + reqwest::get(format!("{}/libp2p-info", &args.coordinator_address).as_str()) + .await? + .text() + .await? + .split('\n') + .map(|s| { + s.parse::() + .with_context(|| "Failed to parse Libp2p bootstrap address") + }) + .collect::>>()?; + + // Print the known libp2p nodes + info!("Known libp2p nodes: {:?}", known_libp2p_nodes); + + // Generate the builder URL we plan to use + let builder_url = Url::parse( + format!( + "http://localhost:{}", + portpicker::pick_unused_port().with_context(|| "Failed to find unused port")? + ) + .as_str(), + ) + .with_context(|| "Failed to parse builder URL")?; + + // Create the known nodes up to the total number of nodes + let known_nodes: Vec<_> = (0..args.total_num_nodes) + .map(peer_info_from_index) + .collect(); + let known_da_nodes: Vec<_> = (0..args.num_da_nodes) + .filter(|i| is_da_node(*i, args.num_da_nodes)) + .map(peer_info_from_index) + .collect(); + + // Create the memberships from the known nodes and known da nodes + let memberships = StaticCommittee::new(known_nodes.clone(), known_da_nodes.clone()); + + // Configure HotShot + let config = HotShotConfig:: { + known_nodes: known_nodes.clone(), + known_da_nodes: known_da_nodes.clone(), + next_view_timeout: 5000, + fixed_leader_for_gpuvid: 0, // This means that we don't have a fixed leader for testing GPU VID + view_sync_timeout: Duration::from_secs(5), + builder_timeout: Duration::from_secs(1), + data_request_delay: Duration::from_millis(200), + builder_urls: vec![builder_url.clone()], + start_proposing_view: u64::MAX, // These just mean the upgrade functionality is disabled + stop_proposing_view: u64::MAX, + start_voting_view: u64::MAX, + stop_voting_view: u64::MAX, + start_proposing_time: u64::MAX, + stop_proposing_time: u64::MAX, + start_voting_time: u64::MAX, + stop_voting_time: u64::MAX, + epoch_height: 0, // This just means epochs aren't enabled + }; + + // Create the network and start consensus + match network_type { + NetworkType::Cdn => { + // Create the network + let network = new_cdn_network( + args.marshal_address, + is_da_node(index, args.num_da_nodes), + &public_key, + &private_key, + ) + .with_context(|| "Failed to create CDN network")?; + + // Start consensus + let join_handle = start_consensus::( + public_key, + private_key, + config, + memberships, + network, + hotshot_initializer, + args.total_num_nodes, + builder_url, + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?; + + // Wait for consensus to finish + join_handle.await?; + } + NetworkType::LibP2P => { + // Create the network + let network = new_libp2p_network( + bind_multiaddr, + args.total_num_nodes, + &public_key, + &private_key, + &known_libp2p_nodes, + ) + .await + .with_context(|| "Failed to create libp2p network")?; + + // Start consensus + let join_handle = start_consensus::( + public_key, + private_key, + config, + memberships, + network, + hotshot_initializer, + args.total_num_nodes, + builder_url, + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?; + + // Wait for consensus to finish + join_handle.await?; + } + NetworkType::Combined => { + // Create the network + let network = new_combined_network( + args.marshal_address, + bind_multiaddr, + args.total_num_nodes, + &known_libp2p_nodes, + is_da_node(index, args.num_da_nodes), + &public_key, + &private_key, + ) + .await + .with_context(|| "Failed to create combined network")?; + + // Start consensus + let join_handle = start_consensus::( + public_key, + private_key, + config, + memberships, + network, + hotshot_initializer, + args.total_num_nodes, + builder_url, + args.num_transactions_per_view, + args.transaction_size, + args.num_views, + ) + .await?; + + // Wait for consensus to finish + join_handle.await?; + } + }; + + Ok(()) +} diff --git a/crates/hotshot/Cargo.toml b/crates/hotshot/Cargo.toml index a74bc74746..be270cdced 100644 --- a/crates/hotshot/Cargo.toml +++ b/crates/hotshot/Cargo.toml @@ -23,7 +23,6 @@ anyhow = { workspace = true } async-broadcast = { workspace = true } async-lock = { workspace = true } async-trait = { workspace = true } -bimap = "0.6" bincode = { workspace = true } blake3 = { workspace = true } cdn-broker = { workspace = true, features = ["global-permits"] } @@ -42,12 +41,11 @@ libp2p-identity = { workspace = true } libp2p-networking = { workspace = true } lru = { workspace = true } num_enum = "0.7" -parking_lot = "0.12" +parking_lot = { workspace = true } portpicker = "0.1" primitive-types = { workspace = true } rand = { workspace = true } serde = { workspace = true, features = ["rc"] } -sha2 = { workspace = true } time = { workspace = true } tokio = { workspace = true } diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index 8055ae595c..cd091b8c67 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -32,7 +32,6 @@ pub mod helpers; use std::{ collections::{BTreeMap, HashMap}, - num::NonZeroUsize, sync::Arc, time::Duration, }; @@ -141,9 +140,6 @@ pub struct SystemContext, V: Versi InactiveReceiver>>, ), - /// uid for instrumentation - pub id: u64, - /// Reference to the internal storage for consensus datum. pub storage: Arc>, @@ -173,7 +169,6 @@ impl, V: Versions> Clone external_event_stream: self.external_event_stream.clone(), anchored_leaf: self.anchored_leaf.clone(), internal_event_stream: self.internal_event_stream.clone(), - id: self.id, storage: Arc::clone(&self.storage), upgrade_lock: self.upgrade_lock.clone(), marketplace_config: self.marketplace_config.clone(), @@ -197,7 +192,6 @@ impl, V: Versions> SystemContext::PrivateKey, - nonce: u64, config: HotShotConfig, memberships: Arc>, network: Arc, @@ -226,7 +220,6 @@ impl, V: Versions> SystemContext, V: Versions> SystemContext::PrivateKey, - nonce: u64, config: HotShotConfig, memberships: Arc>, network: Arc, @@ -356,7 +348,6 @@ impl, V: Versions> SystemContext> = Arc::new(SystemContext { - id: nonce, consensus: OuterConsensus::new(consensus), instance_state: Arc::new(instance_state), public_key, @@ -383,7 +374,7 @@ impl, V: Versions> SystemContext, V: Versions> SystemContext, V: Versions> SystemContext Leaf2 { self.consensus.read().await.decided_leaf() } @@ -572,7 +563,7 @@ impl, V: Versions> SystemContext Option> { self.consensus.try_read().map(|guard| guard.decided_leaf()) } @@ -581,7 +572,7 @@ impl, V: Versions> SystemContext Arc { Arc::clone(&self.consensus.read().await.decided_state()) } @@ -593,7 +584,7 @@ impl, V: Versions> SystemContext Option> { self.consensus.read().await.state(view).cloned() } @@ -615,7 +606,6 @@ impl, V: Versions> SystemContext::PrivateKey, - node_id: u64, config: HotShotConfig, memberships: Arc>, network: Arc, @@ -634,7 +624,6 @@ impl, V: Versions> SystemContext::PrivateKey, - nonce: u64, config: HotShotConfig, memberships: Arc>, network: Arc, @@ -794,7 +782,6 @@ where let left_system_context = SystemContext::new( public_key.clone(), private_key.clone(), - nonce, config.clone(), Arc::clone(&memberships), Arc::clone(&network), @@ -807,7 +794,6 @@ where let right_system_context = SystemContext::new( public_key, private_key, - nonce, config, memberships, network, @@ -961,8 +947,8 @@ impl, V: Versions> TwinsHandlerSta impl, V: Versions> ConsensusApi for SystemContextHandle { - fn total_nodes(&self) -> NonZeroUsize { - self.hotshot.config.num_nodes_with_stake + fn total_nodes(&self) -> usize { + self.hotshot.config.known_nodes.len() } fn builder_timeout(&self) -> Duration { diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index d32a671b94..6de58d5fcf 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -85,7 +85,6 @@ pub fn add_response_task, V: Versi Arc::clone(&handle.memberships), handle.public_key().clone(), handle.private_key().clone(), - handle.hotshot.id, ); handle.network_registry.register(run_response_task::( state, @@ -321,7 +320,6 @@ where &'static mut self, public_key: TYPES::SignatureKey, private_key: ::PrivateKey, - nonce: u64, config: HotShotConfig, memberships: Arc>, network: Arc, @@ -334,7 +332,6 @@ where let hotshot = SystemContext::new( public_key, private_key, - nonce, config, memberships, network, diff --git a/crates/hotshot/src/tasks/task_state.rs b/crates/hotshot/src/tasks/task_state.rs index 9023722a66..2e0cc38460 100644 --- a/crates/hotshot/src/tasks/task_state.rs +++ b/crates/hotshot/src/tasks/task_state.rs @@ -54,7 +54,6 @@ impl, V: Versions> CreateTaskState membership: Arc::clone(&handle.hotshot.memberships), public_key: handle.public_key().clone(), private_key: handle.private_key().clone(), - id: handle.hotshot.id, shutdown_flag: Arc::new(AtomicBool::new(false)), spawned_tasks: BTreeMap::new(), epoch_height: handle.epoch_height, @@ -76,7 +75,6 @@ impl, V: Versions> CreateTaskState vote_collectors: BTreeMap::default(), public_key: handle.public_key().clone(), private_key: handle.private_key().clone(), - id: handle.hotshot.id, start_proposing_view: handle.hotshot.config.start_proposing_view, stop_proposing_view: handle.hotshot.config.stop_proposing_view, start_voting_view: handle.hotshot.config.start_voting_view, @@ -125,7 +123,6 @@ impl, V: Versions> CreateTaskState membership: Arc::clone(&handle.hotshot.memberships), public_key: handle.public_key().clone(), private_key: handle.private_key().clone(), - id: handle.hotshot.id, epoch_height: handle.epoch_height, } } @@ -146,7 +143,6 @@ impl, V: Versions> CreateTaskState vote_collectors: BTreeMap::default(), public_key: handle.public_key().clone(), private_key: handle.private_key().clone(), - id: handle.hotshot.id, storage: Arc::clone(&handle.storage), upgrade_lock: handle.hotshot.upgrade_lock.clone(), } @@ -173,7 +169,6 @@ impl, V: Versions> CreateTaskState commit_relay_map: HashMap::default().into(), finalize_relay_map: HashMap::default().into(), view_sync_timeout: handle.hotshot.config.view_sync_timeout, - id: handle.hotshot.id, last_garbage_collected_view: TYPES::View::new(0), upgrade_lock: handle.hotshot.upgrade_lock.clone(), } @@ -195,7 +190,6 @@ impl, V: Versions> CreateTaskState public_key: handle.public_key().clone(), private_key: handle.private_key().clone(), instance_state: handle.hotshot.instance_state(), - id: handle.hotshot.id, builder_clients: handle .hotshot .config @@ -239,7 +233,6 @@ impl, V: Versions> CreateTaskState membership: Arc::clone(&handle.hotshot.memberships), drb_computation: None, output_event_stream: handle.hotshot.external_event_stream.0.clone(), - id: handle.hotshot.id, storage: Arc::clone(&handle.storage), upgrade_lock: handle.hotshot.upgrade_lock.clone(), epoch_height: handle.hotshot.config.epoch_height, @@ -266,7 +259,6 @@ impl, V: Versions> CreateTaskState private_key: handle.private_key().clone(), storage: Arc::clone(&handle.storage), timeout: handle.hotshot.config.next_view_timeout, - id: handle.hotshot.id, formed_upgrade_certificate: None, upgrade_lock: handle.hotshot.upgrade_lock.clone(), epoch_height: handle.hotshot.config.epoch_height, @@ -293,7 +285,6 @@ impl, V: Versions> CreateTaskState output_event_stream: handle.hotshot.external_event_stream.0.clone(), storage: Arc::clone(&handle.storage), spawned_tasks: BTreeMap::new(), - id: handle.hotshot.id, upgrade_lock: handle.hotshot.upgrade_lock.clone(), epoch_height: handle.hotshot.config.epoch_height, } @@ -323,7 +314,6 @@ impl, V: Versions> CreateTaskState timeout_task: spawn(async {}), timeout: handle.hotshot.config.next_view_timeout, consensus: OuterConsensus::new(consensus), - id: handle.hotshot.id, upgrade_lock: handle.hotshot.upgrade_lock.clone(), epoch_height: handle.hotshot.config.epoch_height, } @@ -337,7 +327,7 @@ impl, V: Versions> CreateTaskState async fn create_from(handle: &SystemContextHandle) -> Self { Self { events: Vec::new(), - id: handle.hotshot.id, + public_key: handle.public_key().clone(), } } } diff --git a/crates/hotshot/src/traits.rs b/crates/hotshot/src/traits.rs index 1ff090fc06..6022efca7e 100644 --- a/crates/hotshot/src/traits.rs +++ b/crates/hotshot/src/traits.rs @@ -10,7 +10,6 @@ mod networking; mod node_implementation; pub use hotshot_types::traits::{BlockPayload, ValidatedState}; -pub use libp2p_networking::network::NetworkNodeConfigBuilder; pub use networking::{NetworkError, NetworkReliability}; pub use node_implementation::{NodeImplementation, TestableNodeImplementation}; @@ -20,7 +19,7 @@ pub mod implementations { combined_network::{CombinedNetworks, UnderlyingCombinedNetworks}, libp2p_network::{ derive_libp2p_keypair, derive_libp2p_multiaddr, derive_libp2p_peer_id, GossipConfig, - Libp2pMetricsValue, Libp2pNetwork, PeerInfoVec, RequestResponseConfig, + Libp2pMetricsValue, Libp2pNetwork, RequestResponseConfig, }, memory_network::{MasterMap, MemoryNetwork}, push_cdn_network::{ diff --git a/crates/hotshot/src/traits/election/randomized_committee.rs b/crates/hotshot/src/traits/election/randomized_committee.rs index 4046123553..3523f45fd8 100644 --- a/crates/hotshot/src/traits/election/randomized_committee.rs +++ b/crates/hotshot/src/traits/election/randomized_committee.rs @@ -16,9 +16,10 @@ use hotshot_types::{ }; use primitive_types::U256; use rand::{rngs::StdRng, Rng}; +use serde::{Deserialize, Serialize}; use utils::anytrace::Result; -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] /// The static committee election diff --git a/crates/hotshot/src/traits/election/randomized_committee_members.rs b/crates/hotshot/src/traits/election/randomized_committee_members.rs index 5c85ad9c07..a979ed1dcc 100644 --- a/crates/hotshot/src/traits/election/randomized_committee_members.rs +++ b/crates/hotshot/src/traits/election/randomized_committee_members.rs @@ -21,11 +21,12 @@ use hotshot_types::{ }; use primitive_types::U256; use rand::{rngs::StdRng, Rng}; +use serde::{Deserialize, Serialize}; use utils::anytrace::Result; use crate::traits::election::helpers::QuorumFilterConfig; -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] /// The static committee election pub struct RandomizedCommitteeMembers { /// The nodes eligible for leadership. diff --git a/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs b/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs index d2635cc273..8883795d9d 100644 --- a/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs +++ b/crates/hotshot/src/traits/election/static_committee_leader_two_views.rs @@ -18,7 +18,6 @@ use primitive_types::U256; use utils::anytrace::Result; #[derive(Clone, Debug, Eq, PartialEq, Hash)] - /// The static committee election pub struct StaticCommitteeLeaderForTwoViews { /// The nodes eligible for leadership. diff --git a/crates/hotshot/src/traits/networking/combined_network.rs b/crates/hotshot/src/traits/networking/combined_network.rs index 7021c3e892..30930c7139 100644 --- a/crates/hotshot/src/traits/networking/combined_network.rs +++ b/crates/hotshot/src/traits/networking/combined_network.rs @@ -255,8 +255,6 @@ pub struct UnderlyingCombinedNetworks( impl TestableNetworkingImplementation for CombinedNetworks { fn generator( expected_node_count: usize, - num_bootstrap: usize, - network_id: usize, da_committee_size: usize, reliability_config: Option>, secondary_network_delay: Duration, @@ -264,16 +262,12 @@ impl TestableNetworkingImplementation for CombinedNetwor let generators = ( as TestableNetworkingImplementation>::generator( expected_node_count, - num_bootstrap, - network_id, da_committee_size, None, Duration::default(), ), as TestableNetworkingImplementation>::generator( expected_node_count, - num_bootstrap, - network_id, da_committee_size, reliability_config, Duration::default(), diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index 39c61e638f..0a8ee6e477 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -7,14 +7,11 @@ //! Libp2p based/production networking implementation //! This module provides a libp2p based networking implementation where each node in the //! network forms a tcp or udp connection to a subset of other nodes in the network -#[cfg(feature = "hotshot-testing")] use std::str::FromStr; use std::{ - cmp::min, - collections::{BTreeSet, HashSet}, + collections::{HashMap, HashSet}, fmt::Debug, net::{IpAddr, ToSocketAddrs}, - num::NonZeroUsize, sync::{ atomic::{AtomicBool, AtomicU64, Ordering}, Arc, @@ -25,7 +22,6 @@ use std::{ use anyhow::{anyhow, Context}; use async_lock::RwLock; use async_trait::async_trait; -use bimap::BiHashMap; use futures::future::join_all; #[cfg(feature = "hotshot-testing")] use hotshot_types::traits::network::{ @@ -35,7 +31,6 @@ use hotshot_types::{ boxed_sync, constants::LOOK_AHEAD, data::ViewNumber, - network::NetworkConfig, traits::{ election::Membership, metrics::{Counter, Gauge, Metrics, NoMetrics}, @@ -45,6 +40,9 @@ use hotshot_types::{ }, BoxSyncFuture, }; +#[cfg(feature = "hotshot-testing")] +use hotshot_types::{light_client::StateVerKey, PeerConfig}; + use libp2p_identity::{ ed25519::{self, SecretKey}, Keypair, PeerId, @@ -53,15 +51,18 @@ pub use libp2p_networking::network::{GossipConfig, RequestResponseConfig}; use libp2p_networking::{ network::{ behaviours::dht::record::{Namespace, RecordKey, RecordValue}, + node::config::Libp2pConfig, spawn_network_node, - transport::construct_auth_message, NetworkEvent::{self, DirectRequest, DirectResponse, GossipMsg}, - NetworkNodeConfig, NetworkNodeConfigBuilder, NetworkNodeHandle, NetworkNodeReceiver, - DEFAULT_REPLICATION_FACTOR, + NetworkNodeHandle, NetworkNodeReceiver, }, reexport::Multiaddr, }; -use rand::{rngs::StdRng, seq::IteratorRandom, SeedableRng}; + +#[cfg(feature = "hotshot-testing")] +use libp2p_networking::network::{node::config::KademliaConfig, transport::construct_auth_message}; +#[cfg(feature = "hotshot-testing")] +use rand::Rng; use serde::Serialize; use tokio::{ select, spawn, @@ -108,12 +109,8 @@ impl Default for Libp2pMetricsValue { } } -/// convenience alias for the type for bootstrap addresses -/// concurrency primitives are needed for having tests -pub type BootstrapAddrs = Arc>>; - -/// hardcoded topic of QC used -pub const QC_TOPIC: &str = "global"; +/// The topic that all nodes subscribe to +pub const GLOBAL_TOPIC: &str = "global"; /// Stubbed out Ack /// @@ -136,25 +133,20 @@ impl Debug for Libp2pNetwork { } } -/// Type alias for a shared collection of peerid, multiaddrs -pub type PeerInfoVec = Arc>>; - /// The underlying state of the libp2p network #[derive(Debug)] struct Libp2pNetworkInner { - /// this node's public key - pk: T::SignatureKey, - /// handle to control the network - handle: Arc>, + /// The HotShot public key + hotshot_public_key: T::SignatureKey, + + /// The handle that we use to send requests to Libp2p + handle: Arc>, /// Message Receiver receiver: Mutex>>, /// Sender for broadcast messages sender: Sender>, /// Sender for node lookup (relevant view number, key of node) (None for shutdown) node_lookup_send: Sender>, - /// this is really cheating to enable local tests - /// hashset of (bootstrap_addr, peer_id) - bootstrap_addrs: PeerInfoVec, /// whether or not the network is ready to send is_ready: Arc, /// max time before dropping message due to DHT error @@ -184,6 +176,37 @@ pub struct Libp2pNetwork { inner: Arc>, } +/// Generate a [testing] multiaddr +#[cfg(feature = "hotshot-testing")] +fn random_multiaddr() -> Multiaddr { + Multiaddr::from_str(&format!( + "/ip4/127.0.0.1/udp/{}/quic-v1", + portpicker::pick_unused_port().expect("All ports are in use") + )) + .expect("Failed to create multiaddr") +} + +/// Generate the expected [testing] multiaddr from a node index +#[cfg(feature = "hotshot-testing")] +fn multiaddr_from_node_index( + i: usize, + bind_addresses: &HashMap, +) -> Multiaddr { + // Generate the node's private key from the node ID + let peers_hotshot_private_key = + T::SignatureKey::generated_from_seed_indexed([0u8; 32], i as u64).1; + + // Derive the Libp2p keypair from the private key + let peers_libp2p_keypair = derive_libp2p_keypair::(&peers_hotshot_private_key) + .expect("Failed to derive libp2p keypair"); + + // Generate the multiaddress using the peer id and port + bind_addresses[&i] + .clone() + .with_p2p(peers_libp2p_keypair.public().to_peer_id()) + .expect("Failed to append libp2p peer id to multiaddr") +} + #[cfg(feature = "hotshot-testing")] impl TestableNetworkingImplementation for Libp2pNetwork { /// Returns a boxed function `f(node_id, public_key) -> Libp2pNetwork` @@ -198,8 +221,6 @@ impl TestableNetworkingImplementation for Libp2pNetwork { #[allow(clippy::panic, clippy::too_many_lines)] fn generator( expected_node_count: usize, - num_bootstrap: usize, - _network_id: usize, da_committee_size: usize, reliability_config: Option>, _secondary_network_delay: Duration, @@ -208,76 +229,116 @@ impl TestableNetworkingImplementation for Libp2pNetwork { da_committee_size <= expected_node_count, "DA committee size must be less than or equal to total # nodes" ); - let bootstrap_addrs: PeerInfoVec = Arc::default(); - let node_ids: Arc>> = Arc::default(); + + // Generate the bind addresses each node will use + let bind_addresses = Arc::new(parking_lot::Mutex::new(HashMap::new())); + for i in 0..expected_node_count { + // Insert a random port for each node + bind_addresses.lock().insert(i, random_multiaddr()); + } // NOTE uncomment this for easier debugging - // let start_port = 5000; Box::pin({ move |node_id| { - info!( - "GENERATOR: Node id {:?}, is bootstrap: {:?}", - node_id, - node_id < num_bootstrap as u64 - ); - - // pick a free, unused UDP port for testing - let port = portpicker::pick_unused_port().expect("Could not find an open port"); - - let addr = - Multiaddr::from_str(&format!("/ip4/127.0.0.1/udp/{port}/quic-v1")).unwrap(); - - // We assign node's public key and stake value rather than read from config file since it's a test - let privkey = T::SignatureKey::generated_from_seed_indexed([0u8; 32], node_id).1; - let pubkey = T::SignatureKey::from_private(&privkey); + // Get and replace our bind address with a newly random port. + // We need this because Libp2p does not release the listener as soon + // as we tell it to. + let mut bind_addresses_lock = bind_addresses.lock(); + let bind_address = bind_addresses_lock + .insert(usize::try_from(node_id).unwrap(), random_multiaddr()) + .unwrap(); + drop(bind_addresses_lock); + + // Deterministically generate the private key from the node ID + let hotshot_private_key = + T::SignatureKey::generated_from_seed_indexed([0u8; 32], node_id).1; + let hotshot_public_key = T::SignatureKey::from_private(&hotshot_private_key); // Derive the Libp2p keypair from the private key - let libp2p_keypair = derive_libp2p_keypair::(&privkey) + let libp2p_keypair = derive_libp2p_keypair::(&hotshot_private_key) .expect("Failed to derive libp2p keypair"); // Sign the lookup record let lookup_record_value = RecordValue::new_signed( - &RecordKey::new(Namespace::Lookup, pubkey.to_bytes()), + &RecordKey::new(Namespace::Lookup, hotshot_public_key.to_bytes()), libp2p_keypair.public().to_peer_id().to_bytes(), - &privkey, + &hotshot_private_key, ) .expect("Failed to sign DHT lookup record"); - // We want at least 2/3 of the nodes to have any given record in the DHT - let replication_factor = - NonZeroUsize::new((2 * expected_node_count).div_ceil(3)).unwrap(); - - // Build the network node configuration - let config = NetworkNodeConfigBuilder::default() - .keypair(libp2p_keypair) - .replication_factor(replication_factor) - .bind_address(Some(addr)) - .to_connect_addrs(HashSet::default()) - .republication_interval(None) - .build() - .expect("Failed to build network node config"); - - let bootstrap_addrs_ref = Arc::clone(&bootstrap_addrs); - let node_ids_ref = Arc::clone(&node_ids); + // Create a random file path for the DHT database + let dht_file_path = + format!("/tmp/libp2p_dht-{}.bin", rand::thread_rng().gen::()); + + // Configure Kademlia with our lookup record value + let kademlia_config = KademliaConfig:: { + // At least 2/3 of the nodes should have any given record + replication_factor: 2 * expected_node_count / 3, + record_ttl: None, + publication_interval: None, + file_path: dht_file_path, + lookup_record_value, + }; + + // Use the default gossip configuration + let gossip_config = GossipConfig::default(); + + // Use the default request response configuration + let request_response_config = RequestResponseConfig::default(); + + // Create and sign an authentication message over our Libp2p `PeerID` + let auth_message = construct_auth_message( + &hotshot_public_key, + &libp2p_keypair.public().to_peer_id(), + &hotshot_private_key, + ) + .expect("Failed to construct authentication message"); + + // Create the list of known peers + let known_peers = (0..expected_node_count) + .map(|i| multiaddr_from_node_index::(i, &bind_addresses.lock().clone())) + .collect(); + + // Collect the `PeerConfig`s of all nodes + let mut all_nodes = Vec::new(); + for i in 0..expected_node_count { + // Calculate the staking key from the node ID + let stake_key = + T::SignatureKey::generated_from_seed_indexed([0u8; 32], i as u64).0; + + // Push the peer config to the list of all nodes + all_nodes.push(PeerConfig { + state_ver_key: StateVerKey::default(), + stake_table_entry: T::SignatureKey::stake_table_entry(&stake_key, 1), + }); + } + + // Create the quorum membership from the list we just made + let quorum_membership = Arc::new(RwLock::new(T::Membership::new( + all_nodes.clone(), + all_nodes, + ))); + + // Create the Libp2p configuration + let config = Libp2pConfig:: { + keypair: libp2p_keypair, + bind_address, + known_peers, + quorum_membership: Some(quorum_membership), + auth_message: Some(auth_message), + kademlia_config, + gossip_config, + request_response_config, + }; + let reliability_config_dup = reliability_config.clone(); Box::pin(async move { - // If it's the second time we are starting this network, clear the bootstrap info - let mut write_ids = node_ids_ref.write().await; - if write_ids.contains(&node_id) { - write_ids.clear(); - bootstrap_addrs_ref.write().await.clear(); - } - write_ids.insert(node_id); - drop(write_ids); Arc::new( match Libp2pNetwork::new( - Libp2pMetricsValue::default(), config, - pubkey.clone(), - lookup_record_value, - bootstrap_addrs_ref, - usize::try_from(node_id).unwrap(), + &hotshot_public_key, + Libp2pMetricsValue::default(), #[cfg(feature = "hotshot-testing")] reliability_config_dup, ) @@ -379,8 +440,7 @@ pub fn derive_libp2p_multiaddr(addr: &String) -> anyhow::Result { } impl Libp2pNetwork { - /// Create and return a Libp2p network from a network config file - /// and various other configuration-specific values. + /// Create and return a Libp2p network from a Libp2p config /// /// # Errors /// If we are unable to parse a Multiaddress @@ -388,150 +448,23 @@ impl Libp2pNetwork { /// # Panics /// If we are unable to calculate the replication factor #[allow(clippy::too_many_arguments)] - pub async fn from_config( - mut config: NetworkConfig, - membership: Arc>, - gossip_config: GossipConfig, - request_response_config: RequestResponseConfig, - bind_address: Multiaddr, - pub_key: &T::SignatureKey, - priv_key: &::PrivateKey, - metrics: Libp2pMetricsValue, - ) -> anyhow::Result { - // Try to take our Libp2p config from our broader network config - let libp2p_config = config - .libp2p_config - .take() - .ok_or(anyhow!("Libp2p config not supplied"))?; - - // Derive our Libp2p keypair from our supplied private key - let keypair = derive_libp2p_keypair::(priv_key)?; - - // Build our libp2p configuration - let mut config_builder = NetworkNodeConfigBuilder::default(); - - // Set the gossip configuration - config_builder.gossip_config(gossip_config.clone()); - config_builder.request_response_config(request_response_config); - - // Construct the auth message - let auth_message = - construct_auth_message(pub_key, &keypair.public().to_peer_id(), priv_key) - .with_context(|| "Failed to construct auth message")?; - - // Set the auth message and stake table - config_builder - .stake_table(Some(membership)) - .auth_message(Some(auth_message)); - - // The replication factor is the minimum of [the default and 2/3 the number of nodes] - let Some(default_replication_factor) = DEFAULT_REPLICATION_FACTOR else { - return Err(anyhow!("Default replication factor not supplied")); - }; - - let replication_factor = NonZeroUsize::new(min( - default_replication_factor.get(), - config.config.num_nodes_with_stake.get() * 2 / 3, - )) - .with_context(|| "Failed to calculate replication factor")?; - - // Sign our DHT lookup record - let lookup_record_value = RecordValue::new_signed( - &RecordKey::new(Namespace::Lookup, pub_key.to_bytes()), - // The value is our Libp2p Peer ID - keypair.public().to_peer_id().to_bytes(), - priv_key, - ) - .with_context(|| "Failed to sign DHT lookup record")?; - - config_builder - .keypair(keypair) - .replication_factor(replication_factor) - .bind_address(Some(bind_address.clone())); - - // Choose `mesh_n` random nodes to connect to for bootstrap - let bootstrap_nodes = libp2p_config - .bootstrap_nodes - .into_iter() - .choose_multiple(&mut StdRng::from_entropy(), gossip_config.mesh_n); - config_builder.to_connect_addrs(HashSet::from_iter(bootstrap_nodes.clone())); - - // Build the node's configuration - let node_config = config_builder.build()?; - - // Calculate all keys so we can keep track of direct message recipients - let mut all_keys = BTreeSet::new(); - - // Insert all known nodes into the set of all keys - for node in config.config.known_nodes_with_stake { - all_keys.insert(T::SignatureKey::public_key(&node.stake_table_entry)); - } - - Ok(Libp2pNetwork::new( - metrics, - node_config, - pub_key.clone(), - lookup_record_value, - Arc::new(RwLock::new(bootstrap_nodes)), - usize::try_from(config.node_index)?, - #[cfg(feature = "hotshot-testing")] - None, - ) - .await?) - } - - /// Returns whether or not the network is currently ready. - #[must_use] - pub fn is_ready(&self) -> bool { - self.inner.is_ready.load(Ordering::Relaxed) - } - - /// Returns only when the network is ready. - pub async fn wait_for_ready(&self) { - loop { - if self.is_ready() { - break; - } - sleep(Duration::from_secs(1)).await; - } - } - - /// Constructs new network for a node. Note that this network is unconnected. - /// One must call `connect` in order to connect. - /// * `config`: the configuration of the node - /// * `pk`: public key associated with the node - /// * `bootstrap_addrs`: rwlock containing the bootstrap addrs - /// # Errors - /// Returns error in the event that the underlying libp2p network - /// is unable to create a network. - /// - /// # Panics - /// - /// This will panic if there are less than 5 bootstrap nodes - #[allow(clippy::too_many_arguments)] pub async fn new( + config: Libp2pConfig, + hotshot_public_key: &T::SignatureKey, metrics: Libp2pMetricsValue, - config: NetworkNodeConfig, - pk: T::SignatureKey, - lookup_record_value: RecordValue, - bootstrap_addrs: BootstrapAddrs, - id: usize, #[cfg(feature = "hotshot-testing")] reliability_config: Option>, - ) -> Result, NetworkError> { - let (mut rx, network_handle) = spawn_network_node::(config.clone(), id) - .await - .map_err(|e| NetworkError::ConfigError(format!("failed to spawn network node: {e}")))?; - - // Add our own address to the bootstrap addresses - let addr = network_handle.listen_addr(); - let pid = network_handle.peer_id(); - bootstrap_addrs.write().await.push((pid, addr)); + ) -> anyhow::Result { + // Warn in case the node was accidentally compiled with the `hotshot-testing` feature + #[cfg(feature = "hotshot-testing")] + warn!("Compiled with `hotshot-testing` feature, Libp2p messages will be unreliable"); - let mut pubkey_pid_map = BiHashMap::new(); - pubkey_pid_map.insert(pk.clone(), network_handle.peer_id()); + // Spawn the network node with a copy of our config + let (mut rx, network_handle) = spawn_network_node::(config.clone()) + .await + .with_context(|| "failed to spawn network node")?; - // Subscribe to the relevant topics - let subscribed_topics = HashSet::from_iter(vec![QC_TOPIC.to_string()]); + // Subscribe only to the global topic (DA messages are direct-broadcasted) + let subscribed_topics = HashSet::from_iter(vec![GLOBAL_TOPIC.to_string()]); // unbounded channels may not be the best choice (spammed?) // if bounded figure out a way to log dropped msgs @@ -545,11 +478,9 @@ impl Libp2pNetwork { handle: Arc::new(network_handle), receiver: Mutex::new(receiver), sender: sender.clone(), - pk, - bootstrap_addrs, + hotshot_public_key: hotshot_public_key.clone(), is_ready: Arc::new(AtomicBool::new(false)), - // This is optimal for 10-30 nodes. TODO: parameterize this for both tests and examples - dht_timeout: config.dht_timeout.unwrap_or(Duration::from_secs(120)), + dht_timeout: Duration::from_secs(60), is_bootstrapped: Arc::new(AtomicBool::new(false)), metrics, subscribed_topics, @@ -569,11 +500,27 @@ impl Libp2pNetwork { result.handle_event_generator(sender, rx); result.spawn_node_lookup(node_lookup_recv); - result.spawn_connect(id, lookup_record_value); + result.spawn_connect(config.kademlia_config.lookup_record_value); Ok(result) } + /// Returns whether or not the network is currently ready. + #[must_use] + pub fn is_ready(&self) -> bool { + self.inner.is_ready.load(Ordering::Relaxed) + } + + /// Returns only when the network is ready. + pub async fn wait_for_ready(&self) { + loop { + if self.is_ready() { + break; + } + sleep(Duration::from_secs(1)).await; + } + } + /// Spawns task for looking up nodes pre-emptively #[allow(clippy::cast_sign_loss, clippy::cast_precision_loss)] fn spawn_node_lookup( @@ -606,21 +553,15 @@ impl Libp2pNetwork { } /// Initiates connection to the outside world - fn spawn_connect(&mut self, id: usize, lookup_record_value: RecordValue) { - let pk = self.inner.pk.clone(); - let bootstrap_ref = Arc::clone(&self.inner.bootstrap_addrs); + fn spawn_connect(&mut self, lookup_record_value: RecordValue) { let handle = Arc::clone(&self.inner.handle); let is_bootstrapped = Arc::clone(&self.inner.is_bootstrapped); let inner = Arc::clone(&self.inner); + let hotshot_public_key = self.inner.hotshot_public_key.clone(); spawn({ let is_ready = Arc::clone(&self.inner.is_ready); async move { - let bs_addrs = bootstrap_ref.read().await.clone(); - - // Add known peers to the network - handle.add_known_peers(bs_addrs).unwrap(); - // Begin the bootstrap process handle.begin_bootstrap()?; while !is_bootstrapped.load(Ordering::Relaxed) { @@ -628,14 +569,14 @@ impl Libp2pNetwork { handle.begin_bootstrap()?; } - // Subscribe to the QC topic - handle.subscribe(QC_TOPIC.to_string()).await.unwrap(); + // Subscribe to the global topic + handle.subscribe(GLOBAL_TOPIC.to_string()).await.unwrap(); // Map our staking key to our Libp2p Peer ID so we can properly // route direct messages while handle .put_record( - RecordKey::new(Namespace::Lookup, pk.to_bytes()), + RecordKey::new(Namespace::Lookup, hotshot_public_key.to_bytes()), lookup_record_value.clone(), ) .await @@ -644,8 +585,8 @@ impl Libp2pNetwork { sleep(Duration::from_secs(1)).await; } - // Wait for the network to connect to the required number of peers - if let Err(e) = handle.wait_to_connect(4, id).await { + // Wait for the network to connect to at least one peer + if let Err(e) = handle.wait_to_connect(1).await { error!("Failed to connect to peers: {:?}", e); return Err::<(), NetworkError>(e); } @@ -878,8 +819,8 @@ impl ConnectedNetwork for Libp2pNetwork { return Err(NetworkError::NotReadyYet); }; - // short circuit if we're dming ourselves - if recipient == self.inner.pk { + // Short circuit if we're trying to message ourselves + if recipient == self.inner.hotshot_public_key { // panic if we already shut down? self.inner.sender.try_send(message).map_err(|_x| { self.inner.metrics.num_failed_messages.add(1); diff --git a/crates/hotshot/src/traits/networking/memory_network.rs b/crates/hotshot/src/traits/networking/memory_network.rs index 5925a85eff..297f116430 100644 --- a/crates/hotshot/src/traits/networking/memory_network.rs +++ b/crates/hotshot/src/traits/networking/memory_network.rs @@ -180,8 +180,6 @@ impl TestableNetworkingImplementation { fn generator( _expected_node_count: usize, - _num_bootstrap: usize, - _network_id: usize, da_committee_size: usize, reliability_config: Option>, _secondary_network_delay: Duration, diff --git a/crates/hotshot/src/traits/networking/push_cdn_network.rs b/crates/hotshot/src/traits/networking/push_cdn_network.rs index 3601c38f2d..9921950f01 100644 --- a/crates/hotshot/src/traits/networking/push_cdn_network.rs +++ b/crates/hotshot/src/traits/networking/push_cdn_network.rs @@ -39,17 +39,23 @@ use hotshot_types::{ traits::{ metrics::{Counter, Metrics, NoMetrics}, network::{BroadcastDelay, ConnectedNetwork, Topic as HotShotTopic}, - node_implementation::NodeType, signature_key::SignatureKey, }, utils::bincode_opts, BoxSyncFuture, }; + +#[cfg(feature = "hotshot-testing")] +use hotshot_types::traits::node_implementation::NodeType; + use num_enum::{IntoPrimitive, TryFromPrimitive}; use parking_lot::Mutex; #[cfg(feature = "hotshot-testing")] use rand::{rngs::StdRng, RngCore, SeedableRng}; -use tokio::{spawn, sync::mpsc::error::TrySendError, time::sleep}; +use tokio::sync::mpsc::error::TrySendError; +#[cfg(feature = "hotshot-testing")] +use tokio::{spawn, time::sleep}; +#[cfg(feature = "hotshot-testing")] use tracing::error; use super::NetworkError; @@ -288,8 +294,6 @@ impl TestableNetworkingImplementation #[allow(clippy::too_many_lines)] fn generator( _expected_node_count: usize, - _num_bootstrap: usize, - _network_id: usize, da_committee_size: usize, _reliability_config: Option>, _secondary_network_delay: Duration, diff --git a/crates/hotshot/src/types/handle.rs b/crates/hotshot/src/types/handle.rs index d1d45473c1..995d5f1cc5 100644 --- a/crates/hotshot/src/types/handle.rs +++ b/crates/hotshot/src/types/handle.rs @@ -358,13 +358,13 @@ impl + 'static, V: Versions> } /// Wrapper to get the view number this node is on. - #[instrument(skip_all, target = "SystemContextHandle", fields(id = self.hotshot.id))] + #[instrument(skip_all, target = "SystemContextHandle")] pub async fn cur_view(&self) -> TYPES::View { self.hotshot.consensus.read().await.cur_view() } /// Wrapper to get the epoch number this node is on. - #[instrument(skip_all, target = "SystemContextHandle", fields(id = self.hotshot.id))] + #[instrument(skip_all, target = "SystemContextHandle")] pub async fn cur_epoch(&self) -> TYPES::Epoch { self.hotshot.consensus.read().await.cur_epoch() } diff --git a/crates/libp2p-networking/Cargo.toml b/crates/libp2p-networking/Cargo.toml index 83856af400..d792542af0 100644 --- a/crates/libp2p-networking/Cargo.toml +++ b/crates/libp2p-networking/Cargo.toml @@ -21,8 +21,6 @@ bincode = { workspace = true } blake3 = { workspace = true } cbor4ii = "0.3" delegate = "0.13" -derive_builder = "0.20" -derive_more = { workspace = true } futures = { workspace = true } hotshot-types = { path = "../types" } lazy_static = { workspace = true } diff --git a/crates/libp2p-networking/src/lib.rs b/crates/libp2p-networking/src/lib.rs index 6b71165223..31bfee245f 100644 --- a/crates/libp2p-networking/src/lib.rs +++ b/crates/libp2p-networking/src/lib.rs @@ -9,7 +9,7 @@ /// Network logic pub mod network; -/// symbols needed to implement a networking instance over libp2p-netorking +/// Re-exports from libp2p pub mod reexport { pub use libp2p::{request_response::ResponseChannel, Multiaddr}; pub use libp2p_identity::PeerId; diff --git a/crates/libp2p-networking/src/network/behaviours/dht/mod.rs b/crates/libp2p-networking/src/network/behaviours/dht/mod.rs index f785d1fa10..a963fc785b 100644 --- a/crates/libp2p-networking/src/network/behaviours/dht/mod.rs +++ b/crates/libp2p-networking/src/network/behaviours/dht/mod.rs @@ -27,7 +27,6 @@ use libp2p::kad::{ use libp2p::kad::{ store::RecordStore, Behaviour as KademliaBehaviour, BootstrapError, Event as KademliaEvent, }; -use libp2p_identity::PeerId; use store::{file_backed::FileBackedStore, validated::ValidatedStore}; use tokio::{spawn, sync::mpsc::UnboundedSender, time::sleep}; use tracing::{debug, error, warn}; @@ -68,10 +67,6 @@ pub struct DHTBehaviour { in_progress_put_record_queries: HashMap, /// State of bootstrapping pub bootstrap_state: Bootstrap, - /// the peer id (useful only for debugging right now) - pub peer_id: PeerId, - /// replication factor - pub replication_factor: NonZeroUsize, /// Sender to retry requests. retry_tx: Option>, /// Sender to the bootstrap task @@ -81,6 +76,13 @@ pub struct DHTBehaviour { phantom: PhantomData, } +/// The default implementation just calls the constructor +impl Default for DHTBehaviour { + fn default() -> Self { + Self::new() + } +} + /// State of bootstrapping #[derive(Debug, Clone)] pub struct Bootstrap { @@ -117,14 +119,13 @@ impl DHTBehaviour { } /// Create a new DHT behaviour #[must_use] - pub fn new(pid: PeerId, replication_factor: NonZeroUsize) -> Self { + pub fn new() -> Self { // needed because otherwise we stay in client mode when testing locally // and don't publish keys stuff // e.g. dht just doesn't work. We'd need to add mdns and that doesn't seem worth it since // we won't have a local network // Self { - peer_id: pid, in_progress_record_queries: HashMap::default(), in_progress_put_record_queries: HashMap::default(), outstanding_dht_query_keys: HashSet::default(), @@ -133,7 +134,6 @@ impl DHTBehaviour { backoff: ExponentialBackoff::new(2, Duration::from_secs(1)), }, in_progress_get_closest_peers: HashMap::default(), - replication_factor, retry_tx: None, bootstrap_tx: None, phantom: PhantomData, @@ -145,7 +145,7 @@ impl DHTBehaviour { &mut self, kadem: &mut KademliaBehaviour>>, ) { - let mut err = format!("KBUCKETS: PID: {:?}, ", self.peer_id); + let mut err = "KBUCKETS: ".to_string(); let v = kadem.kbuckets().collect::>(); for i in v { for j in i.iter() { @@ -159,11 +159,6 @@ impl DHTBehaviour { error!("{:?}", err); } - /// Get the replication factor for queries - #[must_use] - pub fn replication_factor(&self) -> NonZeroUsize { - self.replication_factor - } /// Publish a key/value to the kv store. /// Once replicated upon all nodes, the caller is notified over /// `chan` @@ -381,10 +376,7 @@ impl DHTBehaviour { query.progress = DHTProgress::NotStarted; query.backoff.start_next(false); - warn!( - "Put DHT: error performing put: {:?}. Retrying on pid {:?}.", - e, self.peer_id - ); + warn!("Put DHT: error performing put: {:?}. Retrying.", e); // push back onto the queue self.retry_put(query); } diff --git a/crates/libp2p-networking/src/network/def.rs b/crates/libp2p-networking/src/network/def.rs index a52fdae36c..4ed4ded3cd 100644 --- a/crates/libp2p-networking/src/network/def.rs +++ b/crates/libp2p-networking/src/network/def.rs @@ -15,7 +15,7 @@ use libp2p::{ }; use libp2p_identity::PeerId; use libp2p_swarm_derive::NetworkBehaviour; -use tracing::{debug, error}; +use tracing::error; use super::{ behaviours::dht::store::{file_backed::FileBackedStore, validated::ValidatedStore}, @@ -27,32 +27,27 @@ use super::{ /// - direct messaging /// - p2p broadcast /// - connection management -#[derive(NetworkBehaviour, derive_more::Debug)] +#[derive(NetworkBehaviour)] #[behaviour(to_swarm = "NetworkEventInternal")] pub struct NetworkDef { /// purpose: broadcasting messages to many peers /// NOTE gossipsub works ONLY for sharing messages right now /// in the future it may be able to do peer discovery and routing /// - #[debug(skip)] gossipsub: GossipBehaviour, /// The DHT store. We use a `FileBackedStore` to occasionally save the DHT to /// a file on disk and a `ValidatedStore` to validate the records stored. - #[debug(skip)] pub dht: libp2p::kad::Behaviour>>, /// purpose: identifying the addresses from an outside POV - #[debug(skip)] identify: IdentifyBehaviour, /// purpose: directly messaging peer - #[debug(skip)] pub direct_message: cbor::Behaviour, Vec>, /// Auto NAT behaviour to determine if we are publicly reachable and /// by which address - #[debug(skip)] pub autonat: libp2p::autonat::Behaviour, } diff --git a/crates/libp2p-networking/src/network/mod.rs b/crates/libp2p-networking/src/network/mod.rs index c4348af023..b1873dc861 100644 --- a/crates/libp2p-networking/src/network/mod.rs +++ b/crates/libp2p-networking/src/network/mod.rs @@ -9,7 +9,7 @@ pub mod behaviours; /// defines the swarm and network definition (internal) mod def; /// functionality of a libp2p network node -mod node; +pub mod node; /// Alternative Libp2p transport implementations pub mod transport; @@ -22,7 +22,6 @@ use async_lock::RwLock; use futures::channel::oneshot::Sender; use hotshot_types::traits::{network::NetworkError, node_implementation::NodeType}; use libp2p::{ - build_multiaddr, core::{muxing::StreamMuxerBox, transport::Boxed}, dns::tokio::Transport as DnsTransport, gossipsub::Event as GossipEvent, @@ -30,7 +29,7 @@ use libp2p::{ identity::Keypair, quic, request_response::ResponseChannel, - Multiaddr, Transport, + Transport, }; use libp2p_identity::PeerId; use quic::tokio::Transport as QuicTransport; @@ -40,9 +39,8 @@ use transport::StakeTableAuthentication; pub use self::{ def::NetworkDef, node::{ - spawn_network_node, GossipConfig, NetworkNode, NetworkNodeConfig, NetworkNodeConfigBuilder, - NetworkNodeConfigBuilderError, NetworkNodeHandle, NetworkNodeReceiver, - RequestResponseConfig, DEFAULT_REPLICATION_FACTOR, + spawn_network_node, GossipConfig, NetworkNode, NetworkNodeHandle, NetworkNodeReceiver, + RequestResponseConfig, }, }; @@ -72,8 +70,6 @@ pub enum ClientRequest { DirectResponse(ResponseChannel>, Vec), /// prune a peer Prune(PeerId), - /// add vec of known peers or addresses - AddKnownPeers(Vec<(PeerId, Multiaddr)>), /// Ignore peers. Only here for debugging purposes. /// Allows us to have nodes that are never pruned IgnorePeers(Vec), @@ -139,13 +135,6 @@ pub enum NetworkEventInternal { AutonatEvent(libp2p::autonat::Event), } -/// Bind all interfaces on port `port` -/// NOTE we may want something more general in the fture. -#[must_use] -pub fn gen_multiaddr(port: u16) -> Multiaddr { - build_multiaddr!(Ip4([0, 0, 0, 0]), Udp(port), QuicV1) -} - /// `BoxedTransport` is a type alias for a boxed tuple containing a `PeerId` and a `StreamMuxerBox`. /// /// This type is used to represent a transport in the libp2p network framework. The `PeerId` is a unique identifier for each peer in the network, and the `StreamMuxerBox` is a type of multiplexer that can handle multiple substreams over a single connection. @@ -160,7 +149,7 @@ type BoxedTransport = Boxed<(PeerId, StreamMuxerBox)>; #[instrument(skip(identity))] pub async fn gen_transport( identity: Keypair, - stake_table: Option>>, + quorum_membership: Option>>, auth_message: Option>, ) -> Result { // Create the initial `Quic` transport @@ -172,7 +161,7 @@ pub async fn gen_transport( // Require authentication against the stake table let transport: StakeTableAuthentication<_, T, _> = - StakeTableAuthentication::new(transport, stake_table, auth_message); + StakeTableAuthentication::new(transport, quorum_membership, auth_message); // Support DNS resolution let transport = { diff --git a/crates/libp2p-networking/src/network/node.rs b/crates/libp2p-networking/src/network/node.rs index 279d035949..5ae9cad238 100644 --- a/crates/libp2p-networking/src/network/node.rs +++ b/crates/libp2p-networking/src/network/node.rs @@ -5,23 +5,22 @@ // along with the HotShot repository. If not, see . /// configuration for the libp2p network (e.g. how it should be built) -mod config; +pub mod config; /// libp2p network handle /// allows for control over the libp2p network mod handle; use std::{ - collections::{HashMap, HashSet}, - iter, + collections::HashSet, num::{NonZeroU32, NonZeroUsize}, - time::Duration, + time::{Duration, Instant}, }; +use anyhow::{anyhow, Context}; +use config::{KademliaConfig, Libp2pConfig}; use futures::{channel::mpsc, SinkExt, StreamExt}; -use hotshot_types::{ - constants::KAD_DEFAULT_REPUB_INTERVAL_SEC, traits::node_implementation::NodeType, -}; +use hotshot_types::traits::node_implementation::NodeType; use libp2p::{ autonat, core::transport::ListenerId, @@ -33,8 +32,8 @@ use libp2p::{ Behaviour as IdentifyBehaviour, Config as IdentifyConfig, Event as IdentifyEvent, Info as IdentifyInfo, }, - identity::Keypair, kad::{store::MemoryStore, Behaviour, Config, Mode, Record}, + multiaddr::Protocol, request_response::{ Behaviour as RequestResponse, Config as Libp2pRequestResponseConfig, ProtocolSupport, }, @@ -42,18 +41,15 @@ use libp2p::{ Multiaddr, StreamProtocol, Swarm, SwarmBuilder, }; use libp2p_identity::PeerId; -use rand::{prelude::SliceRandom, thread_rng}; use tokio::{ select, spawn, sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, + time::timeout, }; -use tracing::{debug, error, info, info_span, instrument, warn, Instrument}; +use tracing::{debug, error, info, info_span, instrument, trace, warn, Instrument}; pub use self::{ - config::{ - GossipConfig, NetworkNodeConfig, NetworkNodeConfigBuilder, NetworkNodeConfigBuilderError, - RequestResponseConfig, DEFAULT_REPLICATION_FACTOR, - }, + config::{GossipConfig, RequestResponseConfig}, handle::{spawn_network_node, NetworkNodeHandle, NetworkNodeReceiver}, }; use super::{ @@ -81,21 +77,20 @@ pub const ESTABLISHED_LIMIT: NonZeroU32 = pub const ESTABLISHED_LIMIT_UNWR: u32 = 10; /// Network definition -#[derive(derive_more::Debug)] pub struct NetworkNode { - /// peer id of network node - peer_id: PeerId, - /// the swarm of networkbehaviours - #[debug(skip)] + /// The swarm of network behaviours swarm: Swarm>, - /// the listener id we are listening on, if it exists + /// The listener id we are listening on, if it is initialized listener_id: Option, - /// Handler for direct messages - direct_message_state: DMBehaviour, - /// Handler for DHT Events + /// The handler for direct messages + direct_message_handler: DMBehaviour, + /// The handler for DHT events dht_handler: DHTBehaviour, - /// Channel to resend requests, set to Some when we call `spawn_listeners` + /// Channel to resend requests (set when we call `spawn_listeners`) resend_tx: Option>, + + /// The Kademlia config + kademlia_config: KademliaConfig, } impl NetworkNode { @@ -109,69 +104,61 @@ impl NetworkNode { self.swarm.connected_peers().copied().collect() } - /// starts the swarm listening on `listen_addr` - /// and optionally dials into peer `known_peer` - /// returns the address the swarm is listening upon + /// Bind the swarm to a given address #[instrument(skip(self))] - pub async fn start_listen( - &mut self, - listen_addr: Multiaddr, - ) -> Result { - self.listener_id = Some(self.swarm.listen_on(listen_addr).map_err(|err| { - NetworkError::ListenError(format!("failed to listen for Libp2p: {err}")) - })?); - let addr = loop { - if let Some(SwarmEvent::NewListenAddr { address, .. }) = self.swarm.next().await { - break address; + pub async fn bind_to(&mut self, bind_address: &Multiaddr) -> anyhow::Result<()> { + // Debug log the address we are binding to + debug!("Libp2p binding to {:?}", bind_address); + + // Start listening on the given address + self.listener_id = Some( + self.swarm + .listen_on(bind_address.clone()) + .with_context(|| "failed to bind to address")?, + ); + + // Wait for the listener to be bound with a 60s timeout + let start_time = Instant::now(); + loop { + match timeout(Duration::from_secs(60), self.swarm.next()).await { + // If we successfully get a NewListenAddr event, break + Ok(Some(SwarmEvent::NewListenAddr { .. })) => break, + // If we timeout, return an error + Err(_) => { + return Err(anyhow!("Timed out binding to address")); + } + // If we get any other event, continue waiting + _ => {} } - }; - info!("Libp2p listening on {:?}", addr); - Ok(addr) - } - /// initialize the DHT with known peers - /// add the peers to kademlia and then - /// the `spawn_listeners` function - /// will start connecting to peers - #[instrument(skip(self))] - pub fn add_known_peers(&mut self, known_peers: &[(PeerId, Multiaddr)]) { - debug!("Adding {} known peers", known_peers.len()); - let behaviour = self.swarm.behaviour_mut(); - let mut bs_nodes = HashMap::>::new(); - let mut shuffled = known_peers.iter().collect::>(); - shuffled.shuffle(&mut thread_rng()); - for (peer_id, addr) in shuffled { - if *peer_id != self.peer_id { - behaviour.dht.add_address(peer_id, addr.clone()); - behaviour.autonat.add_server(*peer_id, Some(addr.clone())); - bs_nodes.insert(*peer_id, iter::once(addr.clone()).collect()); + // If we've been waiting for more than 60 seconds, return an error + if start_time.elapsed() > Duration::from_secs(60) { + return Err(anyhow!("Timed out binding to address")); } } + + // Log the address we are listening on + info!("Libp2p listening on {:?}", bind_address); + + Ok(()) } - /// Creates a new `Network` with the given settings. + /// Creates a new `NetworkNode` with the given Libp2p configuration /// - /// Currently: - /// * Generates a random key pair and associated [`PeerId`] - /// * Launches a hopefully production ready transport: - /// QUIC v1 (RFC 9000) + DNS - /// * Generates a connection to the "broadcast" topic - /// * Creates a swarm to manage peers and events - #[instrument] - pub async fn new(config: NetworkNodeConfig) -> Result { - // Generate a random `KeyPair` if one is not specified - let keypair = config - .keypair - .clone() - .unwrap_or_else(Keypair::generate_ed25519); - + /// # Errors + /// If the network node cannot be created + /// + /// # Panics + /// If `5 == 0` + #[allow(clippy::too_many_lines)] + pub async fn new(config: &Libp2pConfig) -> anyhow::Result { // Get the `PeerId` from the `KeyPair` - let peer_id = PeerId::from(keypair.public()); + let peer_id = PeerId::from(config.keypair.public()); // Generate the transport from the keypair, stake table, and auth message let transport: BoxedTransport = gen_transport::( - keypair.clone(), - config.stake_table.clone(), + config.keypair.clone(), + config.quorum_membership.clone(), config.auth_message.clone(), ) .await?; @@ -215,62 +202,41 @@ impl NetworkNode { NetworkError::ConfigError(format!("error building gossipsub config: {err:?}")) })?; - // - Build a gossipsub network behavior + // Create the Gossipsub behaviour let gossipsub: Gossipsub = Gossipsub::new( - MessageAuthenticity::Signed(keypair.clone()), + MessageAuthenticity::Signed(config.keypair.clone()), gossipsub_config, ) .map_err(|err| { NetworkError::ConfigError(format!("error building gossipsub behaviour: {err:?}")) })?; - // Build a identify network behavior needed for own - // node connection information - // E.g. this will answer the question: how are other nodes - // seeing the peer from behind a NAT + // Configure and create the Identify behaviour let identify_cfg = - IdentifyConfig::new("HotShot/identify/1.0".to_string(), keypair.public()); + IdentifyConfig::new("HotShot/identify/1.0".to_string(), config.keypair.public()); let identify = IdentifyBehaviour::new(identify_cfg); - // - Build DHT needed for peer discovery + // Configure the Kademlia behaviour let mut kconfig = Config::new(StreamProtocol::new("/ipfs/kad/1.0.0")); - // 8 hours by default - let record_republication_interval = config - .republication_interval - .unwrap_or(Duration::from_secs(KAD_DEFAULT_REPUB_INTERVAL_SEC)); - let ttl = Some(config.ttl.unwrap_or(16 * record_republication_interval)); kconfig .set_parallelism(NonZeroUsize::new(5).unwrap()) - .set_provider_publication_interval(Some(record_republication_interval)) - .set_publication_interval(Some(record_republication_interval)) - .set_record_ttl(ttl); - - // allowing panic here because something is very wrong if this fales - #[allow(clippy::panic)] - if let Some(factor) = config.replication_factor { - kconfig.set_replication_factor(factor); - } else { - panic!("Replication factor not set"); - } + .set_provider_publication_interval(config.kademlia_config.publication_interval) + .set_publication_interval(config.kademlia_config.publication_interval) + .set_record_ttl(config.kademlia_config.record_ttl); - // Extract the DHT file path from the config, defaulting to `libp2p_dht.json` - let dht_file_path = config - .dht_file_path - .clone() - .unwrap_or_else(|| "libp2p_dht.bin".into()); - - // Create the DHT behaviour + // Create the Kademlia behaviour let mut kadem = Behaviour::with_config( peer_id, FileBackedStore::new( ValidatedStore::new(MemoryStore::new(peer_id)), - dht_file_path, + config.kademlia_config.file_path.clone(), 10, ), kconfig, ); kadem.set_mode(Some(Mode::Server)); + // Use the default request response config let rrconfig = Libp2pRequestResponseConfig::default(); // Create a new `cbor` codec with the given request and response sizes @@ -279,6 +245,7 @@ impl NetworkNode { config.request_response_config.response_size_maximum, ); + // Create the direct message behaviour with our configured codec let direct_message: super::cbor::Behaviour, Vec> = RequestResponse::with_codec( cbor, @@ -303,8 +270,8 @@ impl NetworkNode { autonat::Behaviour::new(peer_id, autonat_config), ); - // build swarm - let swarm = SwarmBuilder::with_existing_identity(keypair.clone()); + // Build the swarm + let swarm = SwarmBuilder::with_existing_identity(config.keypair.clone()); let swarm = swarm.with_tokio(); swarm @@ -314,24 +281,41 @@ impl NetworkNode { .unwrap() .build() }; - for (peer, addr) in &config.to_connect_addrs { - if peer != swarm.local_peer_id() { - swarm.behaviour_mut().add_address(peer, addr.clone()); + + // Add the known peers to Libp2p + let mut num_known_peers_added = 0; + for mut known_peer in config.known_peers.clone() { + // Lob off the last protocol from the multiaddr + if let Some(protocol) = known_peer.pop() { + // Make sure it is the P2P protocol (which includes the peer id) + if let Protocol::P2p(peer_id) = protocol { + // Add the address to Libp2p behaviors + swarm.behaviour_mut().add_address(&peer_id, known_peer); + num_known_peers_added += 1; + } else { + warn!("Known peer {:?} has no P2P address", known_peer); + } + } else { + warn!("Known peer {:?} has no address", known_peer); } } + // If we hadn't found any suitable known peers, return an error + if num_known_peers_added == 0 { + return Err(anyhow!("No suitable known peers known")); + } + + // Create our own DHT handler + let dht_handler = DHTBehaviour::new(); + + // Create and return the new network node Ok(Self { - peer_id, swarm, listener_id: None, - direct_message_state: DMBehaviour::default(), - dht_handler: DHTBehaviour::new( - peer_id, - config - .replication_factor - .unwrap_or(NonZeroUsize::new(4).unwrap()), - ), + direct_message_handler: DMBehaviour::default(), + dht_handler, resend_tx: None, + kademlia_config: config.kademlia_config.clone(), }) } @@ -344,7 +328,7 @@ impl NetworkNode { match self.swarm.behaviour_mut().dht.put_record( record, libp2p::kad::Quorum::N( - NonZeroUsize::try_from(self.dht_handler.replication_factor().get() / 2) + NonZeroUsize::try_from(self.kademlia_config.replication_factor / 2) .expect("replication factor should be bigger than 0"), ), ) { @@ -352,7 +336,7 @@ impl NetworkNode { // failed try again later query.progress = DHTProgress::NotStarted; query.backoff.start_next(false); - error!("Error publishing to DHT: {e:?} for peer {:?}", self.peer_id); + warn!("Error publishing to DHT: {e:?}"); } Ok(qid) => { debug!("Published record to DHT with qid {:?}", qid); @@ -475,14 +459,11 @@ impl NetworkNode { backoff: ExponentialBackoff::default(), retry_count, }; - self.direct_message_state.add_direct_request(req, id); + self.direct_message_handler.add_direct_request(req, id); } ClientRequest::DirectResponse(chan, msg) => { behaviour.add_direct_response(chan, msg); } - ClientRequest::AddKnownPeers(peers) => { - self.add_known_peers(&peers); - } ClientRequest::Prune(pid) => { if self.swarm.disconnect_peer_id(pid).is_err() { warn!("Could not disconnect from {:?}", pid); @@ -635,7 +616,7 @@ impl NetworkNode { } }, NetworkEventInternal::DMEvent(e) => self - .direct_message_state + .direct_message_handler .handle_dm_event(e, self.resend_tx.clone()), NetworkEventInternal::AutonatEvent(e) => { match e { @@ -712,8 +693,10 @@ impl NetworkNode { /// Spawn a task to listen for requests on the returned channel /// as well as any events produced by libp2p - #[instrument] - pub async fn spawn_listeners( + /// + /// # Errors + /// If the listeners cannot be spawned + pub fn spawn_listeners( mut self, ) -> Result< ( @@ -734,14 +717,13 @@ impl NetworkNode { loop { select! { event = self.swarm.next() => { - debug!("peerid {:?}\t\thandling maybe event {:?}", self.peer_id, event); if let Some(event) = event { - debug!("peerid {:?}\t\thandling event {:?}", self.peer_id, event); + trace!("Libp2p handling event {:?}", event); self.handle_swarm_events(event, &r_input).await?; } }, msg = s_output.recv() => { - debug!("peerid {:?}\t\thandling msg {:?}", self.peer_id, msg); + trace!("Libp2p handling client request {:?}", msg); let shutdown = self.handle_client_requests(msg).await?; if shutdown { let _ = bootstrap_tx.send(InputEvent::ShutdownBootstrap).await; @@ -756,9 +738,4 @@ impl NetworkNode { ); Ok((s_input, r_output)) } - - /// Get a reference to the network node's peer id. - pub fn peer_id(&self) -> PeerId { - self.peer_id - } } diff --git a/crates/libp2p-networking/src/network/node/config.rs b/crates/libp2p-networking/src/network/node/config.rs index 3958d17f6d..36866c3610 100644 --- a/crates/libp2p-networking/src/network/node/config.rs +++ b/crates/libp2p-networking/src/network/node/config.rs @@ -4,90 +4,16 @@ // You should have received a copy of the MIT License // along with the HotShot repository. If not, see . -use std::{collections::HashSet, num::NonZeroUsize, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; use async_lock::RwLock; use hotshot_types::traits::node_implementation::NodeType; -use libp2p::{identity::Keypair, Multiaddr}; -use libp2p_identity::PeerId; +use libp2p::Multiaddr; +use libp2p_identity::Keypair; -use super::MAX_GOSSIP_MSG_SIZE; - -/// The default Kademlia replication factor -pub const DEFAULT_REPLICATION_FACTOR: Option = NonZeroUsize::new(10); - -/// describe the configuration of the network -#[derive(Default, derive_builder::Builder, derive_more::Debug)] -pub struct NetworkNodeConfig { - /// The keypair for the node - #[builder(setter(into, strip_option), default)] - #[debug(skip)] - pub keypair: Option, - - /// The address to bind to - #[builder(default)] - pub bind_address: Option, - - /// Replication factor for entries in the DHT - #[builder(setter(into, strip_option), default = "DEFAULT_REPLICATION_FACTOR")] - pub replication_factor: Option, - - #[builder(default)] - /// Configuration for `GossipSub` - pub gossip_config: GossipConfig, - - #[builder(default)] - /// Configuration for `RequestResponse` - pub request_response_config: RequestResponseConfig, - - /// list of addresses to connect to at initialization - pub to_connect_addrs: HashSet<(PeerId, Multiaddr)>, - - /// republication interval in DHT, must be much less than `ttl` - #[builder(default)] - pub republication_interval: Option, - - /// expiratiry for records in DHT - #[builder(default)] - pub ttl: Option, - - /// The stake table. Used for authenticating other nodes. If not supplied - /// we will not check other nodes against the stake table - #[builder(default)] - pub stake_table: Option>>, - - /// The path to the file to save the DHT to - #[builder(default)] - pub dht_file_path: Option, - - /// The signed authentication message sent to the remote peer - /// If not supplied we will not send an authentication message during the handshake - #[builder(default)] - pub auth_message: Option>, +use crate::network::behaviours::dht::record::RecordValue; - #[builder(default)] - /// The timeout for DHT lookups. - pub dht_timeout: Option, -} - -impl Clone for NetworkNodeConfig { - fn clone(&self) -> Self { - Self { - keypair: self.keypair.clone(), - bind_address: self.bind_address.clone(), - replication_factor: self.replication_factor, - gossip_config: self.gossip_config.clone(), - request_response_config: self.request_response_config.clone(), - to_connect_addrs: self.to_connect_addrs.clone(), - republication_interval: self.republication_interval, - ttl: self.ttl, - stake_table: self.stake_table.as_ref().map(Arc::clone), - dht_file_path: self.dht_file_path.clone(), - auth_message: self.auth_message.clone(), - dht_timeout: self.dht_timeout, - } - } -} +use super::MAX_GOSSIP_MSG_SIZE; /// Configuration for Libp2p's Gossipsub #[derive(Clone, Debug)] @@ -201,3 +127,41 @@ impl Default for RequestResponseConfig { } } } + +/// Configuration for Libp2p's Kademlia +#[derive(Clone, Debug)] +pub struct KademliaConfig { + /// The replication factor + pub replication_factor: usize, + /// The record ttl + pub record_ttl: Option, + /// The publication interval + pub publication_interval: Option, + /// The file path for the [file-backed] record store + pub file_path: String, + /// The lookup record value (a signed peer ID so it can be verified by any node) + pub lookup_record_value: RecordValue, +} + +/// Configuration for Libp2p +#[derive(Clone)] +pub struct Libp2pConfig { + /// The Libp2p keypair + pub keypair: Keypair, + /// The address to bind Libp2p to + pub bind_address: Multiaddr, + /// Addresses of known peers to add to Libp2p on startup + pub known_peers: Vec, + + /// The quorum membership + pub quorum_membership: Option>>, + /// The (signed) authentication message + pub auth_message: Option>, + + /// The gossip config + pub gossip_config: GossipConfig, + /// The request response config + pub request_response_config: RequestResponseConfig, + /// The kademlia config + pub kademlia_config: KademliaConfig, +} diff --git a/crates/libp2p-networking/src/network/node/handle.rs b/crates/libp2p-networking/src/network/node/handle.rs index b7a6832286..18fd00dd0d 100644 --- a/crates/libp2p-networking/src/network/node/handle.rs +++ b/crates/libp2p-networking/src/network/node/handle.rs @@ -4,50 +4,46 @@ // You should have received a copy of the MIT License // along with the HotShot repository. If not, see . -use std::{collections::HashSet, fmt::Debug, time::Duration}; +use std::{collections::HashSet, fmt::Debug, marker::PhantomData, time::Duration}; -use hotshot_types::traits::{network::NetworkError, node_implementation::NodeType}; -use libp2p::{request_response::ResponseChannel, Multiaddr}; +use anyhow::Context; +use hotshot_types::traits::{ + network::NetworkError, node_implementation::NodeType, signature_key::SignatureKey, +}; +use libp2p::request_response::ResponseChannel; use libp2p_identity::PeerId; use tokio::{ sync::mpsc::{Receiver, UnboundedReceiver, UnboundedSender}, time::{sleep, timeout}, }; -use tracing::{debug, info, instrument}; +use tracing::{info, instrument}; use crate::network::{ behaviours::dht::record::{Namespace, RecordKey, RecordValue}, - gen_multiaddr, ClientRequest, NetworkEvent, NetworkNode, NetworkNodeConfig, + ClientRequest, NetworkEvent, NetworkNode, }; +use super::config::Libp2pConfig; + /// A handle containing: /// - A reference to the state /// - Controls for the swarm #[derive(Debug, Clone)] -pub struct NetworkNodeHandle { - /// network configuration - network_config: NetworkNodeConfig, - - /// send an action to the networkbehaviour - send_network: UnboundedSender, - - /// the local address we're listening on - listen_addr: Multiaddr, +pub struct NetworkNodeHandle { + /// For sending requests to the network behaviour + request_sender: UnboundedSender, - /// the peer id of the networkbehaviour - peer_id: PeerId, - - /// human readable id - id: usize, + /// Phantom data + pd: PhantomData, } /// internal network node receiver #[derive(Debug)] pub struct NetworkNodeReceiver { - /// the receiver - receiver: UnboundedReceiver, + /// The receiver for requests from the application + request_receiver: UnboundedReceiver, - ///kill switch + /// The kill switch for the receiver recv_kill: Option>, } @@ -56,7 +52,7 @@ impl NetworkNodeReceiver { /// # Errors /// Errors if the receiver channel is closed pub async fn recv(&mut self) -> Result { - self.receiver + self.request_receiver .recv() .await .ok_or(NetworkError::ChannelReceiveError( @@ -78,42 +74,40 @@ impl NetworkNodeReceiver { /// # Errors /// Errors if spawning the task fails pub async fn spawn_network_node( - config: NetworkNodeConfig, - id: usize, -) -> Result<(NetworkNodeReceiver, NetworkNodeHandle), NetworkError> { - let mut network = NetworkNode::new(config.clone()) + config: Libp2pConfig, +) -> anyhow::Result<(NetworkNodeReceiver, NetworkNodeHandle)> { + // Create the network node (what we send requests to and from in the application) + let mut network = NetworkNode::new(&config) + .await + .with_context(|| "failed to create network node")?; + + // Bind the swarm to the given address + network + .bind_to(&config.bind_address) .await - .map_err(|e| NetworkError::ConfigError(format!("failed to create network node: {e}")))?; - // randomly assigned port - let listen_addr = config - .bind_address - .clone() - .unwrap_or_else(|| gen_multiaddr(0)); - let peer_id = network.peer_id(); - let listen_addr = network.start_listen(listen_addr).await.map_err(|e| { - NetworkError::ListenError(format!("failed to start listening on Libp2p: {e}")) - })?; - // pin here to force the future onto the heap since it can be large - // in the case of flume - let (send_chan, recv_chan) = Box::pin(network.spawn_listeners()).await.map_err(|err| { - NetworkError::ListenError(format!("failed to spawn listeners for Libp2p: {err}")) - })?; + .with_context(|| format!("failed to bind to {:?}", config.bind_address))?; + + // Spawn the listeners and get the request sender and receiver + let (request_sender, request_receiver) = network + .spawn_listeners() + .with_context(|| "failed to spawn listeners")?; + + // Create the receiver let receiver = NetworkNodeReceiver { - receiver: recv_chan, + request_receiver, recv_kill: None, }; - let handle = NetworkNodeHandle:: { - network_config: config, - send_network: send_chan, - listen_addr, - peer_id, - id, + // Create the handle (what the application uses to interact with the network) + let handle = NetworkNodeHandle { + request_sender, + pd: PhantomData, }; + Ok((receiver, handle)) } -impl NetworkNodeHandle { +impl NetworkNodeHandle { /// Cleanly shuts down a swarm node /// This is done by sending a message to /// the swarm itself to spin down @@ -131,12 +125,6 @@ impl NetworkNodeHandle { self.send_request(req) } - /// Get a reference to the network node handle's listen addr. - #[must_use] - pub fn listen_addr(&self) -> Multiaddr { - self.listen_addr.clone() - } - /// Print out the routing table used by kademlia /// NOTE: only for debugging purposes currently /// # Errors @@ -152,11 +140,7 @@ impl NetworkNodeHandle { /// /// # Errors /// If the channel closes before the result can be sent back - pub async fn wait_to_connect( - &self, - num_required_peers: usize, - node_id: usize, - ) -> Result<(), NetworkError> { + pub async fn wait_to_connect(&self, num_required_peers: usize) -> Result<(), NetworkError> { // Wait for the required number of peers to connect loop { // Get the number of currently connected peers @@ -167,8 +151,8 @@ impl NetworkNodeHandle { // Log the number of connected peers info!( - "Node {} connected to {}/{} peers", - node_id, num_connected, num_required_peers + "Libp2p connected to {}/{} peers", + num_connected, num_required_peers ); // Sleep for a second before checking again @@ -215,7 +199,7 @@ impl NetworkNodeHandle { pub async fn put_record( &self, key: RecordKey, - value: RecordValue, + value: RecordValue, ) -> Result<(), NetworkError> { // Serialize the key let key = key.to_bytes(); @@ -261,7 +245,7 @@ impl NetworkNodeHandle { let result = r.await.map_err(|_| NetworkError::RequestCancelled)?; // Deserialize the record's value - let record: RecordValue = bincode::deserialize(&result) + let record: RecordValue = bincode::deserialize(&result) .map_err(|e| NetworkError::FailedToDeserialize(e.to_string()))?; Ok(record.value().to_vec()) @@ -290,7 +274,7 @@ impl NetworkNodeHandle { pub async fn put_record_timeout( &self, key: RecordKey, - value: RecordValue, + value: RecordValue, timeout_duration: Duration, ) -> Result<(), NetworkError> { timeout(timeout_duration, self.put_record(key, value)) @@ -396,24 +380,12 @@ impl NetworkNodeHandle { self.send_request(req) } - /// Tell libp2p about known network nodes - /// # Errors - /// - Will return [`NetworkError::ChannelSendError`] when underlying `NetworkNode` has been killed - pub fn add_known_peers( - &self, - known_peers: Vec<(PeerId, Multiaddr)>, - ) -> Result<(), NetworkError> { - debug!("Adding {} known peers", known_peers.len()); - let req = ClientRequest::AddKnownPeers(known_peers); - self.send_request(req) - } - /// Send a client request to the network /// /// # Errors /// - Will return [`NetworkError::ChannelSendError`] when underlying `NetworkNode` has been killed fn send_request(&self, req: ClientRequest) -> Result<(), NetworkError> { - self.send_network + self.request_sender .send(req) .map_err(|err| NetworkError::ChannelSendError(err.to_string())) } @@ -445,22 +417,4 @@ impl NetworkNodeHandle { self.send_request(req)?; Ok(r.await.unwrap()) } - - /// Get a reference to the network node handle's id. - #[must_use] - pub fn id(&self) -> usize { - self.id - } - - /// Get a reference to the network node handle's peer id. - #[must_use] - pub fn peer_id(&self) -> PeerId { - self.peer_id - } - - /// Return a reference to the network config - #[must_use] - pub fn config(&self) -> &NetworkNodeConfig { - &self.network_config - } } diff --git a/crates/orchestrator/Cargo.toml b/crates/orchestrator/Cargo.toml deleted file mode 100644 index a80dcba85a..0000000000 --- a/crates/orchestrator/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "hotshot-orchestrator" -version = { workspace = true } -edition = { workspace = true } - -[dependencies] -anyhow = { workspace = true } -async-lock = { workspace = true } -blake3 = { workspace = true } -clap = { workspace = true } -csv = "1" -futures = { workspace = true } -hotshot-types = { path = "../types" } -libp2p-identity = { workspace = true } -multiaddr = { workspace = true } -serde = { workspace = true } -surf-disco = { workspace = true } -tide-disco = { workspace = true } -tokio = { workspace = true } -toml = { workspace = true } -tracing = { workspace = true } -vbs = { workspace = true } - -[lints] -workspace = true diff --git a/crates/orchestrator/README.md b/crates/orchestrator/README.md deleted file mode 100644 index 1bf5ade9c2..0000000000 --- a/crates/orchestrator/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Orchestrator - -This crate implements an orchestrator that coordinates starting the network with a particular configuration. It is useful for testing and benchmarking. Like the web server, the orchestrator is built using [Tide Disco](https://github.com/EspressoSystems/tide-disco). - -To run the orchestrator: `just example orchestrator http://0.0.0.0:3333 ./crates/orchestrator/run-config.toml` \ No newline at end of file diff --git a/crates/orchestrator/api.toml b/crates/orchestrator/api.toml deleted file mode 100644 index 7a18fbb193..0000000000 --- a/crates/orchestrator/api.toml +++ /dev/null @@ -1,104 +0,0 @@ -[meta] -NAME = "orchestrator" -DESCRIPTION = "Orchestrator for HotShot" -FORMAT_VERSION = "0.1.0" - -# POST node's identity -[route.post_identity] -PATH = ["identity"] -METHOD = "POST" -DOC = """ -POST a node's identity (IP address) to the orchestrator. Returns the node's node_index. -""" - -# POST retrieve the network configuration -[route.post_getconfig] -PATH = ["config/:node_index"] -METHOD = "POST" -":node_index" = "Integer" -DOC = """ -Get networking configuration needed for nodes to initialize HotShot and themselves. See `config.rs` for more information. -This must be a POST request so we can update the OrchestratorState in the server accordingly. Must use the node_index previously -received from the 'identity' endpoint -""" - -# POST the latest temporary node index only for generating validator's key pair -[route.get_tmp_node_index] -PATH = ["get_tmp_node_index"] -METHOD = "POST" -DOC = """ -Get the latest temporary node index only for generating validator's key pair for testing in hotshot, later the generated key pairs might be bound with other node_index. -""" - -# POST the node's node index for pubkey and is_da collection -[route.post_pubkey] -PATH = ["pubkey/:is_da"] -METHOD = "POST" -":is_da" = "Boolean" -DOC = """ -Post a node's node_index so that its public key could be posted and collected by the orchestrator. -Supply whether or not we are DA. -""" - -# GET whether or not the config with all peers' public keys / configs are ready -[route.peer_pubconfig_ready] -PATH = ["peer_pub_ready"] -DOC = """ -Get whether the node can collect the final config which includes all peer's public config/info like public keys, returns a boolean. -""" - -# POST the updated config with all peers' public keys / configs -[route.post_config_after_peer_collected] -PATH = ["post_config_after_peer_collected"] -METHOD = "POST" -DOC = """ -Get the updated config with all peers' public keys / configs, returns a NetworkConfig. -""" - -# POST whether the node is ready to begin the run -# TODO ED Use the node index parameter -[route.post_ready] -PATH = ["ready"] -METHOD = "POST" -":node_index" = "Integer" -DOC = """ -Post whether the node with node_index is ready to start the run -""" - -# GET whether or not to start the run -[route.get_start] -PATH = ["start"] -DOC = """ -Get whether the node should start the run, returns a boolean -""" - -# POST the run results -[route.post_results] -PATH = ["results"] -METHOD = "POST" -DOC = """ -Post run results. -""" - -# POST to manually start the run -[route.post_manual_start] -PATH = ["manual_start"] -METHOD = "POST" -DOC = """ -Post whether the orchestrator should start the run immediately, with the nodes that have already registered. -""" - -# GET builder URLs -[route.get_builders] -PATH = ["builders"] -DOC = """ -Get list of builder URLs -""" - -# POST builder URL -[route.post_builder] -PATH = ["builder"] -METHOD = "POST" -DOC = """ -Register a builder URL to orchestrator's pool of builder URLs -""" diff --git a/crates/orchestrator/run-config.toml b/crates/orchestrator/run-config.toml deleted file mode 100644 index 1e5a8700d7..0000000000 --- a/crates/orchestrator/run-config.toml +++ /dev/null @@ -1,68 +0,0 @@ -rounds = 100 -indexed_da = true -transactions_per_round = 10 -transaction_size = 1000 -node_index = 0 -seed = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -cdn_marshal_address = "127.0.0.1:8999" -public_keys = [ - { stake_table_key = "BLS_VER_KEY~bQszS-QKYvUij2g20VqS8asttGSb95NrTu2PUj0uMh1CBUxNy1FqyPDjZqB29M7ZbjWqj79QkEOWkpga84AmDYUeTuWmy-0P1AdKHD3ehc-dKvei78BDj5USwXPJiDUlCxvYs_9rWYhagaq-5_LXENr78xel17spftNd5MA1Mw5U", state_ver_key = "SCHNORR_VER_KEY~lJqDaVZyM0hWP2Br52IX5FeE-dCAIC-dPX7bL5-qUx-vjbunwe-ENOeZxj6FuOyvDCFzoGeP7yZ0fM995qF-CRE", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~IBRoz_Q1EXvcm1pNZcmVlyYZU8hZ7qmy337ePAjEMhz8Hl2q8vWPFOd3BaLwgRS1UzAPW3z4E-XIgRDGcRBTAMZX9b_0lKYjlyTlNF2EZfNnKmvv-xJ0yurkfjiveeYEsD2l5d8q_rJJbH1iZdXy-yPEbwI0SIvQfwdlcaKw9po4", state_ver_key = "SCHNORR_VER_KEY~tyuplKrHzvjODsjPKMHVFYfoMcgklQsMye-2aSCktBcbW_CIzLOq3wZXRIPBbw3FiV6_QoUXYAlpZ5up0zG_ANY", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~4zQnaCOFJ7m95OjxeNls0QOOwWbz4rfxaL3NwmN2zSdnf8t5Nw_dfmMHq05ee8jCegw6Bn5T8inmrnGGAsQJMMWLv77nd7FJziz2ViAbXg-XGGF7o4HyzELCmypDOIYF3X2UWferFE_n72ZX0iQkUhOvYZZ7cfXToXxRTtb_mwRR", state_ver_key = "SCHNORR_VER_KEY~qQAC373HPv4s0mTTpdmSaynfUXC4SfPCuGD2fbeigSpexFB2ycCeXV9UAjuR86CC9udPhopgMsFLyD29VO2iJSg", stake = 1, da = true}, - - { stake_table_key = "BLS_VER_KEY~rO2PIjyY30HGfapFcloFe3mNDKMIFi6JlOLkH5ZWBSYoRm5fE2-Rm6Lp3EvmAcB5r7KFJ0c1Uor308x78r04EY_sfjcsDCWt7RSJdL4cJoD_4fSTCv_bisO8k98hs_8BtqQt8BHlPeJohpUXvcfnK8suXJETiJ6Er97pfxRbzgAL", state_ver_key = "SCHNORR_VER_KEY~le6RHdTasbBsTcbMqArt0XWFwfIJTY7RbUwaCvdxswL8LpXpO3eb86iyYUr63dtv4GGa5fIJaRH97nCd1lV9H8g", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~r6b-Cwzp-b3czlt0MHmYPJIow5kMsXbrNmZsLSYg9RV49oCCO4WEeCRFR02x9bqLCa_sgNFMrIeNdEa11qNiBAohApYFIvrSa-zP5QGj3xbZaMOCrshxYit6E2TR-XsWvv6gjOrypmugjyTAth-iqQzTboSfmO9DD1-gjJIdCaD7", state_ver_key = "SCHNORR_VER_KEY~LfL6fFJQ8UZWR1Jro6LHtKm_y5-VQZBapO0XhcB8ABAmsVght9B8k7NntrgniffAMD8_OJ6Zjg8XUklhbb42CIw", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~kEUEUJFBtCXl68fM_2roQw856wQlu1ZoDmPn8uu4bQgeZwyb5oz5_kMl-oAJ_OtbYV1serjWE--eXB_qYIpQLZka42-cML6WjCQjNl1hGSejtoBDkExNeUNcweFQBbEsaDiIy3-sgHTrfYpFd1icKeAVihLRn5_RtSU_RUu1TQqR", state_ver_key = "SCHNORR_VER_KEY~qKOggsQMNIIvmiIPM3smiGk40kYGXCVupsmgIrf3RgMmua683F3vUwzWcx0s7mxdzLXJwPAB06LD96cxip7JLJM", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~96hAcdFZxQT8CEHcyV8j2ILJRsXagquENPkc9AwLSx3u6AE_uMupIKGbNJRiM99oFneK2vI5g1u61HidWeuTLRPM2537xAXeaO8e-wJYx4FaPKw_xTcLPrIm0OZT7SsLAMwFuqfMbDdKM71-RyrLwhff5517xXBKEk5Tg9iT9Qrr", state_ver_key = "SCHNORR_VER_KEY~y0nltwKyKSpwO3ki9Czu5asjAt5g1Ya3XmAywcerOSUg__FuZOcYq6tKxMsnsjE7ylpWLZv8R5W4-6WkP0DWI94", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~-pVi7j6TEBeG7ABata4uWWDRM2SrY8wWotWsGnTpIhnOVYJI_lNWyig6VJUuFmBsMS8rLMU7nDxDm8SbObxyA-SLFcr_jCkZqsbx8GcVQrnBAfjNRWuPZP0xcTDMu2IkQqtc3L0OpzbMEgGRGE8Wj09pNqouzl-xhPoYjTmD06Bw", state_ver_key = "SCHNORR_VER_KEY~6rPZ_plXxp8Uoh-E8VPb37csDRLND66zAorA3crYOhf9ARJapk8151RRVXWHe5Q2uF_RmQQmOCAov6tIpJ4yHz0", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~IUPSdnsNUHgNx_74ZhBPrICcDZ9Bp_DAt-6kFz8vSwJES2Vy1Ws8NJ1mxb9XGE1u13sw0FRe8kn5Ib3p2stbEtR_1Qgbuif6aoLrGaSUzy0MvwrO58u9kHZk3rXIuSAN7n4ok3-KKk2CmnBfx7fchFoqT56FXCd1EJ7XRrYj8wTh", state_ver_key = "SCHNORR_VER_KEY~qLqeTM1ZT1ecLEpzHwmlr-GeMUOest-kAm5nKOnKnB-W_TRj4IL77lnmamYvUdXR_ddQp24wQh2IlOIdp5jKEgw", stake = 1, da = true }, - - { stake_table_key = "BLS_VER_KEY~PAAQNgOYfj3GiVX7LxSlkXfOCDSnNKZDqPVYQ_jBMxKzOCn0PXbqQ62kKPenWOmCxiCE7X158s-VenBna6MjHJgf61eBAO-3-OyTP5NWVx49RTgHhQf2iMTKk2iqK2gjnjZimBU135YU4lQFtrG-ZgRezwqkC5vy8V-q46fschIG", state_ver_key = "SCHNORR_VER_KEY~APKGX39-mOmApq6jMdIEiuuyddJ_k8xFeIwU1Zs2zShH1rI--eZR180Us5vqNWmK3zAidScvVcW4bAsOMHB3LPg", stake = 1, da = true } -] - -[config] -num_nodes_with_stake = 10 -start_threshold = [8, 10] -staked_da_nodes = 10 -fixed_leader_for_gpuvid = 1 -next_view_timeout = 30000 -num_bootstrap = 5 -epoch_height = 0 - -[random_builder] -txn_in_block = 100 -blocks_per_second = 1 -txn_size = { start = 20, end = 100 } - -[combined_network_config.delay_duration] -secs = 1 -nanos = 0 - -[config.view_sync_timeout] -secs = 2 -nanos = 0 - -[config.data_request_delay] -secs = 0 -nanos = 200_000_000 - -[config.builder_timeout] -secs = 10 -nanos = 0 - -[config.upgrade] -start_proposing_view = 1 -stop_proposing_view = 0 -start_voting_view = 1 -stop_voting_view = 0 -start_proposing_time = 1 -stop_proposing_time = 0 -start_voting_time = 1 -stop_voting_time = 0 diff --git a/crates/orchestrator/src/client.rs b/crates/orchestrator/src/client.rs deleted file mode 100644 index de167ff505..0000000000 --- a/crates/orchestrator/src/client.rs +++ /dev/null @@ -1,522 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -use std::{net::SocketAddr, time::Duration}; - -use clap::Parser; -use futures::{Future, FutureExt}; -use hotshot_types::{ - network::{NetworkConfig, NetworkConfigSource}, - traits::signature_key::SignatureKey, - PeerConfig, ValidatorConfig, -}; -use libp2p_identity::PeerId; -use multiaddr::Multiaddr; -use surf_disco::{error::ClientError, Client}; -use tide_disco::Url; -use tokio::time::sleep; -use tracing::{info, instrument}; -use vbs::BinarySerializer; - -use crate::OrchestratorVersion; - -/// Holds the client connection to the orchestrator -pub struct OrchestratorClient { - /// the client - pub client: surf_disco::Client, -} - -/// Struct describing a benchmark result -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default, PartialEq)] -pub struct BenchResults { - /// Whether it's partial collected results - pub partial_results: String, - /// The average latency of the transactions - pub avg_latency_in_sec: i64, - /// The number of transactions that were latency measured - pub num_latency: i64, - /// The minimum latency of the transactions - pub minimum_latency_in_sec: i64, - /// The maximum latency of the transactions - pub maximum_latency_in_sec: i64, - /// The throughput of the consensus protocol = number of transactions committed per second * transaction size in bytes - pub throughput_bytes_per_sec: u64, - /// The number of transactions committed during benchmarking - pub total_transactions_committed: u64, - /// The size of each transaction in bytes - pub transaction_size_in_bytes: u64, - /// The total time elapsed for benchmarking - pub total_time_elapsed_in_sec: u64, - /// The total number of views during benchmarking - pub total_num_views: usize, - /// The number of failed views during benchmarking - pub failed_num_views: usize, - /// The membership committee type used - pub committee_type: String, -} - -impl BenchResults { - /// printout the results of one example run - pub fn printout(&self) { - println!("====================="); - println!("{0} Benchmark results:", self.partial_results); - println!("Committee type: {}", self.committee_type); - println!( - "Average latency: {} seconds, Minimum latency: {} seconds, Maximum latency: {} seconds", - self.avg_latency_in_sec, self.minimum_latency_in_sec, self.maximum_latency_in_sec - ); - println!("Throughput: {} bytes/sec", self.throughput_bytes_per_sec); - println!( - "Total transactions committed: {}", - self.total_transactions_committed - ); - println!( - "Total number of views: {}, Failed number of views: {}", - self.total_num_views, self.failed_num_views - ); - println!("====================="); - } -} - -/// Struct describing a benchmark result needed for download, also include the config -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default, PartialEq)] -pub struct BenchResultsDownloadConfig { - // Config starting here - /// The commit this benchmark was run on - pub commit_sha: String, - /// Total number of nodes - pub total_nodes: usize, - /// The size of the da committee - pub da_committee_size: usize, - /// The number of fixed_leader_for_gpuvid when we enable the feature [fixed-leader-election] - pub fixed_leader_for_gpuvid: usize, - /// Number of transactions submitted per round - pub transactions_per_round: usize, - /// The size of each transaction in bytes - pub transaction_size: u64, - /// The number of rounds - pub rounds: usize, - - // Results starting here - /// Whether the results are partially collected - /// "One" when the results are collected for one node - /// "Half" when the results are collective for half running nodes if not all nodes terminate successfully - /// "Full" if the results are successfully collected from all nodes - pub partial_results: String, - /// The average latency of the transactions - pub avg_latency_in_sec: i64, - /// The minimum latency of the transactions - pub minimum_latency_in_sec: i64, - /// The maximum latency of the transactions - pub maximum_latency_in_sec: i64, - /// The throughput of the consensus protocol = number of transactions committed per second * transaction size in bytes - pub throughput_bytes_per_sec: u64, - /// The number of transactions committed during benchmarking - pub total_transactions_committed: u64, - /// The total time elapsed for benchmarking - pub total_time_elapsed_in_sec: u64, - /// The total number of views during benchmarking - pub total_num_views: usize, - /// The number of failed views during benchmarking - pub failed_num_views: usize, - /// The membership committee type used - pub committee_type: String, -} - -// VALIDATOR - -#[derive(Parser, Debug, Clone)] -#[command( - name = "Multi-machine consensus", - about = "Simulates consensus among multiple machines" -)] -/// Arguments passed to the validator -pub struct ValidatorArgs { - /// The address the orchestrator runs on - pub url: Url, - /// The optional advertise address to use for Libp2p - pub advertise_address: Option, - /// Optional address to run builder on. Address must be accessible by other nodes - pub builder_address: Option, - /// An optional network config file to save to/load from - /// Allows for rejoining the network on a complete state loss - #[arg(short, long)] - pub network_config_file: Option, -} - -/// arguments to run multiple validators -#[derive(Parser, Debug, Clone)] -pub struct MultiValidatorArgs { - /// Number of validators to run - pub num_nodes: u16, - /// The address the orchestrator runs on - pub url: Url, - /// The optional advertise address to use for Libp2p - pub advertise_address: Option, - /// An optional network config file to save to/load from - /// Allows for rejoining the network on a complete state loss - #[arg(short, long)] - pub network_config_file: Option, -} - -/// Asynchronously retrieves a `NetworkConfig` from an orchestrator. -/// The retrieved one includes correct `node_index` and peer's public config. -/// -/// # Errors -/// If we are unable to get the configuration from the orchestrator -pub async fn get_complete_config( - client: &OrchestratorClient, - mut validator_config: ValidatorConfig, - libp2p_advertise_address: Option, - libp2p_public_key: Option, -) -> anyhow::Result<(NetworkConfig, ValidatorConfig, NetworkConfigSource)> { - // get the configuration from the orchestrator - let run_config: NetworkConfig = client - .post_and_wait_all_public_keys::( - &mut validator_config, - libp2p_advertise_address, - libp2p_public_key, - ) - .await; - - info!( - "Retrieved config; our node index is {}. DA committee member: {}", - run_config.node_index, validator_config.is_da - ); - Ok(( - run_config, - validator_config, - NetworkConfigSource::Orchestrator, - )) -} - -impl ValidatorArgs { - /// Constructs `ValidatorArgs` from `MultiValidatorArgs` and a node index. - /// - /// If `network_config_file` is present in `MultiValidatorArgs`, it appends the node index to it to create a unique file name for each node. - /// - /// # Arguments - /// - /// * `multi_args` - A `MultiValidatorArgs` instance containing the base arguments for the construction. - /// * `node_index` - A `u16` representing the index of the node for which the args are being constructed. - /// - /// # Returns - /// - /// This function returns a new instance of `ValidatorArgs`. - /// - /// # Examples - /// - /// ```ignore - /// // NOTE this is a toy example, - /// // the user will need to construct a multivalidatorargs since `new` does not exist - /// # use hotshot_orchestrator::client::MultiValidatorArgs; - /// let multi_args = MultiValidatorArgs::new(); - /// let node_index = 1; - /// let instance = Self::from_multi_args(multi_args, node_index); - /// ``` - #[must_use] - pub fn from_multi_args(multi_args: MultiValidatorArgs, node_index: u16) -> Self { - Self { - url: multi_args.url, - advertise_address: multi_args.advertise_address, - builder_address: None, - network_config_file: multi_args - .network_config_file - .map(|s| format!("{s}-{node_index}")), - } - } -} - -impl OrchestratorClient { - /// Creates the client that will connect to the orchestrator - #[must_use] - pub fn new(url: Url) -> Self { - let client = surf_disco::Client::::new(url); - // TODO ED: Add healthcheck wait here - OrchestratorClient { client } - } - - /// Get the config from the orchestrator. - /// If the identity is provided, register the identity with the orchestrator. - /// If not, just retrieving the config (for passive observers) - /// - /// # Panics - /// if unable to convert the node index from usize into u64 - /// (only applicable on 32 bit systems) - /// - /// # Errors - /// If we were unable to serialize the Libp2p data - #[allow(clippy::type_complexity)] - pub async fn get_config_without_peer( - &self, - libp2p_advertise_address: Option, - libp2p_public_key: Option, - ) -> anyhow::Result> { - // Serialize our (possible) libp2p-specific data - let request_body = vbs::Serializer::::serialize(&( - libp2p_advertise_address, - libp2p_public_key, - ))?; - - let identity = |client: Client| { - // We need to clone here to move it into the closure - let request_body = request_body.clone(); - async move { - let node_index: Result = client - .post("api/identity") - .body_binary(&request_body) - .expect("failed to set request body") - .send() - .await; - - node_index - } - .boxed() - }; - let node_index = self.wait_for_fn_from_orchestrator(identity).await; - - // get the corresponding config - let f = |client: Client| { - async move { - let config: Result, ClientError> = client - .post(&format!("api/config/{node_index}")) - .send() - .await; - config - } - .boxed() - }; - - let mut config = self.wait_for_fn_from_orchestrator(f).await; - config.node_index = From::::from(node_index); - - Ok(config) - } - - /// Post to the orchestrator and get the latest `node_index` - /// Then return it for the init validator config - /// # Panics - /// if unable to post - #[instrument(skip_all, name = "orchestrator node index for validator config")] - pub async fn get_node_index_for_init_validator_config(&self) -> u16 { - let cur_node_index = |client: Client| { - async move { - let cur_node_index: Result = client - .post("api/get_tmp_node_index") - .send() - .await - .inspect_err(|err| tracing::error!("{err}")); - - cur_node_index - } - .boxed() - }; - self.wait_for_fn_from_orchestrator(cur_node_index).await - } - - /// Requests the configuration from the orchestrator with the stipulation that - /// a successful call requires all nodes to be registered. - /// - /// Does not fail, retries internally until success. - #[instrument(skip_all, name = "orchestrator config")] - pub async fn get_config_after_collection(&self) -> NetworkConfig { - // Define the request for post-register configurations - let get_config_after_collection = |client: Client| { - async move { - let result = client - .post("api/post_config_after_peer_collected") - .send() - .await; - - if let Err(ref err) = result { - tracing::error!("{err}"); - } - - result - } - .boxed() - }; - - // Loop until successful - self.wait_for_fn_from_orchestrator(get_config_after_collection) - .await - } - - /// Registers a builder URL with the orchestrator - /// - /// # Panics - /// if unable to serialize `address` - pub async fn post_builder_addresses(&self, addresses: Vec) { - let send_builder_f = |client: Client| { - let request_body = vbs::Serializer::::serialize(&addresses) - .expect("Failed to serialize request"); - - async move { - let result: Result<_, ClientError> = client - .post("api/builder") - .body_binary(&request_body) - .unwrap() - .send() - .await - .inspect_err(|err| tracing::error!("{err}")); - result - } - .boxed() - }; - self.wait_for_fn_from_orchestrator::<_, _, ()>(send_builder_f) - .await; - } - - /// Requests a builder URL from orchestrator - pub async fn get_builder_addresses(&self) -> Vec { - // Define the request for post-register configurations - let get_builder = |client: Client| { - async move { - let result = client.get("api/builders").send().await; - - if let Err(ref err) = result { - tracing::error!("{err}"); - } - - result - } - .boxed() - }; - - // Loop until successful - self.wait_for_fn_from_orchestrator(get_builder).await - } - - /// Sends my public key to the orchestrator so that it can collect all public keys - /// And get the updated config - /// Blocks until the orchestrator collects all peer's public keys/configs - /// # Panics - /// if unable to post - #[instrument(skip(self), name = "orchestrator public keys")] - pub async fn post_and_wait_all_public_keys( - &self, - validator_config: &mut ValidatorConfig, - libp2p_advertise_address: Option, - libp2p_public_key: Option, - ) -> NetworkConfig { - let pubkey: Vec = PeerConfig::::to_bytes(&validator_config.public_config()).clone(); - let da_requested: bool = validator_config.is_da; - - // Serialize our (possible) libp2p-specific data - let request_body = vbs::Serializer::::serialize(&( - pubkey, - libp2p_advertise_address, - libp2p_public_key, - )) - .expect("failed to serialize request"); - - // register our public key with the orchestrator - let (node_index, is_da): (u64, bool) = loop { - let result = self - .client - .post(&format!("api/pubkey/{da_requested}")) - .body_binary(&request_body) - .expect("Failed to form request") - .send() - .await - .inspect_err(|err| tracing::error!("{err}")); - - if let Ok((index, is_da)) = result { - break (index, is_da); - } - - sleep(Duration::from_millis(250)).await; - }; - - validator_config.is_da = is_da; - - // wait for all nodes' public keys - let wait_for_all_nodes_pub_key = |client: Client| { - async move { - client - .get("api/peer_pub_ready") - .send() - .await - .inspect_err(|err| tracing::error!("{err}")) - } - .boxed() - }; - self.wait_for_fn_from_orchestrator::<_, _, ()>(wait_for_all_nodes_pub_key) - .await; - - let mut network_config = self.get_config_after_collection().await; - - network_config.node_index = node_index; - - network_config - } - - /// Tells the orchestrator this validator is ready to start - /// Blocks until the orchestrator indicates all nodes are ready to start - /// # Panics - /// Panics if unable to post. - #[instrument(skip(self), name = "orchestrator ready signal")] - pub async fn wait_for_all_nodes_ready(&self, peer_config: Vec) -> bool { - let send_ready_f = |client: Client| { - let pk = peer_config.clone(); - async move { - let result: Result<_, ClientError> = client - .post("api/ready") - .body_binary(&pk) - .unwrap() - .send() - .await - .inspect_err(|err| tracing::error!("{err}")); - result - } - .boxed() - }; - self.wait_for_fn_from_orchestrator::<_, _, ()>(send_ready_f) - .await; - - let wait_for_all_nodes_ready_f = |client: Client| { - async move { client.get("api/start").send().await }.boxed() - }; - self.wait_for_fn_from_orchestrator(wait_for_all_nodes_ready_f) - .await - } - - /// Sends the benchmark metrics to the orchestrator - /// # Panics - /// Panics if unable to post - #[instrument(skip_all, name = "orchestrator metrics")] - pub async fn post_bench_results(&self, bench_results: BenchResults) { - let _send_metrics_f: Result<(), ClientError> = self - .client - .post("api/results") - .body_json(&bench_results) - .unwrap() - .send() - .await - .inspect_err(|err| tracing::warn!("{err}")); - } - - /// Generic function that waits for the orchestrator to return a non-error - /// Returns whatever type the given function returns - #[instrument(skip_all, name = "waiting for orchestrator")] - async fn wait_for_fn_from_orchestrator(&self, f: F) -> GEN - where - F: Fn(Client) -> Fut, - Fut: Future>, - { - loop { - let client = self.client.clone(); - let res = f(client).await; - match res { - Ok(x) => break x, - Err(err) => { - tracing::info!("{err}"); - sleep(Duration::from_millis(250)).await; - } - } - } - } -} diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs deleted file mode 100644 index bafdccbe2c..0000000000 --- a/crates/orchestrator/src/lib.rs +++ /dev/null @@ -1,881 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -//! Orchestrator for manipulating nodes and recording results during a run of `HotShot` tests - -/// The orchestrator's clients -pub mod client; - -use std::{ - collections::{HashMap, HashSet}, - fs, - fs::OpenOptions, - io::{self, ErrorKind}, - time::Duration, -}; - -use async_lock::RwLock; -use client::{BenchResults, BenchResultsDownloadConfig}; -use csv::Writer; -use futures::{stream::FuturesUnordered, FutureExt, StreamExt}; -use hotshot_types::{ - network::{BuilderType, NetworkConfig, PublicKeysFile}, - traits::signature_key::{SignatureKey, StakeTableEntryType}, - PeerConfig, -}; -use libp2p_identity::{ - ed25519::{Keypair as EdKeypair, SecretKey}, - Keypair, PeerId, -}; -use multiaddr::Multiaddr; -use surf_disco::Url; -use tide_disco::{ - api::ApiError, - error::ServerError, - method::{ReadState, WriteState}, - Api, App, RequestError, -}; -use vbs::{ - version::{StaticVersion, StaticVersionType}, - BinarySerializer, -}; - -/// Orchestrator is not, strictly speaking, bound to the network; it can have its own versioning. -/// Orchestrator Version (major) -pub const ORCHESTRATOR_MAJOR_VERSION: u16 = 0; -/// Orchestrator Version (minor) -pub const ORCHESTRATOR_MINOR_VERSION: u16 = 1; -/// Orchestrator Version as a type -pub type OrchestratorVersion = - StaticVersion; -/// Orchestrator Version as a type-binding instance -pub const ORCHESTRATOR_VERSION: OrchestratorVersion = StaticVersion {}; - -/// Generate an keypair based on a `seed` and an `index` -/// # Panics -/// This panics if libp2p is unable to generate a secret key from the seed -#[must_use] -pub fn libp2p_generate_indexed_identity(seed: [u8; 32], index: u64) -> Keypair { - let mut hasher = blake3::Hasher::new(); - hasher.update(&seed); - hasher.update(&index.to_le_bytes()); - let new_seed = *hasher.finalize().as_bytes(); - let sk_bytes = SecretKey::try_from_bytes(new_seed).unwrap(); - >::from(sk_bytes).into() -} - -/// The state of the orchestrator -#[derive(Default, Clone)] -#[allow(clippy::struct_excessive_bools)] -struct OrchestratorState { - /// Tracks the latest node index we have generated a configuration for - latest_index: u16, - /// Tracks the latest temporary index we have generated for init validator's key pair - tmp_latest_index: u16, - /// The network configuration - config: NetworkConfig, - /// Whether the network configuration has been updated with all the peer's public keys/configs - peer_pub_ready: bool, - /// A map from public keys to `(node_index, is_da)`. - pub_posted: HashMap, (u64, bool)>, - /// Whether nodes should start their HotShot instances - /// Will be set to true once all nodes post they are ready to start - start: bool, - /// The total nodes that have posted they are ready to start - nodes_connected: HashSet>, - /// The results of the benchmarks - bench_results: BenchResults, - /// The number of nodes that have posted their results - nodes_post_results: u64, - /// Whether the orchestrator can be started manually - manual_start_allowed: bool, - /// Whether we are still accepting new keys for registration - accepting_new_keys: bool, - /// Builder address pool - builders: Vec, - /// whether we are using a fixed stake table, disabling public key registration - fixed_stake_table: bool, -} - -impl OrchestratorState { - /// create a new [`OrchestratorState`] - pub fn new(network_config: NetworkConfig) -> Self { - let mut peer_pub_ready = false; - let mut fixed_stake_table = false; - - if network_config.config.known_nodes_with_stake.is_empty() { - println!("No nodes were loaded from the config file. Nodes will be allowed to register dynamically."); - } else { - println!("Initializing orchestrator with fixed stake table."); - peer_pub_ready = true; - fixed_stake_table = true; - } - - let builders = if matches!(network_config.builder, BuilderType::External) { - network_config.config.builder_urls.clone().into() - } else { - vec![] - }; - - OrchestratorState { - latest_index: 0, - tmp_latest_index: 0, - config: network_config, - peer_pub_ready, - pub_posted: HashMap::new(), - nodes_connected: HashSet::new(), - start: false, - bench_results: BenchResults::default(), - nodes_post_results: 0, - manual_start_allowed: true, - accepting_new_keys: true, - builders, - fixed_stake_table, - } - } - - /// Output the results to a csv file according to orchestrator state - pub fn output_to_csv(&self) { - let output_csv = BenchResultsDownloadConfig { - commit_sha: self.config.commit_sha.clone(), - total_nodes: self.config.config.num_nodes_with_stake.into(), - da_committee_size: self.config.config.da_staked_committee_size, - fixed_leader_for_gpuvid: self.config.config.fixed_leader_for_gpuvid, - transactions_per_round: self.config.transactions_per_round, - transaction_size: self.bench_results.transaction_size_in_bytes, - rounds: self.config.rounds, - partial_results: self.bench_results.partial_results.clone(), - avg_latency_in_sec: self.bench_results.avg_latency_in_sec, - minimum_latency_in_sec: self.bench_results.minimum_latency_in_sec, - maximum_latency_in_sec: self.bench_results.maximum_latency_in_sec, - throughput_bytes_per_sec: self.bench_results.throughput_bytes_per_sec, - total_transactions_committed: self.bench_results.total_transactions_committed, - total_time_elapsed_in_sec: self.bench_results.total_time_elapsed_in_sec, - total_num_views: self.bench_results.total_num_views, - failed_num_views: self.bench_results.failed_num_views, - committee_type: self.bench_results.committee_type.clone(), - }; - // Open the CSV file in append mode - let results_csv_file = OpenOptions::new() - .create(true) - .append(true) // Open in append mode - .open("scripts/benchmarks_results/results.csv") - .unwrap(); - // Open a file for writing - let mut wtr = Writer::from_writer(results_csv_file); - let _ = wtr.serialize(output_csv); - let _ = wtr.flush(); - println!("Results successfully saved in scripts/benchmarks_results/results.csv"); - } -} - -/// An api exposed by the orchestrator -pub trait OrchestratorApi { - /// Post an identity to the orchestrator. Takes in optional - /// arguments so others can identify us on the Libp2p network. - /// # Errors - /// If we were unable to serve the request - fn post_identity( - &mut self, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result; - /// post endpoint for each node's config - /// # Errors - /// if unable to serve - fn post_getconfig(&mut self, _node_index: u16) -> Result, ServerError>; - /// get endpoint for the next available temporary node index - /// # Errors - /// if unable to serve - fn get_tmp_node_index(&mut self) -> Result; - /// post endpoint for each node's public key - /// # Errors - /// if unable to serve - fn register_public_key( - &mut self, - pubkey: &mut Vec, - is_da: bool, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result<(u64, bool), ServerError>; - /// post endpoint for whether or not all peers public keys are ready - /// # Errors - /// if unable to serve - fn peer_pub_ready(&self) -> Result; - /// get endpoint for the network config after all peers public keys are collected - /// # Errors - /// if unable to serve - fn post_config_after_peer_collected(&mut self) -> Result, ServerError>; - /// get endpoint for whether or not the run has started - /// # Errors - /// if unable to serve - fn get_start(&self) -> Result; - /// post endpoint for the results of the run - /// # Errors - /// if unable to serve - fn post_run_results(&mut self, metrics: BenchResults) -> Result<(), ServerError>; - /// A node POSTs its public key to let the orchestrator know that it is ready - /// # Errors - /// if unable to serve - fn post_ready(&mut self, peer_config: &PeerConfig) -> Result<(), ServerError>; - /// post endpoint for manually starting the orchestrator - /// # Errors - /// if unable to serve - fn post_manual_start(&mut self, password_bytes: Vec) -> Result<(), ServerError>; - /// post endpoint for registering a builder with the orchestrator - /// # Errors - /// if unable to serve - fn post_builder(&mut self, builder: Url) -> Result<(), ServerError>; - /// get endpoints for builders - /// # Errors - /// if not all builders are registered yet - fn get_builders(&self) -> Result, ServerError>; -} - -impl OrchestratorState -where - KEY: serde::Serialize + Clone + SignatureKey + 'static, -{ - /// register a node with an unknown public key. - /// this method should be used when we don't have a fixed stake table - fn register_unknown( - &mut self, - pubkey: &mut Vec, - da_requested: bool, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result<(u64, bool), ServerError> { - if let Some((node_index, is_da)) = self.pub_posted.get(pubkey) { - return Ok((*node_index, *is_da)); - } - - if !self.accepting_new_keys { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: - "Network has been started manually, and is no longer registering new keys." - .to_string(), - }); - } - - let node_index = self.pub_posted.len() as u64; - - // Deserialize the public key - let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); - - self.config - .config - .known_nodes_with_stake - .push(staked_pubkey.clone()); - - let mut added_to_da = false; - - let da_full = - self.config.config.known_da_nodes.len() >= self.config.config.da_staked_committee_size; - - #[allow(clippy::nonminimal_bool)] - // We add the node to the DA committee depending on either its node index or whether it requested membership. - // - // Since we issue `node_index` incrementally, if we are deciding DA membership by node_index - // we only need to check that the DA committee is not yet full. - // - // Note: this logically simplifies to (self.config.indexed_da || da_requested) && !da_full, - // but writing it that way makes it a little less clear to me. - if (self.config.indexed_da || (!self.config.indexed_da && da_requested)) && !da_full { - self.config.config.known_da_nodes.push(staked_pubkey); - added_to_da = true; - } - - self.pub_posted - .insert(pubkey.clone(), (node_index, added_to_da)); - - // If the orchestrator is set up for libp2p and we have supplied the proper - // Libp2p data, add our node to the list of bootstrap nodes. - if self.config.libp2p_config.clone().is_some() { - if let (Some(libp2p_public_key), Some(libp2p_address)) = - (libp2p_public_key, libp2p_address) - { - // Push to our bootstrap nodes - self.config - .libp2p_config - .as_mut() - .unwrap() - .bootstrap_nodes - .push((libp2p_public_key, libp2p_address)); - } - } - - tracing::error!("Posted public key for node_index {node_index}"); - - // node_index starts at 0, so once it matches `num_nodes_with_stake` - // we will have registered one node too many. hence, we want `node_index + 1`. - if node_index + 1 >= (self.config.config.num_nodes_with_stake.get() as u64) { - self.peer_pub_ready = true; - self.accepting_new_keys = false; - } - Ok((node_index, added_to_da)) - } - - /// register a node on the fixed stake table, which was loaded at startup - fn register_from_list( - &mut self, - pubkey: &mut Vec, - da_requested: bool, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result<(u64, bool), ServerError> { - // if we've already registered this node before, we just retrieve its info from `pub_posted` - if let Some((node_index, is_da)) = self.pub_posted.get(pubkey) { - return Ok((*node_index, *is_da)); - } - - // Deserialize the public key - let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); - - // Check if the node is allowed to connect, returning its index and config entry if so. - let Some((node_index, node_config)) = - self.config.public_keys.iter().enumerate().find(|keys| { - keys.1.stake_table_key == staked_pubkey.stake_table_entry.public_key() - }) - else { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "You are unauthorized to register with the orchestrator".to_string(), - }); - }; - - // Check that our recorded DA status for the node matches what the node actually requested - if node_config.da != da_requested { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: format!("Mismatch in DA status in registration for node {}. DA requested: {}, expected: {}", node_index, da_requested, node_config.da), - }); - } - - let added_to_da = node_config.da; - - self.pub_posted - .insert(pubkey.clone(), (node_index as u64, added_to_da)); - - // If the orchestrator is set up for libp2p and we have supplied the proper - // Libp2p data, add our node to the list of bootstrap nodes. - if self.config.libp2p_config.clone().is_some() { - if let (Some(libp2p_public_key), Some(libp2p_address)) = - (libp2p_public_key, libp2p_address) - { - // Push to our bootstrap nodes - self.config - .libp2p_config - .as_mut() - .unwrap() - .bootstrap_nodes - .push((libp2p_public_key, libp2p_address)); - } - } - - tracing::error!("Node {node_index} has registered."); - - Ok((node_index as u64, added_to_da)) - } -} - -impl OrchestratorApi for OrchestratorState -where - KEY: serde::Serialize + Clone + SignatureKey + 'static, -{ - /// Post an identity to the orchestrator. Takes in optional - /// arguments so others can identify us on the Libp2p network. - /// # Errors - /// If we were unable to serve the request - fn post_identity( - &mut self, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result { - let node_index = self.latest_index; - self.latest_index += 1; - - if usize::from(node_index) >= self.config.config.num_nodes_with_stake.get() { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Network has reached capacity".to_string(), - }); - } - - // If the orchestrator is set up for libp2p and we have supplied the proper - // Libp2p data, add our node to the list of bootstrap nodes. - if self.config.libp2p_config.clone().is_some() { - if let (Some(libp2p_public_key), Some(libp2p_address)) = - (libp2p_public_key, libp2p_address) - { - // Push to our bootstrap nodes - self.config - .libp2p_config - .as_mut() - .unwrap() - .bootstrap_nodes - .push((libp2p_public_key, libp2p_address)); - } - } - Ok(node_index) - } - - // Assumes nodes will set their own index that they received from the - // 'identity' endpoint - fn post_getconfig(&mut self, _node_index: u16) -> Result, ServerError> { - Ok(self.config.clone()) - } - - // Assumes one node do not get twice - fn get_tmp_node_index(&mut self) -> Result { - let tmp_node_index = self.tmp_latest_index; - self.tmp_latest_index += 1; - - if usize::from(tmp_node_index) >= self.config.config.num_nodes_with_stake.get() { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Node index getter for key pair generation has reached capacity" - .to_string(), - }); - } - Ok(tmp_node_index) - } - - #[allow(clippy::cast_possible_truncation)] - fn register_public_key( - &mut self, - pubkey: &mut Vec, - da_requested: bool, - libp2p_address: Option, - libp2p_public_key: Option, - ) -> Result<(u64, bool), ServerError> { - if self.fixed_stake_table { - self.register_from_list(pubkey, da_requested, libp2p_address, libp2p_public_key) - } else { - self.register_unknown(pubkey, da_requested, libp2p_address, libp2p_public_key) - } - } - - fn peer_pub_ready(&self) -> Result { - if !self.peer_pub_ready { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Peer's public configs are not ready".to_string(), - }); - } - Ok(self.peer_pub_ready) - } - - fn post_config_after_peer_collected(&mut self) -> Result, ServerError> { - if !self.peer_pub_ready { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Peer's public configs are not ready".to_string(), - }); - } - - Ok(self.config.clone()) - } - - fn get_start(&self) -> Result { - // println!("{}", self.start); - if !self.start { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Network is not ready to start".to_string(), - }); - } - Ok(self.start) - } - - // Assumes nodes do not post 'ready' twice - fn post_ready(&mut self, peer_config: &PeerConfig) -> Result<(), ServerError> { - // If we have not disabled registration verification. - // Is this node allowed to connect? - if !self - .config - .config - .known_nodes_with_stake - .contains(peer_config) - { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "You are unauthorized to register with the orchestrator".to_string(), - }); - } - - // `HashSet::insert()` returns whether the node was newly inserted (true) or not - if self.nodes_connected.insert(peer_config.clone()) { - tracing::error!( - "Node {peer_config} connected. Total nodes connected: {}", - self.nodes_connected.len() - ); - } - - // i.e. nodes_connected >= num_nodes_with_stake * (start_threshold.0 / start_threshold.1) - if self.nodes_connected.len() as u64 * self.config.config.start_threshold.1 - >= (self.config.config.num_nodes_with_stake.get() as u64) - * self.config.config.start_threshold.0 - { - self.accepting_new_keys = false; - self.manual_start_allowed = false; - self.start = true; - } - - Ok(()) - } - - /// Manually start the network - fn post_manual_start(&mut self, password_bytes: Vec) -> Result<(), ServerError> { - if !self.manual_start_allowed { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "Configs have already been distributed to nodes, and the network can no longer be started manually.".to_string(), - }); - } - - let password = String::from_utf8(password_bytes) - .expect("Failed to decode raw password as UTF-8 string."); - - // Check that the password matches - if self.config.manual_start_password != Some(password) { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "Incorrect password.".to_string(), - }); - } - - let registered_nodes_with_stake = self.config.config.known_nodes_with_stake.len(); - let registered_da_nodes = self.config.config.known_da_nodes.len(); - - if registered_da_nodes > 1 { - self.config.config.num_nodes_with_stake = - std::num::NonZeroUsize::new(registered_nodes_with_stake) - .expect("Failed to convert to NonZeroUsize; this should be impossible."); - - self.config.config.da_staked_committee_size = registered_da_nodes; - } else { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: format!("We cannot manually start the network, because we only have {registered_nodes_with_stake} nodes with stake registered, with {registered_da_nodes} DA nodes.") - }); - } - - self.accepting_new_keys = false; - self.manual_start_allowed = false; - self.peer_pub_ready = true; - self.start = true; - - Ok(()) - } - - // Aggregates results of the run from all nodes - fn post_run_results(&mut self, metrics: BenchResults) -> Result<(), ServerError> { - if metrics.total_transactions_committed != 0 { - // Deal with the bench results - if self.bench_results.total_transactions_committed == 0 { - self.bench_results = metrics; - } else { - // Deal with the bench results from different nodes - let cur_metrics = self.bench_results.clone(); - self.bench_results.avg_latency_in_sec = (metrics.avg_latency_in_sec - * metrics.num_latency - + cur_metrics.avg_latency_in_sec * cur_metrics.num_latency) - / (metrics.num_latency + cur_metrics.num_latency); - self.bench_results.num_latency += metrics.num_latency; - self.bench_results.minimum_latency_in_sec = metrics - .minimum_latency_in_sec - .min(cur_metrics.minimum_latency_in_sec); - self.bench_results.maximum_latency_in_sec = metrics - .maximum_latency_in_sec - .max(cur_metrics.maximum_latency_in_sec); - self.bench_results.throughput_bytes_per_sec = metrics - .throughput_bytes_per_sec - .max(cur_metrics.throughput_bytes_per_sec); - self.bench_results.total_transactions_committed = metrics - .total_transactions_committed - .max(cur_metrics.total_transactions_committed); - self.bench_results.total_time_elapsed_in_sec = metrics - .total_time_elapsed_in_sec - .max(cur_metrics.total_time_elapsed_in_sec); - self.bench_results.total_num_views = - metrics.total_num_views.min(cur_metrics.total_num_views); - self.bench_results.failed_num_views = - metrics.failed_num_views.max(cur_metrics.failed_num_views); - } - } - self.nodes_post_results += 1; - if self.bench_results.partial_results == "Unset" { - self.bench_results.partial_results = "One".to_string(); - self.bench_results.printout(); - self.output_to_csv(); - } - if self.bench_results.partial_results == "One" - && self.nodes_post_results >= (self.config.config.da_staked_committee_size as u64 / 2) - { - self.bench_results.partial_results = "HalfDA".to_string(); - self.bench_results.printout(); - self.output_to_csv(); - } - if self.bench_results.partial_results == "HalfDA" - && self.nodes_post_results >= (self.config.config.num_nodes_with_stake.get() as u64 / 2) - { - self.bench_results.partial_results = "Half".to_string(); - self.bench_results.printout(); - self.output_to_csv(); - } - if self.bench_results.partial_results != "Full" - && self.nodes_post_results >= (self.config.config.num_nodes_with_stake.get() as u64) - { - self.bench_results.partial_results = "Full".to_string(); - self.bench_results.printout(); - self.output_to_csv(); - } - Ok(()) - } - - fn post_builder(&mut self, builder: Url) -> Result<(), ServerError> { - self.builders.push(builder); - Ok(()) - } - - fn get_builders(&self) -> Result, ServerError> { - if !matches!(self.config.builder, BuilderType::External) - && self.builders.len() != self.config.config.da_staked_committee_size - { - return Err(ServerError { - status: tide_disco::StatusCode::NOT_FOUND, - message: "Not all builders are registered yet".to_string(), - }); - } - Ok(self.builders.clone()) - } -} - -/// Sets up all API routes -#[allow(clippy::too_many_lines)] -fn define_api() -> Result, ApiError> -where - State: 'static + Send + Sync + ReadState + WriteState, - ::State: Send + Sync + OrchestratorApi, - KEY: serde::Serialize + SignatureKey, - VER: StaticVersionType + 'static, -{ - let api_toml = toml::from_str::(include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/api.toml" - ))) - .expect("API file is not valid toml"); - let mut api = Api::::new(api_toml)?; - api.post("post_identity", |req, state| { - async move { - // Read the bytes from the body - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - - // Decode the libp2p data so we can add to our bootstrap nodes (if supplied) - let Ok((libp2p_address, libp2p_public_key)) = - vbs::Serializer::::deserialize(&body_bytes) - else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - - // Call our state function to process the request - state.post_identity(libp2p_address, libp2p_public_key) - } - .boxed() - })? - .post("post_getconfig", |req, state| { - async move { - let node_index = req.integer_param("node_index")?; - state.post_getconfig(node_index) - } - .boxed() - })? - .post("get_tmp_node_index", |_req, state| { - async move { state.get_tmp_node_index() }.boxed() - })? - .post("post_pubkey", |req, state| { - async move { - let is_da = req.boolean_param("is_da")?; - // Read the bytes from the body - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - - // Decode the libp2p data so we can add to our bootstrap nodes (if supplied) - let Ok((mut pubkey, libp2p_address, libp2p_public_key)) = - vbs::Serializer::::deserialize(&body_bytes) - else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - - state.register_public_key(&mut pubkey, is_da, libp2p_address, libp2p_public_key) - } - .boxed() - })? - .get("peer_pubconfig_ready", |_req, state| { - async move { state.peer_pub_ready() }.boxed() - })? - .post("post_config_after_peer_collected", |_req, state| { - async move { state.post_config_after_peer_collected() }.boxed() - })? - .post( - "post_ready", - |req, state: &mut ::State| { - async move { - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - // Decode the payload-supplied pubkey - let Some(pubkey) = PeerConfig::::from_bytes(&body_bytes) else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - state.post_ready(&pubkey) - } - .boxed() - }, - )? - .post( - "post_manual_start", - |req, state: &mut ::State| { - async move { - let password = req.body_bytes(); - state.post_manual_start(password) - } - .boxed() - }, - )? - .get("get_start", |_req, state| { - async move { state.get_start() }.boxed() - })? - .post("post_results", |req, state| { - async move { - let metrics: Result = req.body_json(); - state.post_run_results(metrics.unwrap()) - } - .boxed() - })? - .post("post_builder", |req, state| { - async move { - // Read the bytes from the body - let mut body_bytes = req.body_bytes(); - body_bytes.drain(..12); - - let Ok(urls) = - vbs::Serializer::::deserialize::>(&body_bytes) - else { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "Malformed body".to_string(), - }); - }; - - let mut futures = urls - .into_iter() - .map(|url| async { - let client: surf_disco::Client = - surf_disco::client::Client::builder(url.clone()).build(); - if client.connect(Some(Duration::from_secs(2))).await { - Some(url) - } else { - None - } - }) - .collect::>() - .filter_map(futures::future::ready); - - if let Some(url) = futures.next().await { - state.post_builder(url) - } else { - Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "No reachable addresses".to_string(), - }) - } - } - .boxed() - })? - .get("get_builders", |_req, state| { - async move { state.get_builders() }.boxed() - })?; - Ok(api) -} - -/// Runs the orchestrator -/// # Errors -/// This errors if tide disco runs into an issue during serving -/// # Panics -/// This panics if unable to register the api with tide disco -pub async fn run_orchestrator( - mut network_config: NetworkConfig, - url: Url, -) -> io::Result<()> -where - KEY: SignatureKey + 'static + serde::Serialize, -{ - let env_password = std::env::var("ORCHESTRATOR_MANUAL_START_PASSWORD"); - - if env_password.is_ok() { - tracing::warn!("Took orchestrator manual start password from the environment variable: ORCHESTRATOR_MANUAL_START_PASSWORD={:?}", env_password); - network_config.manual_start_password = env_password.ok(); - } - - // Try to overwrite the network_config public keys - // from the file the env var points to, or panic. - { - let env_public_keys = std::env::var("ORCHESTRATOR_PUBLIC_KEYS"); - - if let Ok(filepath) = env_public_keys { - #[allow(clippy::panic)] - let config_file_as_string: String = fs::read_to_string(filepath.clone()) - .unwrap_or_else(|_| panic!("Could not read config file located at {filepath}")); - - let file: PublicKeysFile = - toml::from_str::>(&config_file_as_string) - .expect("Unable to convert config file to TOML"); - - network_config.public_keys = file.public_keys; - } - } - - network_config.config.known_nodes_with_stake = network_config - .public_keys - .iter() - .map(|keys| PeerConfig { - stake_table_entry: keys.stake_table_key.stake_table_entry(keys.stake), - state_ver_key: keys.state_ver_key.clone(), - }) - .collect(); - - network_config.config.known_da_nodes = network_config - .public_keys - .iter() - .filter(|keys| keys.da) - .map(|keys| PeerConfig { - stake_table_entry: keys.stake_table_key.stake_table_entry(keys.stake), - state_ver_key: keys.state_ver_key.clone(), - }) - .collect(); - - let web_api = - define_api().map_err(|_e| io::Error::new(ErrorKind::Other, "Failed to define api")); - - let state: RwLock> = RwLock::new(OrchestratorState::new(network_config)); - - let mut app = App::>, ServerError>::with_state(state); - app.register_module::("api", web_api.unwrap()) - .expect("Error registering api"); - tracing::error!("listening on {:?}", url); - app.serve(url, ORCHESTRATOR_VERSION).await -} diff --git a/crates/orchestrator/staging-config.toml b/crates/orchestrator/staging-config.toml deleted file mode 100644 index 61c5adb696..0000000000 --- a/crates/orchestrator/staging-config.toml +++ /dev/null @@ -1,76 +0,0 @@ -rounds = 10 -indexed_da = false -transactions_per_round = 10 -manual_start_password = "tuktu6-tohnaX-gihxib" -node_index = 0 -seed = [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 -] -transaction_size = 100 -builder = "Simple" - -[config] -start_threshold = [ 8, 10 ] -num_nodes_with_stake = 10 -staked_da_nodes = 10 -fixed_leader_for_gpuvid = 1 -next_view_timeout = 15_000 -num_bootstrap = 5 -builder_urls = [ "https://builder.staging.testnet.espresso.network/" ] - -[config.view_sync_timeout] -secs = 15 -nanos = 0 - -[config.builder_timeout] -secs = 8 -nanos = 0 - -[config.data_request_delay] -secs = 5 -nanos = 0 - -[config.upgrade] -start_proposing_view = 1 -stop_proposing_view = 0 -start_voting_view = 1 -stop_voting_view = 0 -start_proposing_time = 1 -stop_proposing_time = 0 -start_voting_time = 1 -stop_voting_time = 0 - -[combined_network_config.delay_duration] -secs = 5 -nanos = 0 diff --git a/crates/task-impls/src/consensus/handlers.rs b/crates/task-impls/src/consensus/handlers.rs index 437e066ea0..5d1ae8c7c6 100644 --- a/crates/task-impls/src/consensus/handlers.rs +++ b/crates/task-impls/src/consensus/handlers.rs @@ -70,7 +70,6 @@ pub(crate) async fn handle_quorum_vote_recv< task_state.public_key.clone(), &task_state.membership, vote.data.epoch, - task_state.id, &event, sender, &task_state.upgrade_lock, @@ -91,7 +90,6 @@ pub(crate) async fn handle_quorum_vote_recv< task_state.public_key.clone(), &task_state.membership, vote.data.epoch, - task_state.id, &event, sender, &task_state.upgrade_lock, @@ -134,7 +132,6 @@ pub(crate) async fn handle_timeout_vote_recv< task_state.public_key.clone(), &task_state.membership, vote.data.epoch, - task_state.id, &event, sender, &task_state.upgrade_lock, diff --git a/crates/task-impls/src/consensus/mod.rs b/crates/task-impls/src/consensus/mod.rs index fe25a9ec2e..a0b42e8237 100644 --- a/crates/task-impls/src/consensus/mod.rs +++ b/crates/task-impls/src/consensus/mod.rs @@ -89,9 +89,6 @@ pub struct ConsensusTaskState, V: /// A reference to the metrics trait. pub consensus: OuterConsensus, - /// The node's id - pub id: u64, - /// Lock for a decided upgrade pub upgrade_lock: UpgradeLock, @@ -101,7 +98,7 @@ pub struct ConsensusTaskState, V: impl, V: Versions> ConsensusTaskState { /// Handles a consensus event received on the event stream - #[instrument(skip_all, fields(id = self.id, cur_view = *self.cur_view, cur_epoch = *self.cur_epoch), name = "Consensus replica task", level = "error", target = "ConsensusTaskState")] + #[instrument(skip_all, fields(cur_view = *self.cur_view, cur_epoch = *self.cur_epoch), name = "Consensus replica task", level = "error", target = "ConsensusTaskState")] pub async fn handle( &mut self, event: Arc>, diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index c4c7f0ab46..848a0b111f 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -70,9 +70,6 @@ pub struct DaTaskState, V: Version /// This Nodes private key pub private_key: ::PrivateKey, - /// This state's ID - pub id: u64, - /// This node's storage ref pub storage: Arc>, @@ -82,7 +79,7 @@ pub struct DaTaskState, V: Version impl, V: Versions> DaTaskState { /// main task event handler - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "DA Main Task", level = "error", target = "DaTaskState")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "DA Main Task", level = "error", target = "DaTaskState")] pub async fn handle( &mut self, event: Arc>, @@ -281,7 +278,6 @@ impl, V: Versions> DaTaskState, V: Versions> DaTaskState 1 { diff --git a/crates/task-impls/src/helpers.rs b/crates/task-impls/src/helpers.rs index 251cb049f0..76734570c7 100644 --- a/crates/task-impls/src/helpers.rs +++ b/crates/task-impls/src/helpers.rs @@ -573,7 +573,7 @@ pub(crate) async fn parent_leaf_and_state( /// # Errors /// If any validation or state update fails. #[allow(clippy::too_many_lines)] -#[instrument(skip_all, fields(id = validation_info.id, view = *proposal.data.view_number()))] +#[instrument(skip_all, fields(view = *proposal.data.view_number()))] pub async fn validate_proposal_safety_and_liveness< TYPES: NodeType, I: NodeImplementation, diff --git a/crates/task-impls/src/quorum_proposal/handlers.rs b/crates/task-impls/src/quorum_proposal/handlers.rs index 412e3fb41d..c9e5af67e8 100644 --- a/crates/task-impls/src/quorum_proposal/handlers.rs +++ b/crates/task-impls/src/quorum_proposal/handlers.rs @@ -111,9 +111,6 @@ pub struct ProposalDependencyHandle { /// Lock for a decided upgrade pub upgrade_lock: UpgradeLock, - /// The node's id - pub id: u64, - /// The time this view started pub view_start_time: Instant, @@ -260,7 +257,7 @@ impl ProposalDependencyHandle { /// Publishes a proposal given the [`CommitmentAndMetadata`], [`VidDisperse`] /// and high qc [`hotshot_types::simple_certificate::QuorumCertificate`], /// with optional [`ViewChangeEvidence`]. - #[instrument(skip_all, fields(id = self.id, view_number = *self.view_number, latest_proposed_view = *self.latest_proposed_view))] + #[instrument(skip_all, fields(view_number = *self.view_number, latest_proposed_view = *self.latest_proposed_view))] async fn publish_proposal( &self, commitment_and_metadata: CommitmentAndMetadata, diff --git a/crates/task-impls/src/quorum_proposal/mod.rs b/crates/task-impls/src/quorum_proposal/mod.rs index 9eb75188a3..421129e3da 100644 --- a/crates/task-impls/src/quorum_proposal/mod.rs +++ b/crates/task-impls/src/quorum_proposal/mod.rs @@ -69,9 +69,6 @@ pub struct QuorumProposalTaskState /// Shared consensus task state pub consensus: OuterConsensus, - /// The node's id - pub id: u64, - /// The most recent upgrade certificate this node formed. /// Note: this is ONLY for certificates that have been formed internally, /// so that we can propose with them. @@ -94,14 +91,13 @@ impl, V: Versions> QuorumProposalTaskState { /// Create an event dependency - #[instrument(skip_all, fields(id = self.id, latest_proposed_view = *self.latest_proposed_view), name = "Create event dependency", level = "info")] + #[instrument(skip_all, fields(latest_proposed_view = *self.latest_proposed_view), name = "Create event dependency", level = "info")] fn create_event_dependency( &self, dependency_type: ProposalDependency, view_number: TYPES::View, event_receiver: Receiver>>, ) -> EventDependency>> { - let id = self.id; EventDependency::new( event_receiver, Box::new(move |event| { @@ -163,7 +159,7 @@ impl, V: Versions> let valid = event_view == view_number; if valid { tracing::debug!( - "Dependency {dependency_type:?} is complete for view {event_view:?}, my id is {id:?}!", + "Dependency {dependency_type:?} is complete for view {event_view:?}", ); } valid @@ -270,7 +266,7 @@ impl, V: Versions> /// dependency as already completed. This allows for the task to receive a proposable event /// without losing the data that it received, as the dependency task would otherwise have no /// ability to receive the event and, thus, would never propose. - #[instrument(skip_all, fields(id = self.id, latest_proposed_view = *self.latest_proposed_view), name = "Create dependency task", level = "error")] + #[instrument(skip_all, fields(latest_proposed_view = *self.latest_proposed_view), name = "Create dependency task", level = "error")] async fn create_dependency_task_if_new( &mut self, view_number: TYPES::View, @@ -331,7 +327,6 @@ impl, V: Versions> timeout: self.timeout, formed_upgrade_certificate: self.formed_upgrade_certificate.clone(), upgrade_lock: self.upgrade_lock.clone(), - id: self.id, view_start_time: Instant::now(), highest_qc: self.highest_qc.clone(), epoch_height: self.epoch_height, @@ -344,7 +339,7 @@ impl, V: Versions> } /// Update the latest proposed view number. - #[instrument(skip_all, fields(id = self.id, latest_proposed_view = *self.latest_proposed_view), name = "Update latest proposed view", level = "error")] + #[instrument(skip_all, fields(latest_proposed_view = *self.latest_proposed_view), name = "Update latest proposed view", level = "error")] async fn update_latest_proposed_view(&mut self, new_view: TYPES::View) -> bool { if *self.latest_proposed_view < *new_view { tracing::debug!( @@ -369,7 +364,7 @@ impl, V: Versions> } /// Handles a consensus event received on the event stream - #[instrument(skip_all, fields(id = self.id, latest_proposed_view = *self.latest_proposed_view, epoch = *self.cur_epoch), name = "handle method", level = "error", target = "QuorumProposalTaskState")] + #[instrument(skip_all, fields(latest_proposed_view = *self.latest_proposed_view, epoch = *self.cur_epoch), name = "handle method", level = "error", target = "QuorumProposalTaskState")] pub async fn handle( &mut self, event: Arc>, diff --git a/crates/task-impls/src/quorum_proposal_recv/mod.rs b/crates/task-impls/src/quorum_proposal_recv/mod.rs index f6ed129a6e..a0afc10b6d 100644 --- a/crates/task-impls/src/quorum_proposal_recv/mod.rs +++ b/crates/task-impls/src/quorum_proposal_recv/mod.rs @@ -73,9 +73,6 @@ pub struct QuorumProposalRecvTaskState>>, - /// The node's id - pub id: u64, - /// Lock for a decided upgrade pub upgrade_lock: UpgradeLock, @@ -86,9 +83,6 @@ pub struct QuorumProposalRecvTaskState, V: Versions> { - /// The node's id - pub id: u64, - /// Our public key pub(crate) public_key: TYPES::SignatureKey, @@ -129,7 +123,7 @@ impl, V: Versions> } /// Handles all consensus events relating to propose and vote-enabling events. - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "Consensus replica task", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "Consensus replica task", level = "error")] #[allow(unused_variables)] pub async fn handle( &mut self, @@ -146,7 +140,6 @@ impl, V: Versions> return; } let validation_info = ValidationInfo:: { - id: self.id, public_key: self.public_key.clone(), private_key: self.private_key.clone(), consensus: self.consensus.clone(), diff --git a/crates/task-impls/src/quorum_vote/handlers.rs b/crates/task-impls/src/quorum_vote/handlers.rs index fe11d69745..3cc279123f 100644 --- a/crates/task-impls/src/quorum_vote/handlers.rs +++ b/crates/task-impls/src/quorum_vote/handlers.rs @@ -338,7 +338,7 @@ async fn store_drb_seed_and_result } /// Handles the `QuorumProposalValidated` event. -#[instrument(skip_all, fields(id = task_state.id, view = *proposal.view_number))] +#[instrument(skip_all, fields(view = *proposal.view_number))] pub(crate) async fn handle_quorum_proposal_validated< TYPES: NodeType, I: NodeImplementation, diff --git a/crates/task-impls/src/quorum_vote/mod.rs b/crates/task-impls/src/quorum_vote/mod.rs index 541c1d5fb2..fbeef1a67a 100644 --- a/crates/task-impls/src/quorum_vote/mod.rs +++ b/crates/task-impls/src/quorum_vote/mod.rs @@ -93,9 +93,6 @@ pub struct VoteDependencyHandle, V /// The consensus metrics pub consensus_metrics: Arc, - /// The node's id - pub id: u64, - /// Number of blocks in an epoch, zero means there are no epochs pub epoch_height: u64, } @@ -106,7 +103,7 @@ impl + 'static, V: Versions> Handl type Output = Vec>>; #[allow(clippy::too_many_lines)] - #[instrument(skip_all, fields(id = self.id, view = *self.view_number))] + #[instrument(skip_all, fields(view = *self.view_number))] async fn handle_dep_result(self, res: Self::Output) { let mut payload_commitment = None; let mut leaf = None; @@ -300,9 +297,6 @@ pub struct QuorumVoteTaskState, V: /// Output events to application pub output_event_stream: async_broadcast::Sender>, - /// The node's id - pub id: u64, - /// The consensus metrics pub consensus_metrics: Arc, @@ -318,14 +312,13 @@ pub struct QuorumVoteTaskState, V: impl, V: Versions> QuorumVoteTaskState { /// Create an event dependency. - #[instrument(skip_all, fields(id = self.id, latest_voted_view = *self.latest_voted_view), name = "Quorum vote create event dependency", level = "error")] + #[instrument(skip_all, fields(latest_voted_view = *self.latest_voted_view), name = "Quorum vote create event dependency", level = "error")] fn create_event_dependency( &self, dependency_type: VoteDependency, view_number: TYPES::View, event_receiver: Receiver>>, ) -> EventDependency>> { - let id = self.id; EventDependency::new( event_receiver.clone(), Box::new(move |event| { @@ -355,10 +348,9 @@ impl, V: Versions> QuorumVoteTaskS }; if event_view == view_number { tracing::trace!( - "Vote dependency {:?} completed for view {:?}, my id is {:?}", + "Vote dependency {:?} completed for view {:?}", dependency_type, view_number, - id, ); return true; } @@ -369,7 +361,7 @@ impl, V: Versions> QuorumVoteTaskS /// Create and store an [`AndDependency`] combining [`EventDependency`]s associated with the /// given view number if it doesn't exist. - #[instrument(skip_all, fields(id = self.id, latest_voted_view = *self.latest_voted_view), name = "Quorum vote crete dependency task if new", level = "error")] + #[instrument(skip_all, fields(latest_voted_view = *self.latest_voted_view), name = "Quorum vote crete dependency task if new", level = "error")] fn create_dependency_task_if_new( &mut self, view_number: TYPES::View, @@ -416,7 +408,6 @@ impl, V: Versions> QuorumVoteTaskS sender: event_sender.clone(), receiver: event_receiver.clone().deactivate(), upgrade_lock: self.upgrade_lock.clone(), - id: self.id, epoch_height: self.epoch_height, consensus_metrics: Arc::clone(&self.consensus_metrics), }, @@ -426,7 +417,7 @@ impl, V: Versions> QuorumVoteTaskS } /// Update the latest voted view number. - #[instrument(skip_all, fields(id = self.id, latest_voted_view = *self.latest_voted_view), name = "Quorum vote update latest voted view", level = "error")] + #[instrument(skip_all, fields(latest_voted_view = *self.latest_voted_view), name = "Quorum vote update latest voted view", level = "error")] async fn update_latest_voted_view(&mut self, new_view: TYPES::View) -> bool { if *self.latest_voted_view < *new_view { tracing::debug!( @@ -460,7 +451,7 @@ impl, V: Versions> QuorumVoteTaskS } /// Handle a vote dependent event received on the event stream - #[instrument(skip_all, fields(id = self.id, latest_voted_view = *self.latest_voted_view), name = "Quorum vote handle", level = "error", target = "QuorumVoteTaskState")] + #[instrument(skip_all, fields(latest_voted_view = *self.latest_voted_view), name = "Quorum vote handle", level = "error", target = "QuorumVoteTaskState")] pub async fn handle( &mut self, event: Arc>, diff --git a/crates/task-impls/src/request.rs b/crates/task-impls/src/request.rs index 3e1d75dff2..33bd0a0078 100644 --- a/crates/task-impls/src/request.rs +++ b/crates/task-impls/src/request.rs @@ -75,9 +75,6 @@ pub struct NetworkRequestState> { /// This nodes private/signing key, used to sign requests. pub private_key: ::PrivateKey, - /// The node's id - pub id: u64, - /// A flag indicating that `HotShotEvent::Shutdown` has been received pub shutdown_flag: Arc, @@ -102,7 +99,7 @@ type Signature = impl> TaskState for NetworkRequestState { type Event = HotShotEvent; - #[instrument(skip_all, target = "NetworkRequestState", fields(id = self.id))] + #[instrument(skip_all, target = "NetworkRequestState")] async fn handle_event( &mut self, event: Arc, @@ -223,7 +220,6 @@ impl> NetworkRequestState = spawn(async move { // Do the delay only if primary is up and then start sending if !network.is_primary_down() { @@ -265,9 +261,8 @@ impl> NetworkRequestState { /// This replicas private key private_key: ::PrivateKey, - - /// The node's id - id: u64, } impl NetworkResponseState { @@ -53,14 +50,12 @@ impl NetworkResponseState { membership: Arc>, pub_key: TYPES::SignatureKey, private_key: ::PrivateKey, - id: u64, ) -> Self { Self { consensus, membership, pub_key, private_key, - id, } } @@ -140,7 +135,7 @@ impl NetworkResponseState { /// Get the VID share from consensus storage, or calculate it from the payload for /// the view, if we have the payload. Stores all the shares calculated from the payload /// if the calculation was done - #[instrument(skip_all, target = "NetworkResponseState", fields(id = self.id))] + #[instrument(skip_all, target = "NetworkResponseState")] async fn get_or_calc_vid_share( &self, view: TYPES::View, diff --git a/crates/task-impls/src/rewind.rs b/crates/task-impls/src/rewind.rs index 4f62359aeb..9e30e95c1d 100644 --- a/crates/task-impls/src/rewind.rs +++ b/crates/task-impls/src/rewind.rs @@ -9,7 +9,7 @@ use std::{fs::OpenOptions, io::Write, sync::Arc}; use async_broadcast::{Receiver, Sender}; use async_trait::async_trait; use hotshot_task::task::TaskState; -use hotshot_types::traits::node_implementation::NodeType; +use hotshot_types::{traits::node_implementation::NodeType, utils::mnemonic}; use utils::anytrace::Result; use crate::events::HotShotEvent; @@ -20,8 +20,8 @@ pub struct RewindTaskState { /// All events received by this node since the beginning of time. pub events: Vec>>, - /// The id of this node - pub id: u64, + /// The public key of this node + pub public_key: TYPES::SignatureKey, } impl RewindTaskState { @@ -46,8 +46,8 @@ impl TaskState for RewindTaskState { } fn cancel_subtasks(&mut self) { - tracing::info!("Node ID {} Recording {} events", self.id, self.events.len()); - let filename = format!("rewind_{}.log", self.id); + tracing::info!("Recording {} events", self.events.len()); + let filename = format!("rewind_{}.log", mnemonic(&self.public_key)); let mut file = match OpenOptions::new() .write(true) .create(true) diff --git a/crates/task-impls/src/transactions.rs b/crates/task-impls/src/transactions.rs index cb911e8522..d27c9c557b 100644 --- a/crates/task-impls/src/transactions.rs +++ b/crates/task-impls/src/transactions.rs @@ -109,9 +109,6 @@ pub struct TransactionTaskState, V /// InstanceState pub instance_state: Arc, - /// This state's ID - pub id: u64, - /// Lock for a decided upgrade pub upgrade_lock: UpgradeLock, @@ -151,7 +148,7 @@ impl, V: Versions> TransactionTask } /// legacy view change handler - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view), name = "Transaction task", level = "error", target = "TransactionTaskState")] + #[instrument(skip_all, fields(view = *self.cur_view), name = "Transaction task", level = "error", target = "TransactionTaskState")] pub async fn handle_view_change_legacy( &mut self, event_stream: &Sender>>, @@ -442,7 +439,7 @@ impl, V: Versions> TransactionTask } /// epochs view change handler - #[instrument(skip_all, fields(id = self.id, view_number = *self.cur_view))] + #[instrument(skip_all, fields(view_number = *self.cur_view))] pub async fn handle_view_change_epochs( &mut self, event_stream: &Sender>>, @@ -459,7 +456,7 @@ impl, V: Versions> TransactionTask } /// main task event handler - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "Transaction task", level = "error", target = "TransactionTaskState")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "Transaction task", level = "error", target = "TransactionTaskState")] pub async fn handle( &mut self, event: Arc>, @@ -509,7 +506,7 @@ impl, V: Versions> TransactionTask /// Get VID commitment for the last successful view before `block_view`. /// Returns None if we don't have said commitment recorded. - #[instrument(skip_all, target = "TransactionTaskState", fields(id = self.id, cur_view = *self.cur_view, block_view = *block_view))] + #[instrument(skip_all, target = "TransactionTaskState", fields(cur_view = *self.cur_view, block_view = *block_view))] async fn last_vid_commitment_retry( &self, block_view: TYPES::View, @@ -530,7 +527,7 @@ impl, V: Versions> TransactionTask /// Get VID commitment for the last successful view before `block_view`. /// Returns None if we don't have said commitment recorded. - #[instrument(skip_all, target = "TransactionTaskState", fields(id = self.id, cur_view = *self.cur_view, block_view = *block_view))] + #[instrument(skip_all, target = "TransactionTaskState", fields(cur_view = *self.cur_view, block_view = *block_view))] async fn last_vid_commitment( &self, block_view: TYPES::View, @@ -568,7 +565,7 @@ impl, V: Versions> TransactionTask } } - #[instrument(skip_all, fields(id = self.id, cur_view = *self.cur_view, block_view = *block_view), name = "wait_for_block", level = "error")] + #[instrument(skip_all, fields(cur_view = *self.cur_view, block_view = *block_view), name = "wait_for_block", level = "error")] async fn wait_for_block(&self, block_view: TYPES::View) -> Option> { let task_start_time = Instant::now(); @@ -698,7 +695,7 @@ impl, V: Versions> TransactionTask /// # Errors /// If none of the builder reports any available blocks or claiming block fails for all of the /// builders. - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view), name = "block_from_builder", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view), name = "block_from_builder", level = "error")] async fn block_from_builder( &self, parent_comm: VidCommitment, diff --git a/crates/task-impls/src/upgrade.rs b/crates/task-impls/src/upgrade.rs index 3ff8e073a8..c6aaaa8360 100644 --- a/crates/task-impls/src/upgrade.rs +++ b/crates/task-impls/src/upgrade.rs @@ -62,9 +62,6 @@ pub struct UpgradeTaskState { /// This Nodes private key pub private_key: ::PrivateKey, - /// This state's ID - pub id: u64, - /// View to start proposing an upgrade pub start_proposing_view: u64, @@ -104,7 +101,7 @@ impl UpgradeTaskState { } /// main task event handler - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "Upgrade Task", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "Upgrade Task", level = "error")] pub async fn handle( &mut self, event: Arc>, @@ -241,7 +238,6 @@ impl UpgradeTaskState { self.public_key.clone(), &self.membership, self.cur_epoch, - self.id, &event, &tx, &self.upgrade_lock, diff --git a/crates/task-impls/src/vid.rs b/crates/task-impls/src/vid.rs index 3754e2a01d..51d7e8c3e6 100644 --- a/crates/task-impls/src/vid.rs +++ b/crates/task-impls/src/vid.rs @@ -54,16 +54,13 @@ pub struct VidTaskState> { /// Our Private Key pub private_key: ::PrivateKey, - /// This state's ID - pub id: u64, - /// Number of blocks in an epoch, zero means there are no epochs pub epoch_height: u64, } impl> VidTaskState { /// main task event handler - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "VID Main Task", level = "error", target = "VidTaskState")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "VID Main Task", level = "error", target = "VidTaskState")] pub async fn handle( &mut self, event: Arc>, diff --git a/crates/task-impls/src/view_sync.rs b/crates/task-impls/src/view_sync.rs index 4f40516b3e..2125f62518 100644 --- a/crates/task-impls/src/view_sync.rs +++ b/crates/task-impls/src/view_sync.rs @@ -82,9 +82,6 @@ pub struct ViewSyncTaskState { /// Our Private Key pub private_key: ::PrivateKey, - /// Our node id; for logging - pub id: u64, - /// How many timeouts we've seen in a row; is reset upon a successful view change pub num_timeouts_tracked: u64, @@ -157,9 +154,6 @@ pub struct ViewSyncReplicaTaskState { /// Timeout task handle, when it expires we try the next relay pub timeout_task: Option>, - /// Our node id; for logging - pub id: u64, - /// Membership for the quorum pub membership: Arc>, @@ -192,7 +186,7 @@ impl TaskState for ViewSyncReplicaTaskState ViewSyncTaskState { - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view), name = "View Sync Main Task", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view), name = "View Sync Main Task", level = "error")] #[allow(clippy::type_complexity)] /// Handles incoming events for the main view sync task pub async fn send_to_or_create_replica( @@ -239,7 +233,6 @@ impl ViewSyncTaskState { public_key: self.public_key.clone(), private_key: self.private_key.clone(), view_sync_timeout: self.view_sync_timeout, - id: self.id, upgrade_lock: self.upgrade_lock.clone(), }; @@ -255,7 +248,7 @@ impl ViewSyncTaskState { task_map.insert(view, replica_state); } - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "View Sync Main Task", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "View Sync Main Task", level = "error")] #[allow(clippy::type_complexity)] /// Handles incoming events for the main view sync task pub async fn handle( @@ -323,8 +316,7 @@ impl ViewSyncTaskState { public_key: self.public_key.clone(), membership: Arc::clone(&self.membership), view: vote_view, - id: self.id, - epoch: vote.data.epoch, + epoch: self.cur_epoch, }; let vote_collector = create_vote_accumulator( &info, @@ -372,8 +364,7 @@ impl ViewSyncTaskState { public_key: self.public_key.clone(), membership: Arc::clone(&self.membership), view: vote_view, - id: self.id, - epoch: vote.data.epoch, + epoch: self.cur_epoch, }; let vote_collector = create_vote_accumulator( @@ -421,8 +412,7 @@ impl ViewSyncTaskState { public_key: self.public_key.clone(), membership: Arc::clone(&self.membership), view: vote_view, - id: self.id, - epoch: vote.data.epoch, + epoch: self.cur_epoch, }; let vote_collector = create_vote_accumulator( &info, @@ -530,7 +520,7 @@ impl ViewSyncTaskState { } impl ViewSyncReplicaTaskState { - #[instrument(skip_all, fields(id = self.id, view = *self.cur_view, epoch = *self.cur_epoch), name = "View Sync Replica Task", level = "error")] + #[instrument(skip_all, fields(view = *self.cur_view, epoch = *self.cur_epoch), name = "View Sync Replica Task", level = "error")] /// Handle incoming events for the view sync replica task pub async fn handle( &mut self, diff --git a/crates/task-impls/src/vote_collection.rs b/crates/task-impls/src/vote_collection.rs index 902630794a..3649fe66d9 100644 --- a/crates/task-impls/src/vote_collection.rs +++ b/crates/task-impls/src/vote_collection.rs @@ -63,9 +63,6 @@ pub struct VoteCollectionTaskState< /// The epoch which we are collecting votes for pub epoch: TYPES::Epoch, - /// Node id - pub id: u64, - /// Whether we should check if we are the leader when handling a vote pub transition_indicator: EpochTransitionIndicator, } @@ -187,9 +184,6 @@ pub struct AccumulatorInfo { /// Epoch of the votes we are collecting pub epoch: TYPES::Epoch, - - /// This nodes id - pub id: u64, } /// Generic function for spawning a vote task. Returns the event stream id of the spawned task if created @@ -234,7 +228,6 @@ where accumulator: Some(new_accumulator), view: info.view, epoch: info.epoch, - id: info.id, transition_indicator, }; @@ -263,7 +256,6 @@ pub async fn handle_vote< public_key: TYPES::SignatureKey, membership: &Arc>, epoch: TYPES::Epoch, - id: u64, event: &Arc>, event_stream: &Sender>>, upgrade_lock: &UpgradeLock, @@ -280,7 +272,6 @@ where membership: Arc::clone(membership), view: vote.view_number(), epoch, - id, }; let collector = create_vote_accumulator( &info, diff --git a/crates/testing/src/block_builder/random.rs b/crates/testing/src/block_builder/random.rs index b5c1fb93f7..368ad40691 100644 --- a/crates/testing/src/block_builder/random.rs +++ b/crates/testing/src/block_builder/random.rs @@ -27,7 +27,7 @@ use hotshot_builder_api::v0_1::{ }; use hotshot_example_types::block_types::TestTransaction; use hotshot_types::{ - network::RandomBuilderConfig, + builder::RandomBuilderConfig, traits::{node_implementation::NodeType, signature_key::BuilderSignatureKey}, utils::BuilderCommitment, vid::VidCommitment, diff --git a/crates/testing/src/helpers.rs b/crates/testing/src/helpers.rs index d0a5631fc5..a292c896af 100644 --- a/crates/testing/src/helpers.rs +++ b/crates/testing/src/helpers.rs @@ -102,7 +102,7 @@ pub async fn build_system_handle_from_launcher< .unwrap(); // See whether or not we should be DA - let is_da = node_id < config.da_staked_committee_size as u64; + let is_da = node_id < config.known_da_nodes.len() as u64; // We assign node's public key and stake value rather than read from config file since it's a test let validator_config: ValidatorConfig = @@ -111,14 +111,13 @@ pub async fn build_system_handle_from_launcher< let public_key = validator_config.public_key.clone(); let memberships = Arc::new(RwLock::new(TYPES::Membership::new( - config.known_nodes_with_stake.clone(), + config.known_nodes.clone(), config.known_da_nodes.clone(), ))); SystemContext::init( public_key, private_key, - node_id, config, memberships, network, diff --git a/crates/testing/src/spinning_task.rs b/crates/testing/src/spinning_task.rs index 66438d701d..9f1ac92458 100644 --- a/crates/testing/src/spinning_task.rs +++ b/crates/testing/src/spinning_task.rs @@ -174,11 +174,10 @@ where node_id, 1, // For tests, make the node DA based on its index - node_id < config.da_staked_committee_size as u64, + node_id < config.known_da_nodes.len() as u64, ); TestRunner::add_node_with_config( - node_id, network.clone(), memberships, initializer, @@ -218,8 +217,8 @@ where NodeAction::RestartDown(delay_views) => { let node_id = idx.try_into().unwrap(); if let Some(node) = self.handles.write().await.get_mut(idx) { - tracing::error!("Node {} shutting down", idx); node.handle.shut_down().await; + // For restarted nodes generate the network on correct view let generated_network = (self.channel_generator)(node_id).await; @@ -263,12 +262,12 @@ where node_id, 1, // For tests, make the node DA based on its index - node_id < config.da_staked_committee_size as u64, + node_id < config.known_da_nodes.len() as u64, ); let internal_chan = broadcast(EVENT_CHANNEL_SIZE); + let context = TestRunner::::add_node_with_config_and_channels( - node_id, generated_network.clone(), memberships, initializer, diff --git a/crates/testing/src/test_builder.rs b/crates/testing/src/test_builder.rs index 266dc39b7e..b749ed0e1c 100644 --- a/crates/testing/src/test_builder.rs +++ b/crates/testing/src/test_builder.rs @@ -4,7 +4,7 @@ // You should have received a copy of the MIT License // along with the HotShot repository. If not, see . -use std::{collections::HashMap, num::NonZeroUsize, rc::Rc, sync::Arc, time::Duration}; +use std::{collections::HashMap, rc::Rc, sync::Arc, time::Duration}; use anyhow::{ensure, Result}; use async_lock::RwLock; @@ -65,8 +65,6 @@ pub struct TestDescription, V: Ver /// Whether to skip initializing nodes that will start late, which will catch up later with /// `HotShotInitializer::from_reload` in the spinning task. pub skip_late: bool, - /// number of bootstrap nodes (libp2p usage only) - pub num_bootstrap_nodes: usize, /// Size of the staked DA committee for the test pub da_staked_committee_size: usize, /// overall safety property description @@ -188,7 +186,7 @@ pub async fn create_test_handle< .unwrap(); // See whether or not we should be DA - let is_da = node_id < config.da_staked_committee_size as u64; + let is_da = node_id < config.known_da_nodes.len() as u64; let validator_config: ValidatorConfig = ValidatorConfig::generated_from_seed_indexed([0u8; 32], node_id, 1, is_da); @@ -205,7 +203,6 @@ pub async fn create_test_handle< .spawn_twin_handles( public_key, private_key, - node_id, config, memberships, network, @@ -224,7 +221,6 @@ pub async fn create_test_handle< .spawn_handle( public_key, private_key, - node_id, config, memberships, network, @@ -239,7 +235,6 @@ pub async fn create_test_handle< let hotshot = SystemContext::::new( public_key, private_key, - node_id, config, memberships, network, @@ -300,7 +295,6 @@ impl, V: Versions> TestDescription let num_nodes_with_stake = 100; Self { - num_bootstrap_nodes: num_nodes_with_stake, num_nodes_with_stake, start_nodes: num_nodes_with_stake, overall_safety_properties: OverallSafetyPropertiesDescription:: { @@ -327,7 +321,6 @@ impl, V: Versions> TestDescription pub fn default_multiple_rounds() -> Self { let num_nodes_with_stake = 10; TestDescription:: { - num_bootstrap_nodes: num_nodes_with_stake, num_nodes_with_stake, start_nodes: num_nodes_with_stake, overall_safety_properties: OverallSafetyPropertiesDescription:: { @@ -355,7 +348,6 @@ impl, V: Versions> TestDescription Self { num_nodes_with_stake, start_nodes: num_nodes_with_stake, - num_bootstrap_nodes: num_nodes_with_stake, // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the // following issue. @@ -392,7 +384,6 @@ impl, V: Versions> Default num_nodes_with_stake, start_nodes: num_nodes_with_stake, skip_late: false, - num_bootstrap_nodes: num_nodes_with_stake, da_staked_committee_size: num_nodes_with_stake, spinning_properties: SpinningTaskDescription { node_changes: vec![], @@ -452,7 +443,6 @@ where ) -> TestLauncher { let TestDescription { num_nodes_with_stake, - num_bootstrap_nodes, timing_data, da_staked_committee_size, unreliable_network, @@ -489,20 +479,16 @@ where 0 < da_staked_committee_size, ); let config = HotShotConfig { - start_threshold: (1, 1), - num_nodes_with_stake: NonZeroUsize::new(num_nodes_with_stake).unwrap(), + known_nodes: known_nodes_with_stake, // Currently making this zero for simplicity known_da_nodes, - num_bootstrap: num_bootstrap_nodes, - known_nodes_with_stake, - da_staked_committee_size, fixed_leader_for_gpuvid: 1, next_view_timeout: 500, view_sync_timeout: Duration::from_millis(250), builder_timeout: Duration::from_millis(1000), data_request_delay: Duration::from_millis(200), // Placeholder until we spin up the builder - builder_urls: vec1::vec1![Url::parse("http://localhost:9999").expect("Valid URL")], + builder_urls: vec![Url::parse("http://localhost:9999").expect("Valid URL")], start_proposing_view: u64::MAX, stop_proposing_view: 0, start_voting_view: u64::MAX, @@ -534,7 +520,6 @@ where resource_generator: ResourceGenerators { channel_generator: >::gen_networks( num_nodes_with_stake, - num_bootstrap_nodes, da_staked_committee_size, unreliable_network, secondary_network_delay, diff --git a/crates/testing/src/test_runner.rs b/crates/testing/src/test_runner.rs index 600be48823..4633fe171d 100644 --- a/crates/testing/src/test_runner.rs +++ b/crates/testing/src/test_runner.rs @@ -350,7 +350,7 @@ where Url::parse(&format!("http://localhost:{fallback_builder_port}")).expect("Invalid URL"); let fallback_builder_task = B::start( - config.num_nodes_with_stake.into(), + config.known_nodes.len(), fallback_builder_url.clone(), B::Config::default(), self.launcher.metadata.fallback_builder.changes.clone(), @@ -402,7 +402,7 @@ where // TODO This is only a workaround. Number of nodes changes from epoch to epoch. Builder should be made epoch-aware. let temp_memberships = ::Membership::new( - config.known_nodes_with_stake.clone(), + config.known_nodes.clone(), config.known_da_nodes.clone(), ); let num_nodes = temp_memberships.total_nodes(TYPES::Epoch::new(0)); @@ -426,15 +426,7 @@ where self.next_node_id += 1; tracing::debug!("launch node {}", i); - //let memberships =Arc::new(RwLock::new(::Membership::new( - //config.known_nodes_with_stake.clone(), - //config.known_da_nodes.clone(), - //))); - - config.builder_urls = builder_urls - .clone() - .try_into() - .expect("Non-empty by construction"); + config.builder_urls = builder_urls.clone(); let network = (self.launcher.resource_generator.channel_generator)(node_id).await; let storage = (self.launcher.resource_generator.storage)(node_id); @@ -468,7 +460,7 @@ where LateNodeContextParameters { storage, memberships: ::Membership::new( - config.known_nodes_with_stake.clone(), + config.known_nodes.clone(), config.known_da_nodes.clone(), ), config, @@ -485,17 +477,16 @@ where .unwrap(); // See whether or not we should be DA - let is_da = node_id < config.da_staked_committee_size as u64; + let is_da = node_id < config.known_da_nodes.len() as u64; // We assign node's public key and stake value rather than read from config file since it's a test let validator_config = ValidatorConfig::generated_from_seed_indexed([0u8; 32], node_id, 1, is_da); let hotshot = Self::add_node_with_config( - node_id, network.clone(), ::Membership::new( - config.known_nodes_with_stake.clone(), + config.known_nodes.clone(), config.known_da_nodes.clone(), ), initializer, @@ -518,7 +509,7 @@ where node_id, network, ::Membership::new( - config.known_nodes_with_stake.clone(), + config.known_nodes.clone(), config.known_da_nodes.clone(), ), config, @@ -562,7 +553,7 @@ where ) .await; - match node_id.cmp(&(config.da_staked_committee_size as u64 - 1)) { + match node_id.cmp(&(config.known_da_nodes.len() as u64 - 1)) { std::cmp::Ordering::Less => { if let Some(task) = builder_tasks.pop() { task.start(Box::new(handle.event_stream())) @@ -592,7 +583,6 @@ where /// if unable to initialize the node's `SystemContext` based on the config #[allow(clippy::too_many_arguments)] pub async fn add_node_with_config( - node_id: u64, network: Network, memberships: TYPES::Membership, initializer: HotShotInitializer, @@ -608,7 +598,6 @@ where SystemContext::new( public_key, private_key, - node_id, config, Arc::new(RwLock::new(memberships)), network, @@ -625,7 +614,6 @@ where /// if unable to initialize the node's `SystemContext` based on the config #[allow(clippy::too_many_arguments, clippy::type_complexity)] pub async fn add_node_with_config_and_channels( - node_id: u64, network: Network, memberships: Arc>, initializer: HotShotInitializer, @@ -646,7 +634,6 @@ where SystemContext::new_from_channels( public_key, private_key, - node_id, config, memberships, network, diff --git a/crates/testing/tests/tests_1/block_builder.rs b/crates/testing/tests/tests_1/block_builder.rs index fc29b1c01d..66e262c161 100644 --- a/crates/testing/tests/tests_1/block_builder.rs +++ b/crates/testing/tests/tests_1/block_builder.rs @@ -19,7 +19,7 @@ use hotshot_testing::block_builder::{ BuilderTask, RandomBuilderImplementation, TestBuilderImplementation, }; use hotshot_types::{ - network::RandomBuilderConfig, + builder::RandomBuilderConfig, traits::{ block_contents::vid_commitment, node_implementation::NodeType, signature_key::SignatureKey, BlockPayload, diff --git a/crates/testing/tests/tests_1/gen_key_pair.rs b/crates/testing/tests/tests_1/gen_key_pair.rs deleted file mode 100644 index 301542a0a1..0000000000 --- a/crates/testing/tests/tests_1/gen_key_pair.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -#![allow(clippy::panic)] - -#[cfg(test)] -mod tests { - use core::panic; - use std::{env, fs::File, io::prelude::*}; - - use hotshot::types::{BLSPubKey, SignatureKey}; - use hotshot_types::{validator_config::ValidatorConfigFile, ValidatorConfig}; - #[test] - fn gen_key_pair_gen_from_config_file() { - let config_file = ValidatorConfigFile::from_file("config/ValidatorConfigFile.toml"); - let my_own_validator_config = ValidatorConfig::::from(config_file.clone()); - if config_file.seed == [0u8; 32] && config_file.node_id == 0 { - assert_eq!( - my_own_validator_config.public_key, - ::from_private(&my_own_validator_config.private_key) - ); - } - - let current_working_dir = match env::current_dir() { - Ok(dir) => dir, - Err(e) => { - panic!("get_current_working_dir error: {:?}", e); - } - }; - let filename = current_working_dir.into_os_string().into_string().unwrap() - + "/../../config/ValidatorConfigOutput"; - match File::create(filename) { - Err(why) => panic!("couldn't create file for output key pairs: {}", why), - Ok(mut file) => match write!(file, "{my_own_validator_config:?}",) { - Err(why) => panic!("couldn't generate key pairs and write to the file: {}", why), - Ok(()) => println!("successfully wrote to file for output key pairs"), - }, - } - } -} diff --git a/crates/testing/tests/tests_1/network_task.rs b/crates/testing/tests/tests_1/network_task.rs index a23ae64cd5..f4b2404b03 100644 --- a/crates/testing/tests/tests_1/network_task.rs +++ b/crates/testing/tests/tests_1/network_task.rs @@ -56,7 +56,7 @@ async fn test_network_task() { let validator_config = launcher.resource_generator.validator_config.clone(); let public_key = validator_config.public_key; - let all_nodes = config.known_nodes_with_stake.clone(); + let all_nodes = config.known_nodes.clone(); let membership = Arc::new(RwLock::new(::Membership::new( all_nodes.clone(), @@ -227,7 +227,7 @@ async fn test_network_storage_fail() { let config = launcher.resource_generator.config.clone(); let validator_config = launcher.resource_generator.validator_config.clone(); let public_key = validator_config.public_key; - let all_nodes = config.known_nodes_with_stake.clone(); + let all_nodes = config.known_nodes.clone(); let upgrade_lock = UpgradeLock::::new(); let membership = Arc::new(RwLock::new(::Membership::new( diff --git a/crates/testing/tests/tests_1/test_success.rs b/crates/testing/tests/tests_1/test_success.rs index e3b7441fa7..d962844b9b 100644 --- a/crates/testing/tests/tests_1/test_success.rs +++ b/crates/testing/tests/tests_1/test_success.rs @@ -240,7 +240,6 @@ cross_tests!( Metadata: { let mut metadata = TestDescription::default_more_nodes(); metadata.epoch_height = 0; - metadata.num_bootstrap_nodes = 10; metadata.num_nodes_with_stake = 12; metadata.da_staked_committee_size = 12; metadata.start_nodes = 12; @@ -261,7 +260,6 @@ cross_tests!( Ignore: false, Metadata: { let mut metadata = TestDescription::default_more_nodes(); - metadata.num_bootstrap_nodes = 10; metadata.epoch_height = 10; metadata.num_nodes_with_stake = 12; metadata.da_staked_committee_size = 12; @@ -290,7 +288,6 @@ cross_tests!( ), num_nodes_with_stake: 11, start_nodes: 11, - num_bootstrap_nodes: 11, da_staked_committee_size: 11, epoch_height: 10, ..TestDescription::default() diff --git a/crates/testing/tests/tests_1/test_with_failures_2.rs b/crates/testing/tests/tests_1/test_with_failures_2.rs index cf9ac8c83f..ef291f89dc 100644 --- a/crates/testing/tests/tests_1/test_with_failures_2.rs +++ b/crates/testing/tests/tests_1/test_with_failures_2.rs @@ -43,7 +43,6 @@ cross_tests!( Metadata: { let mut metadata = TestDescription::default_more_nodes(); metadata.epoch_height = 0; - metadata.num_bootstrap_nodes = 10; metadata.num_nodes_with_stake = 12; metadata.da_staked_committee_size = 12; metadata.start_nodes = 12; @@ -127,7 +126,6 @@ cross_tests!( Metadata: { let mut metadata = TestDescription::default_more_nodes(); metadata.epoch_height = 0; - metadata.num_bootstrap_nodes = 10; metadata.num_nodes_with_stake = 12; metadata.da_staked_committee_size = 12; metadata.start_nodes = 12; diff --git a/crates/testing/tests/tests_1/vote_dependency_handle.rs b/crates/testing/tests/tests_1/vote_dependency_handle.rs index 51c91750fb..7a5afa2518 100644 --- a/crates/testing/tests/tests_1/vote_dependency_handle.rs +++ b/crates/testing/tests/tests_1/vote_dependency_handle.rs @@ -96,7 +96,6 @@ async fn test_vote_dependency_handle() { sender: event_sender.clone(), receiver: event_receiver.clone().deactivate(), upgrade_lock: handle.hotshot.upgrade_lock.clone(), - id: handle.hotshot.id, epoch_height: handle.hotshot.config.epoch_height, }; diff --git a/crates/testing/tests/tests_3/test_with_failures_half_f.rs b/crates/testing/tests/tests_3/test_with_failures_half_f.rs index 3dd8e3ceb7..28d00fe1f9 100644 --- a/crates/testing/tests/tests_3/test_with_failures_half_f.rs +++ b/crates/testing/tests/tests_3/test_with_failures_half_f.rs @@ -24,7 +24,6 @@ cross_tests!( Metadata: { let mut metadata = TestDescription::default_more_nodes(); metadata.epoch_height = 0; - metadata.num_bootstrap_nodes = 17; // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the // following issue. @@ -62,7 +61,6 @@ cross_tests!( Metadata: { let mut metadata = TestDescription::default_more_nodes(); metadata.epoch_height = 0; - metadata.num_bootstrap_nodes = 17; metadata.epoch_height = 10; // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the diff --git a/crates/testing/tests/tests_4/test_with_failures_f.rs b/crates/testing/tests/tests_4/test_with_failures_f.rs index 04382e8344..cc33f997ee 100644 --- a/crates/testing/tests/tests_4/test_with_failures_f.rs +++ b/crates/testing/tests/tests_4/test_with_failures_f.rs @@ -27,7 +27,6 @@ cross_tests!( metadata.overall_safety_properties.num_failed_views = 6; // Make sure we keep committing rounds after the bad leaders, but not the full 50 because of the numerous timeouts metadata.overall_safety_properties.num_successful_views = 20; - metadata.num_bootstrap_nodes = 14; // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the // following issue. diff --git a/crates/testing/tests/tests_5/combined_network.rs b/crates/testing/tests/tests_5/combined_network.rs index e47bf9ad2f..80d84c85eb 100644 --- a/crates/testing/tests/tests_5/combined_network.rs +++ b/crates/testing/tests/tests_5/combined_network.rs @@ -235,7 +235,6 @@ async fn test_stress_combined_network_fuzzy() { hotshot::helpers::initialize_logging(); let mut metadata: TestDescription = TestDescription { - num_bootstrap_nodes: 10, num_nodes_with_stake: 20, start_nodes: 20, diff --git a/crates/testing/tests/tests_5/test_with_failures.rs b/crates/testing/tests/tests_5/test_with_failures.rs index 349007d152..01bc01bbee 100644 --- a/crates/testing/tests/tests_5/test_with_failures.rs +++ b/crates/testing/tests/tests_5/test_with_failures.rs @@ -25,7 +25,6 @@ cross_tests!( Metadata: { let mut metadata = TestDescription::default_more_nodes(); metadata.epoch_height = 0; - metadata.num_bootstrap_nodes = 19; // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the // following issue. diff --git a/crates/testing/tests/tests_5/timeout.rs b/crates/testing/tests/tests_5/timeout.rs index 54f6e267d2..69ed531838 100644 --- a/crates/testing/tests/tests_5/timeout.rs +++ b/crates/testing/tests/tests_5/timeout.rs @@ -86,7 +86,6 @@ async fn test_timeout_libp2p() { let mut metadata: TestDescription = TestDescription { num_nodes_with_stake: 10, start_nodes: 10, - num_bootstrap_nodes: 10, ..Default::default() }; let dead_nodes = vec![ChangeNode { diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index e99de2e53d..6ebacde892 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -18,7 +18,6 @@ async-trait = { workspace = true } bincode = { workspace = true } bitvec = { workspace = true } blake3 = { workspace = true } -clap = { workspace = true } committable = { workspace = true } derive_more = { workspace = true, features = ["debug"] } digest = { workspace = true, features = ["rand_core"] } @@ -31,17 +30,13 @@ jf-signature = { workspace = true, features = ["bls", "schnorr"] } jf-utils = { workspace = true } jf-vid = { workspace = true } lazy_static = { workspace = true } -libp2p-identity = { workspace = true } memoize = { workspace = true } mnemonic = "1" -multiaddr = { workspace = true } primitive-types = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } serde = { workspace = true } -serde-inline-default = { workspace = true } serde_bytes = { workspace = true } -serde_json = { workspace = true } sha2 = { workspace = true } tagged-base64 = { workspace = true } thiserror = { workspace = true } diff --git a/crates/types/src/builder.rs b/crates/types/src/builder.rs new file mode 100644 index 0000000000..328de97f8e --- /dev/null +++ b/crates/types/src/builder.rs @@ -0,0 +1,20 @@ +/// Options controlling how the random builder generates blocks +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] +pub struct RandomBuilderConfig { + /// How many transactions to include in a block + pub txn_in_block: u64, + /// How many blocks to generate per second + pub blocks_per_second: u32, + /// Range of how big a transaction can be (in bytes) + pub txn_size: std::ops::Range, +} + +impl Default for RandomBuilderConfig { + fn default() -> Self { + Self { + txn_in_block: 100, + blocks_per_second: 1, + txn_size: 20..100, + } + } +} diff --git a/crates/types/src/constants.rs b/crates/types/src/constants.rs index 6a1969ae31..5634071f05 100644 --- a/crates/types/src/constants.rs +++ b/crates/types/src/constants.rs @@ -17,9 +17,6 @@ pub const BUNDLE_FETCH_TIMEOUT: Duration = Duration::from_millis(500); /// the number of views to gather information for ahead of time pub const LOOK_AHEAD: u64 = 5; -/// the default kademlia record republication interval (in seconds) -pub const KAD_DEFAULT_REPUB_INTERVAL_SEC: u64 = 28800; - /// the number of messages to cache in the combined network pub const COMBINED_NETWORK_CACHE_SIZE: usize = 200_000; diff --git a/crates/types/src/hotshot_config_file.rs b/crates/types/src/hotshot_config_file.rs index 53633d5d7a..f9a0b2623d 100644 --- a/crates/types/src/hotshot_config_file.rs +++ b/crates/types/src/hotshot_config_file.rs @@ -4,10 +4,9 @@ // You should have received a copy of the MIT License // along with the HotShot repository. If not, see . -use std::{num::NonZeroUsize, time::Duration}; +use std::time::Duration; use url::Url; -use vec1::Vec1; use crate::{ constants::REQUEST_DATA_DELAY, traits::signature_key::SignatureKey, @@ -15,42 +14,32 @@ use crate::{ }; /// Default builder URL, used as placeholder -fn default_builder_urls() -> Vec1 { - vec1::vec1![Url::parse("http://0.0.0.0:3311").unwrap()] +fn default_builder_urls() -> Vec { + vec![Url::parse("http://0.0.0.0:3311").unwrap()] } -/// Holds configuration for a `HotShot` +/// Contains configuration values for `HotShot` #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(bound(deserialize = ""))] pub struct HotShotConfigFile { - /// The proportion of nodes required before the orchestrator issues the ready signal, - /// expressed as (numerator, denominator) - pub start_threshold: (u64, u64), - /// Total number of staked nodes in the network - pub num_nodes_with_stake: NonZeroUsize, - #[serde(skip)] - /// The known nodes' public key and stake value - pub known_nodes_with_stake: Vec>, - #[serde(skip)] + /// The known nodes' public key and stake values + #[serde(rename = "known_nodes", alias = "known_nodes_with_stake")] + pub known_nodes: Vec>, /// The known DA nodes' public key and stake values pub known_da_nodes: Vec>, - /// Number of staking DA nodes - pub staked_da_nodes: usize, /// Number of fixed leaders for GPU VID pub fixed_leader_for_gpuvid: usize, - /// Base duration for next-view timeout, in milliseconds + /// The duration for the timeout before the next view, in milliseconds pub next_view_timeout: u64, - /// Duration for view sync round timeout + /// The duration for view sync round timeout pub view_sync_timeout: Duration, - /// Number of network bootstrap nodes - pub num_bootstrap: usize, /// The maximum amount of time a leader can wait to get a block from a builder pub builder_timeout: Duration, /// Time to wait until we request data associated with a proposal pub data_request_delay: Option, /// Builder API base URL #[serde(default = "default_builder_urls")] - pub builder_urls: Vec1, + pub builder_urls: Vec, /// Upgrade config pub upgrade: UpgradeConfig, /// Number of blocks in an epoch, zero means there are no epochs @@ -60,15 +49,11 @@ pub struct HotShotConfigFile { impl From> for HotShotConfig { fn from(val: HotShotConfigFile) -> Self { HotShotConfig { - start_threshold: val.start_threshold, - num_nodes_with_stake: val.num_nodes_with_stake, known_da_nodes: val.known_da_nodes, - known_nodes_with_stake: val.known_nodes_with_stake, - da_staked_committee_size: val.staked_da_nodes, + known_nodes: val.known_nodes, fixed_leader_for_gpuvid: val.fixed_leader_for_gpuvid, next_view_timeout: val.next_view_timeout, view_sync_timeout: val.view_sync_timeout, - num_bootstrap: val.num_bootstrap, builder_timeout: val.builder_timeout, data_request_delay: val .data_request_delay @@ -114,15 +99,11 @@ impl HotShotConfigFile { .collect(); Self { - num_nodes_with_stake: NonZeroUsize::new(10).unwrap(), - start_threshold: (1, 1), - known_nodes_with_stake: gen_known_nodes_with_stake, - staked_da_nodes, + known_nodes: gen_known_nodes_with_stake, known_da_nodes, fixed_leader_for_gpuvid: 1, next_view_timeout: 10000, view_sync_timeout: Duration::from_millis(1000), - num_bootstrap: 5, builder_timeout: Duration::from_secs(10), data_request_delay: Some(Duration::from_millis(REQUEST_DATA_DELAY)), builder_urls: default_builder_urls(), diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 93076491e1..1278b897ff 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -5,7 +5,7 @@ // along with the HotShot repository. If not, see . //! Types and Traits for the `HotShot` consensus module -use std::{fmt::Debug, future::Future, num::NonZeroUsize, pin::Pin, time::Duration}; +use std::{fmt::Debug, future::Future, pin::Pin, time::Duration}; use bincode::Options; use displaydoc::Display; @@ -13,7 +13,6 @@ use light_client::StateVerKey; use tracing::error; use traits::signature_key::SignatureKey; use url::Url; -use vec1::Vec1; use crate::utils::bincode_opts; pub mod bundle; @@ -39,11 +38,12 @@ pub mod simple_vote; pub mod stake_table; pub mod traits; +/// Holds test builder configuration structures +pub mod builder; + /// Holds the upgrade configuration specification for HotShot nodes. pub mod upgrade_config; pub mod utils; -/// Holds the validator configuration specification for HotShot nodes. -pub mod validator_config; pub mod vid; pub mod vote; @@ -164,32 +164,23 @@ impl Default for PeerConfig { #[derive(Clone, derive_more::Debug, serde::Serialize, serde::Deserialize)] #[serde(bound(deserialize = ""))] pub struct HotShotConfig { - /// The proportion of nodes required before the orchestrator issues the ready signal, - /// expressed as (numerator, denominator) - pub start_threshold: (u64, u64), - /// Total number of nodes in the network - // Earlier it was total_nodes - pub num_nodes_with_stake: NonZeroUsize, /// List of known node's public keys and stake value for certificate aggregation, serving as public parameter - pub known_nodes_with_stake: Vec>, + #[serde(rename = "known_nodes", alias = "known_nodes_with_stake")] + pub known_nodes: Vec>, /// All public keys known to be DA nodes pub known_da_nodes: Vec>, - /// List of DA committee (staking)nodes for static DA committee - pub da_staked_committee_size: usize, /// Number of fixed leaders for GPU VID, normally it will be 0, it's only used when running GPU VID pub fixed_leader_for_gpuvid: usize, - /// Base duration for next-view timeout, in milliseconds + /// The duration for the timeout before the next view, in milliseconds pub next_view_timeout: u64, - /// Duration of view sync round timeouts + /// The duration for view sync round timeouts pub view_sync_timeout: Duration, - /// Number of network bootstrap nodes - pub num_bootstrap: usize, /// The maximum amount of time a leader can wait to get a block from a builder pub builder_timeout: Duration, - /// time to wait until we request data associated with a proposal + /// Time to wait until we request data associated with a proposal pub data_request_delay: Duration, /// Builder API base URL - pub builder_urls: Vec1, + pub builder_urls: Vec, /// View to start proposing an upgrade pub start_proposing_view: u64, /// View to stop proposing an upgrade. To prevent proposing an upgrade, set stop_proposing_view <= start_proposing_view. diff --git a/crates/types/src/network.rs b/crates/types/src/network.rs index f2d35b6984..832dfb40aa 100644 --- a/crates/types/src/network.rs +++ b/crates/types/src/network.rs @@ -4,387 +4,13 @@ // You should have received a copy of the MIT License // along with the HotShot repository. If not, see . -use std::{fs, ops::Range, path::Path, time::Duration, vec}; +use std::time::Duration; -use clap::ValueEnum; -use libp2p_identity::PeerId; -use multiaddr::Multiaddr; -use serde_inline_default::serde_inline_default; -use thiserror::Error; -use tracing::error; - -use crate::{ - constants::{ - ORCHESTRATOR_DEFAULT_NUM_ROUNDS, ORCHESTRATOR_DEFAULT_TRANSACTIONS_PER_ROUND, - ORCHESTRATOR_DEFAULT_TRANSACTION_SIZE, REQUEST_DATA_DELAY, - }, - hotshot_config_file::HotShotConfigFile, - light_client::StateVerKey, - traits::signature_key::SignatureKey, - HotShotConfig, ValidatorConfig, -}; - -/// Configuration describing a libp2p node -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -pub struct Libp2pConfig { - /// The bootstrap nodes to connect to (multiaddress, serialized public key) - pub bootstrap_nodes: Vec<(PeerId, Multiaddr)>, -} - -/// configuration for combined network +/// The configuration for the HotShot network #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -pub struct CombinedNetworkConfig { - /// delay duration before sending a message through the secondary network - pub delay_duration: Duration, -} - -/// a network configuration error -#[derive(Error, Debug)] -pub enum NetworkConfigError { - /// Failed to read NetworkConfig from file - #[error("Failed to read NetworkConfig from file")] - ReadFromFileError(std::io::Error), - /// Failed to deserialize loaded NetworkConfig - #[error("Failed to deserialize loaded NetworkConfig")] - DeserializeError(serde_json::Error), - /// Failed to write NetworkConfig to file - #[error("Failed to write NetworkConfig to file")] - WriteToFileError(std::io::Error), - /// Failed to serialize NetworkConfig - #[error("Failed to serialize NetworkConfig")] - SerializeError(serde_json::Error), - /// Failed to recursively create path to NetworkConfig - #[error("Failed to recursively create path to NetworkConfig")] - FailedToCreatePath(std::io::Error), -} - -#[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize, Default, ValueEnum)] -/// configuration for builder type to use -pub enum BuilderType { - /// Use external builder, [config.builder_url] must be - /// set to correct builder address - External, - #[default] - /// Simple integrated builder will be started and used by each hotshot node - Simple, - /// Random integrated builder will be started and used by each hotshot node - Random, -} - -/// Node PeerConfig keys -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -#[serde(bound(deserialize = ""))] -pub struct PeerConfigKeys { - /// The peer's public key - pub stake_table_key: KEY, - /// the peer's state public key - pub state_ver_key: StateVerKey, - /// the peer's stake - pub stake: u64, - /// whether the node is a DA node - pub da: bool, -} - -/// Options controlling how the random builder generates blocks -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -pub struct RandomBuilderConfig { - /// How many transactions to include in a block - pub txn_in_block: u64, - /// How many blocks to generate per second - pub blocks_per_second: u32, - /// Range of how big a transaction can be (in bytes) - pub txn_size: Range, -} - -impl Default for RandomBuilderConfig { - fn default() -> Self { - Self { - txn_in_block: 100, - blocks_per_second: 1, - txn_size: 20..100, - } - } -} - -/// a network configuration -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -#[serde(bound(deserialize = ""))] -pub struct NetworkConfig { - /// number of views to run - pub rounds: usize, - /// whether DA membership is determined by index. - /// if true, the first k nodes to register form the DA committee - /// if false, DA membership is requested by the nodes - pub indexed_da: bool, - /// number of transactions per view - pub transactions_per_round: usize, - /// password to have the orchestrator start the network, - /// regardless of the number of nodes connected. - pub manual_start_password: Option, - /// number of bootstrap nodes - pub num_bootrap: usize, - /// timeout before starting the next view +pub struct NetworkConfig { + /// The timeout before starting the next view pub next_view_timeout: u64, - /// timeout before starting next view sync round + /// The timeout before starting next view sync round pub view_sync_timeout: Duration, - /// The maximum amount of time a leader can wait to get a block from a builder - pub builder_timeout: Duration, - /// time to wait until we request data associated with a proposal - pub data_request_delay: Duration, - /// global index of node (for testing purposes a uid) - pub node_index: u64, - /// unique seed (for randomness? TODO) - pub seed: [u8; 32], - /// size of transactions - pub transaction_size: usize, - /// name of the key type (for debugging) - pub key_type_name: String, - /// the libp2p config - pub libp2p_config: Option, - /// the hotshot config - pub config: HotShotConfig, - /// The address for the Push CDN's "marshal", A.K.A. load balancer - pub cdn_marshal_address: Option, - /// combined network config - pub combined_network_config: Option, - /// the commit this run is based on - pub commit_sha: String, - /// builder to use - pub builder: BuilderType, - /// random builder config - pub random_builder: Option, - /// The list of public keys that are allowed to connect to the orchestrator - pub public_keys: Vec>, -} - -/// the source of the network config -pub enum NetworkConfigSource { - /// we source the network configuration from the orchestrator - Orchestrator, - /// we source the network configuration from a config file on disk - File, -} - -impl NetworkConfig { - /// Get a temporary node index for generating a validator config - #[must_use] - pub fn generate_init_validator_config(cur_node_index: u16, is_da: bool) -> ValidatorConfig { - // This cur_node_index is only used for key pair generation, it's not bound with the node, - // later the node with the generated key pair will get a new node_index from orchestrator. - ValidatorConfig::generated_from_seed_indexed([0u8; 32], cur_node_index.into(), 1, is_da) - } - - /// Loads a `NetworkConfig` from a file. - /// - /// This function takes a file path as a string, reads the file, and then deserializes the contents into a `NetworkConfig`. - /// - /// # Arguments - /// - /// * `file` - A string representing the path to the file from which to load the `NetworkConfig`. - /// - /// # Returns - /// - /// This function returns a `Result` that contains a `NetworkConfig` if the file was successfully read and deserialized, or a `NetworkConfigError` if an error occurred. - /// - /// # Errors - /// - /// This function will return an error if the file cannot be read or if the contents cannot be deserialized into a `NetworkConfig`. - /// - /// # Examples - /// - /// ```ignore - /// # use hotshot_orchestrator::config::NetworkConfig; - /// # use hotshot_types::signature_key::BLSPubKey; - /// // # use hotshot::traits::election::static_committee::StaticElectionConfig; - /// let file = "/path/to/my/config".to_string(); - /// // NOTE: broken due to staticelectionconfig not being importable - /// // cannot import staticelectionconfig from hotshot without creating circular dependency - /// // making this work probably involves the `types` crate implementing a dummy - /// // electionconfigtype just to make this example work - /// let config = NetworkConfig::::from_file(file).unwrap(); - /// ``` - pub fn from_file(file: String) -> Result { - // read from file - let data = match fs::read(file) { - Ok(data) => data, - Err(e) => { - return Err(NetworkConfigError::ReadFromFileError(e)); - } - }; - - // deserialize - match serde_json::from_slice(&data) { - Ok(data) => Ok(data), - Err(e) => Err(NetworkConfigError::DeserializeError(e)), - } - } - - /// Serializes the `NetworkConfig` and writes it to a file. - /// - /// This function takes a file path as a string, serializes the `NetworkConfig` into JSON format using `serde_json` and then writes the serialized data to the file. - /// - /// # Arguments - /// - /// * `file` - A string representing the path to the file where the `NetworkConfig` should be saved. - /// - /// # Returns - /// - /// This function returns a `Result` that contains `()` if the `NetworkConfig` was successfully serialized and written to the file, or a `NetworkConfigError` if an error occurred. - /// - /// # Errors - /// - /// This function will return an error if the `NetworkConfig` cannot be serialized or if the file cannot be written. - /// - /// # Examples - /// - /// ```ignore - /// # use hotshot_orchestrator::config::NetworkConfig; - /// let file = "/path/to/my/config".to_string(); - /// let config = NetworkConfig::from_file(file); - /// config.to_file(file).unwrap(); - /// ``` - pub fn to_file(&self, file: String) -> Result<(), NetworkConfigError> { - // ensure the directory containing the config file exists - if let Some(dir) = Path::new(&file).parent() { - if let Err(e) = fs::create_dir_all(dir) { - return Err(NetworkConfigError::FailedToCreatePath(e)); - } - } - - // serialize - let serialized = match serde_json::to_string_pretty(self) { - Ok(data) => data, - Err(e) => { - return Err(NetworkConfigError::SerializeError(e)); - } - }; - - // write to file - match fs::write(file, serialized) { - Ok(()) => Ok(()), - Err(e) => Err(NetworkConfigError::WriteToFileError(e)), - } - } -} - -impl Default for NetworkConfig { - fn default() -> Self { - Self { - rounds: ORCHESTRATOR_DEFAULT_NUM_ROUNDS, - indexed_da: true, - transactions_per_round: ORCHESTRATOR_DEFAULT_TRANSACTIONS_PER_ROUND, - node_index: 0, - seed: [0u8; 32], - transaction_size: ORCHESTRATOR_DEFAULT_TRANSACTION_SIZE, - manual_start_password: None, - libp2p_config: None, - config: HotShotConfigFile::hotshot_config_5_nodes_10_da().into(), - key_type_name: std::any::type_name::().to_string(), - cdn_marshal_address: None, - combined_network_config: None, - next_view_timeout: 10, - view_sync_timeout: Duration::from_secs(2), - num_bootrap: 5, - builder_timeout: Duration::from_secs(10), - data_request_delay: Duration::from_millis(2500), - commit_sha: String::new(), - builder: BuilderType::default(), - random_builder: None, - public_keys: vec![], - } - } -} - -/// a network config stored in a file -#[serde_inline_default] -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] -#[serde(bound(deserialize = ""))] -pub struct PublicKeysFile { - /// The list of public keys that are allowed to connect to the orchestrator - /// - /// If nonempty, this list becomes the stake table and is used to determine DA membership (ignoring the node's request). - #[serde(default)] - pub public_keys: Vec>, -} - -/// a network config stored in a file -#[serde_inline_default] -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] -#[serde(bound(deserialize = ""))] -pub struct NetworkConfigFile { - /// number of views to run - #[serde_inline_default(ORCHESTRATOR_DEFAULT_NUM_ROUNDS)] - pub rounds: usize, - /// number of views to run - #[serde(default)] - pub indexed_da: bool, - /// number of transactions per view - #[serde_inline_default(ORCHESTRATOR_DEFAULT_TRANSACTIONS_PER_ROUND)] - pub transactions_per_round: usize, - /// password to have the orchestrator start the network, - /// regardless of the number of nodes connected. - #[serde(default)] - pub manual_start_password: Option, - /// global index of node (for testing purposes a uid) - #[serde(default)] - pub node_index: u64, - /// unique seed (for randomness? TODO) - #[serde(default)] - pub seed: [u8; 32], - /// size of transactions - #[serde_inline_default(ORCHESTRATOR_DEFAULT_TRANSACTION_SIZE)] - pub transaction_size: usize, - /// the hotshot config file - #[serde(default = "HotShotConfigFile::hotshot_config_5_nodes_10_da")] - pub config: HotShotConfigFile, - /// The address of the Push CDN's "marshal", A.K.A. load balancer - #[serde(default)] - pub cdn_marshal_address: Option, - /// combined network config - #[serde(default)] - pub combined_network_config: Option, - /// builder to use - #[serde(default)] - pub builder: BuilderType, - /// random builder configuration - #[serde(default)] - pub random_builder: Option, - /// The list of public keys that are allowed to connect to the orchestrator - /// - /// If nonempty, this list becomes the stake table and is used to determine DA membership (ignoring the node's request). - #[serde(default)] - pub public_keys: Vec>, -} - -impl From> for NetworkConfig { - fn from(val: NetworkConfigFile) -> Self { - NetworkConfig { - rounds: val.rounds, - indexed_da: val.indexed_da, - transactions_per_round: val.transactions_per_round, - node_index: 0, - num_bootrap: val.config.num_bootstrap, - manual_start_password: val.manual_start_password, - next_view_timeout: val.config.next_view_timeout, - view_sync_timeout: val.config.view_sync_timeout, - builder_timeout: val.config.builder_timeout, - data_request_delay: val - .config - .data_request_delay - .unwrap_or(Duration::from_millis(REQUEST_DATA_DELAY)), - seed: val.seed, - transaction_size: val.transaction_size, - libp2p_config: Some(Libp2pConfig { - bootstrap_nodes: Vec::new(), - }), - config: val.config.into(), - key_type_name: std::any::type_name::().to_string(), - cdn_marshal_address: val.cdn_marshal_address, - combined_network_config: val.combined_network_config, - commit_sha: String::new(), - builder: val.builder, - random_builder: val.random_builder, - public_keys: val.public_keys, - } - } } diff --git a/crates/types/src/traits/consensus_api.rs b/crates/types/src/traits/consensus_api.rs index 4c4daff9a7..31bceaead5 100644 --- a/crates/types/src/traits/consensus_api.rs +++ b/crates/types/src/traits/consensus_api.rs @@ -6,7 +6,7 @@ //! Contains the [`ConsensusApi`] trait. -use std::{num::NonZeroUsize, time::Duration}; +use std::time::Duration; use async_trait::async_trait; @@ -23,7 +23,7 @@ use crate::{ #[async_trait] pub trait ConsensusApi>: Send + Sync { /// Total number of nodes in the network. Also known as `n`. - fn total_nodes(&self) -> NonZeroUsize; + fn total_nodes(&self) -> usize; /// The maximum amount of time a leader can wait to get a block from a builder. fn builder_timeout(&self) -> Duration; diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index c862062190..67beeacb0c 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -15,7 +15,7 @@ use crate::{traits::signature_key::SignatureKey, PeerConfig}; #[async_trait] /// A protocol for determining membership in and participating in a committee. -pub trait Membership: Debug + Send + Sync { +pub trait Membership: Debug + Send + Sync + Clone { /// The error type returned by methods like `lookup_leader`. type Error: std::fmt::Display; diff --git a/crates/types/src/traits/network.rs b/crates/types/src/traits/network.rs index 2348dc8bc7..5e8d862a6c 100644 --- a/crates/types/src/traits/network.rs +++ b/crates/types/src/traits/network.rs @@ -288,8 +288,6 @@ where #[allow(clippy::type_complexity)] fn generator( expected_node_count: usize, - num_bootstrap: usize, - network_id: usize, da_committee_size: usize, reliability_config: Option>, secondary_network_delay: Duration, diff --git a/crates/types/src/traits/node_implementation.rs b/crates/types/src/traits/node_implementation.rs index 77a4971e24..49b814672a 100644 --- a/crates/types/src/traits/node_implementation.rs +++ b/crates/types/src/traits/node_implementation.rs @@ -101,7 +101,6 @@ pub trait TestableNodeImplementation: NodeImplementation /// Generate the communication channels for testing fn gen_networks( expected_node_count: usize, - num_bootstrap: usize, da_committee_size: usize, reliability_config: Option>, secondary_network_delay: Duration, @@ -143,15 +142,12 @@ where fn gen_networks( expected_node_count: usize, - num_bootstrap: usize, da_committee_size: usize, reliability_config: Option>, secondary_network_delay: Duration, ) -> AsyncGenerator> { >::generator( expected_node_count, - num_bootstrap, - 0, da_committee_size, reliability_config.clone(), secondary_network_delay, diff --git a/crates/types/src/utils.rs b/crates/types/src/utils.rs index 6496e182f8..2398f88ad8 100644 --- a/crates/types/src/utils.rs +++ b/crates/types/src/utils.rs @@ -7,7 +7,7 @@ //! Utility functions, type aliases, helper structs and enum definitions. use std::{ - hash::{Hash, Hasher}, + hash::{DefaultHasher, Hash, Hasher}, ops::Deref, sync::Arc, }; @@ -255,9 +255,15 @@ pub fn epoch_from_block_number(block_number: u64, epoch_height: u64) -> u64 { /// A function for generating a cute little user mnemonic from a hash #[must_use] pub fn mnemonic(bytes: H) -> String { - let mut state = std::collections::hash_map::DefaultHasher::new(); - bytes.hash(&mut state); - mnemonic::to_string(state.finish().to_le_bytes()) + let hash = non_crypto_hash(bytes); + mnemonic::to_string(hash.to_le_bytes()) +} + +/// A helper function to generate a non-cryptographic hash +pub fn non_crypto_hash(val: H) -> u64 { + let mut state = DefaultHasher::new(); + val.hash(&mut state); + state.finish() } /// A helper enum to indicate whether a node is in the epoch transition diff --git a/crates/types/src/validator_config.rs b/crates/types/src/validator_config.rs deleted file mode 100644 index 2622c0d23a..0000000000 --- a/crates/types/src/validator_config.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2021-2024 Espresso Systems (espressosys.com) -// This file is part of the HotShot repository. - -// You should have received a copy of the MIT License -// along with the HotShot repository. If not, see . - -use std::{env, fs, path::PathBuf}; - -use toml; -use tracing::error; - -use crate::{traits::signature_key::SignatureKey, ValidatorConfig}; - -/// Holds configuration for a validator node -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] -#[serde(bound(deserialize = ""))] -pub struct ValidatorConfigFile { - /// The validator's seed - pub seed: [u8; 32], - /// The validator's index, which can be treated as another input to the seed - pub node_id: u64, - // The validator's stake, commented for now - // pub stake_value: u64, - /// Whether or not we are DA - pub is_da: bool, -} - -impl ValidatorConfigFile { - /// read the validator config from a file - /// # Panics - /// Panics if unable to get the current working directory - pub fn from_file(dir_str: &str) -> Self { - let current_working_dir = match env::current_dir() { - Ok(dir) => dir, - Err(e) => { - error!("get_current_working_dir error: {:?}", e); - PathBuf::from("") - } - }; - let filename = - current_working_dir.into_os_string().into_string().unwrap() + "/../../" + dir_str; - let contents = fs::read_to_string(filename.clone()).expect("Could not read file"); - let data: ValidatorConfigFile = - toml::from_str(&contents).expect("Unable to load data from file"); - data - } -} - -impl From for ValidatorConfig { - fn from(val: ValidatorConfigFile) -> Self { - // here stake_value is set to 1, since we don't input stake_value from ValidatorConfigFile for now - ValidatorConfig::generated_from_seed_indexed(val.seed, val.node_id, 1, val.is_da) - } -} diff --git a/docker/cdn-broker.Dockerfile b/docker/cdn-broker.Dockerfile deleted file mode 100644 index 348c7355d5..0000000000 --- a/docker/cdn-broker.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/cdn-broker /usr/local/bin/cdn-broker - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["cdn-broker"] diff --git a/docker/cdn-marshal.Dockerfile b/docker/cdn-marshal.Dockerfile deleted file mode 100644 index 9ea1b497c1..0000000000 --- a/docker/cdn-marshal.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/cdn-marshal /usr/local/bin/cdn-marshal - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["cdn-marshal"] diff --git a/docker/orchestrator.Dockerfile b/docker/orchestrator.Dockerfile deleted file mode 100644 index eda3846754..0000000000 --- a/docker/orchestrator.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/orchestrator /usr/local/bin/orchestrator - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["orchestrator"] diff --git a/docker/validator-cdn-local.Dockerfile b/docker/validator-cdn-local.Dockerfile deleted file mode 100644 index 156fc50571..0000000000 --- a/docker/validator-cdn-local.Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - - -COPY --chmod=0755 ./target/release-lto/examples/validator-push-cdn /usr/local/bin/validator-push-cdn - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["validator-push-cdn"] diff --git a/docker/validator-cdn.Dockerfile b/docker/validator-cdn.Dockerfile deleted file mode 100644 index 50d927ce74..0000000000 --- a/docker/validator-cdn.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-push-cdn /usr/local/bin/validator-push-cdn - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["validator-push-cdn"] diff --git a/docker/validator-combined.Dockerfile b/docker/validator-combined.Dockerfile deleted file mode 100644 index 7e8afa9f16..0000000000 --- a/docker/validator-combined.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-combined /usr/local/bin/validator-combined - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["validator-combined"] diff --git a/docker/validator-libp2p.Dockerfile b/docker/validator-libp2p.Dockerfile deleted file mode 100644 index 42970c1aab..0000000000 --- a/docker/validator-libp2p.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:jammy - -RUN apt-get update \ - && apt-get install -y curl libcurl4 wait-for-it tini \ - && rm -rf /var/lib/apt/lists/* - -ARG TARGETARCH -ARG ASYNC_EXECUTOR - -COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-libp2p /usr/local/bin/validator-libp2p - -# logging -ENV RUST_LOG="warn" - -# log format. JSON no ansi -ENV RUST_LOG_FORMAT="json" - -ENTRYPOINT ["validator-libp2p"] diff --git a/hotshot-emoji.png b/hotshot-emoji.png new file mode 100644 index 0000000000..a219bd53e0 Binary files /dev/null and b/hotshot-emoji.png differ diff --git a/justfile b/justfile index a48c18021c..de5feb3946 100644 --- a/justfile +++ b/justfile @@ -10,15 +10,6 @@ build: build_release *ARGS: cargo build --profile=release {{ARGS}} -example *ARGS: - cargo run --profile=release-lto --package hotshot-examples --no-default-features --example {{ARGS}} - -example_fixed_leader *ARGS: - cargo run --features "fixed-leader-election" --profile=release-lto --example {{ARGS}} - -example_gpuvid_leader *ARGS: - cargo run --features "fixed-leader-election, gpu-vid" --profile=release-lto --example {{ARGS}} - test *ARGS: echo Testing {{ARGS}} cargo test --lib --bins --tests --benches --workspace --no-fail-fast {{ARGS}} -- --test-threads=1 --nocapture --skip crypto_test @@ -148,13 +139,11 @@ clippy_release: fmt: echo Running cargo fmt - cargo fmt -- crates/**/*.rs - cargo fmt -- crates/**/tests/**/**.rs + cargo fmt fmt_check: echo Running cargo fmt --check - cargo fmt --check -- crates/**/*.rs - cargo fmt --check -- crates/**/tests/**/**.rs + cargo fmt --check lint: clippy fmt_check diff --git a/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh b/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh deleted file mode 100755 index 26158570e2..0000000000 --- a/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/bin/bash - -# make sure the following line is added to `~/.bashrc` -source "$HOME/.cargo/env" - -# assign local ip by curl from AWS metadata server: -# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html -AWS_METADATA_IP=`curl http://169.254.169.254/latest/meta-data/local-ipv4` -orchestrator_url=http://"$AWS_METADATA_IP":4444 -cdn_marshal_address="$AWS_METADATA_IP":9000 -keydb_address=redis://"$AWS_METADATA_IP":6379 -current_commit=$(git rev-parse HEAD) -commit_append="" - -# Check if at least two arguments are provided -if [ $# -lt 1 ]; then - echo "Usage: $0 " - exit 1 -fi -REMOTE_USER="$1" - -# this is to prevent "Error: Too many open files (os error 24). Pausing for 500ms" -ulimit -n 65536 -# build to get the bin in advance, uncomment the following if built first time -# just example validator-push-cdn -- http://localhost:4444 & -# # remember to sleep enough time if it's built first time -# sleep 3m -# for pid in $(ps -ef | grep "validator" | awk '{print $2}'); do kill -9 $pid; done - -# docker build and push -docker build . -f ./docker/validator-cdn-local.Dockerfile -t ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio -docker push ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio - -# ecs deploy -ecs deploy --region us-east-2 hotshot hotshot_centralized -i centralized ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio -ecs deploy --region us-east-2 hotshot hotshot_centralized -c centralized ${orchestrator_url} - -# runstart keydb -# docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb & -# server1: marshal -echo -e "\e[35mGoing to start cdn-marshal on local server\e[0m" -just example cdn-marshal -- -d redis://localhost:6379 -b 9000 & -# remember to sleep enough time if it's built first time - -# Function to round up to the nearest integer -round_up() { - echo "scale=0;($1+0.5)/1" | bc -} - -# for a single run -# total_nodes, da_committee_size, transactions_per_round, transaction_size = 100, 10, 1, 4096 -for total_nodes in 10 50 100 200 500 1000 -do - for da_committee_size in 10 - do - if [ $da_committee_size -le $total_nodes ] - then - for transactions_per_round in 1 10 - do - for transaction_size in 100000 1000000 10000000 20000000 - do - for fixed_leader_for_gpuvid in 1 - do - if [ $fixed_leader_for_gpuvid -le $da_committee_size ] - then - for rounds in 100 - do - # server1: broker - echo -e "\e[35mGoing to start cdn-broker on local server\e[0m" - just example cdn-broker -- -d redis://localhost:6379 \ - --public-bind-endpoint 0.0.0.0:1740 \ - --public-advertise-endpoint local_ip:1740 \ - --private-bind-endpoint 0.0.0.0:1741 \ - --private-advertise-endpoint local_ip:1741 & - # server2: broker - # make sure you're able to access the remote host from current host - echo -e "\e[35mGoing to start cdn-broker on remote server\e[0m" - BROKER_COUNTER=0 - for REMOTE_BROKER_HOST in "$@"; do - if [ "$BROKER_COUNTER" -ge 1 ]; then - echo -e "\e[35mstart broker $BROKER_COUNTER on $REMOTE_BROKER_HOST\e[0m" - ssh $REMOTE_USER@$REMOTE_BROKER_HOST << EOF -cd HotShot -nohup bash scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh ${keydb_address} > nohup.out 2>&1 & -exit -EOF - fi - BROKER_COUNTER=$((BROKER_COUNTER + 1)) - done - - # start orchestrator - echo -e "\e[35mGoing to start orchestrator on local server\e[0m" - just example orchestrator -- --config_file ./crates/orchestrator/run-config.toml \ - --orchestrator_url http://0.0.0.0:4444 \ - --total_nodes ${total_nodes} \ - --da_committee_size ${da_committee_size} \ - --transactions_per_round ${transactions_per_round} \ - --transaction_size ${transaction_size} \ - --rounds ${rounds} \ - --fixed_leader_for_gpuvid ${fixed_leader_for_gpuvid} \ - --cdn_marshal_address ${cdn_marshal_address} \ - --commit_sha ${current_commit}${commit_append} & - sleep 30 - - # start validators - echo -e "\e[35mGoing to start validators on remote servers\e[0m" - ecs scale --region us-east-2 hotshot hotshot_centralized ${total_nodes} --timeout -1 - base=100 - mul=$(echo "l($transaction_size * $transactions_per_round)/l($base)" | bc -l) - mul=$(round_up $mul) - sleep_time=$(( ($rounds + $total_nodes / 2 ) * $mul )) - echo -e "\e[35msleep_time: $sleep_time\e[0m" - sleep $sleep_time - - # kill them - echo -e "\e[35mGoing to stop validators on remote servers\e[0m" - ecs scale --region us-east-2 hotshot hotshot_centralized 0 --timeout -1 - for pid in $(ps -ef | grep "orchestrator" | awk '{print $2}'); do kill -9 $pid; done - # shut down brokers - echo -e "\e[35mGoing to stop cdn-broker\e[0m" - killall -9 cdn-broker - BROKER_COUNTER=0 - for REMOTE_BROKER_HOST in "$@"; do - if [ "$BROKER_COUNTER" -ge 1 ]; then - echo -e "\e[35mstop broker $BROKER_COUNTER on $REMOTE_BROKER_HOST\e[0m" - ssh $REMOTE_USER@$REMOTE_BROKER_HOST "killall -9 cdn-broker && exit" - fi - BROKER_COUNTER=$((BROKER_COUNTER + 1)) - done - # remove brokers from keydb - # you'll need to do `echo DEL brokers | keydb-cli -a THE_PASSWORD` and set it to whatever password you set - echo DEL brokers | keydb-cli - # make sure you sleep at least 1 min - sleep $(( $total_nodes + 60)) - done - fi - done - done - done - fi - done -done - -# shut down all related threads -echo -e "\e[35mGoing to stop cdn-marshal\e[0m" -killall -9 cdn-marshal -# for pid in $(ps -ef | grep "keydb-server" | awk '{print $2}'); do sudo kill -9 $pid; done \ No newline at end of file diff --git a/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh b/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh deleted file mode 100755 index ad8bdb8b87..0000000000 --- a/scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh +++ /dev/null @@ -1,168 +0,0 @@ -#!/bin/bash - -# make sure the following line is added to `~/.bashrc` -source "$HOME/.cargo/env" - -# assign local ip by curl from AWS metadata server: -# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html -AWS_METADATA_IP=`curl http://169.254.169.254/latest/meta-data/local-ipv4` -orchestrator_url=http://"$AWS_METADATA_IP":4444 -cdn_marshal_address="$AWS_METADATA_IP":9000 -keydb_address=redis://"$AWS_METADATA_IP":6379 -current_commit=$(git rev-parse HEAD) -commit_append="_gpu" - -# Check if at least two arguments are provided -if [ $# -lt 3 ]; then - echo "Usage: $0 " - exit 1 -fi -REMOTE_USER="$1" -REMOTE_GPU_HOST="$2" - -# this is to prevent "Error: Too many open files (os error 24). Pausing for 500ms" -ulimit -n 65536 - -# build to get the bin in advance, uncomment the following if built first time -just example_fixed_leader validator-push-cdn -- http://localhost:4444 & -# remember to sleep enough time if it's built first time -sleep 3m -for pid in $(ps -ef | grep "validator" | awk '{print $2}'); do kill -9 $pid; done - -# docker build and push -docker build . -f ./docker/validator-cdn-local.Dockerfile -t ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio -docker push ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio - -# ecs deploy -ecs deploy --region us-east-2 hotshot hotshot_centralized -i centralized ghcr.io/espressosystems/hotshot/validator-push-cdn:main-tokio -ecs deploy --region us-east-2 hotshot hotshot_centralized -c centralized ${orchestrator_url} - -# runstart keydb -# docker run --rm -p 0.0.0.0:6379:6379 eqalpha/keydb & -# echo DEL brokers | keydb-cli -# server1: marshal -echo -e "\e[35mGoing to start cdn-marshal on local server\e[0m" -just example cdn-marshal -- -d redis://localhost:6379 -b 9000 & -# remember to sleep enough time if it's built first time - -# Function to round up to the nearest integer -round_up() { - echo "scale=0;($1+0.5)/1" | bc -} - -# for a single run -# total_nodes, da_committee_size, transactions_per_round, transaction_size = 100, 10, 1, 4096 -for total_nodes in 10 50 100 200 500 1000 -do - for da_committee_size in 5 10 50 100 - do - if [ $da_committee_size -le $total_nodes ] - then - for transactions_per_round in 1 10 - do - for transaction_size in 100000 1000000 10000000 20000000 - do - for fixed_leader_for_gpuvid in 1 5 10 - do - if [ $fixed_leader_for_gpuvid -le $da_committee_size ] - then - for rounds in 100 - do - # server1: broker - echo -e "\e[35mGoing to start cdn-broker on local server\e[0m" - just example cdn-broker -- -d redis://localhost:6379 \ - --public-bind-endpoint 0.0.0.0:1740 \ - --public-advertise-endpoint local_ip:1740 \ - --private-bind-endpoint 0.0.0.0:1741 \ - --private-advertise-endpoint local_ip:1741 & - # server2: broker - # make sure you're able to access the remote host from current host - echo -e "\e[35mGoing to start cdn-broker on remote server\e[0m" - BROKER_COUNTER=0 - for REMOTE_BROKER_HOST in "$@"; do - if [ "$BROKER_COUNTER" -ge 2 ]; then - echo -e "\e[35mstart broker $((BROKER_COUNTER - 1)) on $REMOTE_BROKER_HOST\e[0m" - ssh $REMOTE_USER@$REMOTE_BROKER_HOST << EOF -cd HotShot -nohup bash scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh ${keydb_address} > nohup.out 2>&1 & -exit -EOF - fi - BROKER_COUNTER=$((BROKER_COUNTER + 1)) - done - - # start orchestrator - echo -e "\e[35mGoing to start orchestrator on local server\e[0m" - just example_fixed_leader orchestrator -- --config_file ./crates/orchestrator/run-config.toml \ - --orchestrator_url http://0.0.0.0:4444 \ - --total_nodes ${total_nodes} \ - --da_committee_size ${da_committee_size} \ - --transactions_per_round ${transactions_per_round} \ - --transaction_size ${transaction_size} \ - --rounds ${rounds} \ - --fixed_leader_for_gpuvid ${fixed_leader_for_gpuvid} \ - --cdn_marshal_address ${cdn_marshal_address} \ - --commit_sha ${current_commit}${commit_append} & - sleep 30 - - # start leaders need to run on GPU FIRST - # and WAIT for enough time till it registered at orchestrator - # make sure you're able to access the remote nvidia gpu server - echo -e "\e[35mGoing to start leaders on remote gpu server $REMOTE_GPU_HOST\e[0m" - - ssh $REMOTE_USER@$REMOTE_GPU_HOST << EOF -cd HotShot -nohup bash scripts/benchmark_scripts/benchmarks_start_leader_gpu.sh ${fixed_leader_for_gpuvid} ${orchestrator_url} > nohup.out 2>&1 & -exit -EOF - - sleep 1m - - # start validators - echo -e "\e[35mGoing to start validators on remote cpu servers\e[0m" - ecs scale --region us-east-2 hotshot hotshot_centralized $(($total_nodes - $fixed_leader_for_gpuvid)) --timeout -1 - base=100 - mul=$(echo "l($transaction_size * $transactions_per_round)/l($base)" | bc -l) - mul=$(round_up $mul) - sleep_time=$(( ($rounds + $total_nodes / 2) * $mul )) - echo -e "\e[35msleep_time: $sleep_time\e[0m" - sleep $sleep_time - - # kill them - # shut down nodes - echo -e "\e[35mGoing to stop validators on remote cpu servers\e[0m" - ecs scale --region us-east-2 hotshot hotshot_centralized 0 --timeout -1 - # shut down leaders on gpu - echo -e "\e[35mGoing to stop leaders on remote gpu server\e[0m" - ssh $REMOTE_GPU_USER@$REMOTE_GPU_HOST "for pid in $(ps -ef | grep "validator" | awk '{print $2}'); do kill -9 $pid; done && exit" - echo -e "\e[35mGoing to stop orchestrator\e[0m" - for pid in $(ps -ef | grep "orchestrator" | awk '{print $2}'); do kill -9 $pid; done - # shut down brokers - echo -e "\e[35mGoing to stop cdn-broker\e[0m" - killall -9 cdn-broker - BROKER_COUNTER=0 - for REMOTE_BROKER_HOST in "$@"; do - if [ "$BROKER_COUNTER" -ge 2 ]; then - echo -e "\e[35mstop broker $((BROKER_COUNTER - 1)) on $REMOTE_BROKER_HOST\e[0m" - ssh $REMOTE_USER@$REMOTE_BROKER_HOST "killall -9 cdn-broker && exit" - fi - BROKER_COUNTER=$((BROKER_COUNTER + 1)) - done - # remove brokers from keydb - # you'll need to do `echo DEL brokers | keydb-cli -a THE_PASSWORD` and set it to whatever password you set - echo DEL brokers | keydb-cli - # make sure you sleep at least 1 min to wait for DB to forget brokers and marshals - sleep $(( $total_nodes + 60)) - done - fi - done - done - done - fi - done -done - -# shut down all related threads -echo -e "\e[35mGoing to stop cdn-marshal\e[0m" -killall -9 cdn-marshal -# for pid in $(ps -ef | grep "keydb-server" | awk '{print $2}'); do sudo kill -9 $pid; done diff --git a/scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh b/scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh deleted file mode 100755 index e438ce8692..0000000000 --- a/scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# make sure the following line is added to `~/.bashrc` -source "$HOME/.cargo/env" - - -if [ -z "$1" ]; then - echo "No arguments provided. Usage: $0 " - exit 1 -fi - -echo "Argument 1: $1" -keydb_address="$1" - -just example cdn-broker -- -d $keydb_address \ - --public-bind-endpoint 0.0.0.0:1740 \ - --public-advertise-endpoint local_ip:1740 \ - --private-bind-endpoint 0.0.0.0:1741 \ - --private-advertise-endpoint local_ip:1741 & diff --git a/scripts/benchmark_scripts/benchmarks_start_leader_gpu.sh b/scripts/benchmark_scripts/benchmarks_start_leader_gpu.sh deleted file mode 100755 index 353516eb68..0000000000 --- a/scripts/benchmark_scripts/benchmarks_start_leader_gpu.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# make sure the following line is added to `~/.bashrc` -source "$HOME/.cargo/env" - -# Check if at least two arguments are provided -if [ $# -lt 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -echo "Argument 1: $1" -echo "Argument 2: $2" -fixed_leader_for_gpuvid="$1" -orchestrator_url="$2" - -just example_gpuvid_leader multi-validator-push-cdn -- $fixed_leader_for_gpuvid $orchestrator_url & - - \ No newline at end of file diff --git a/scripts/benchmarks_results/README.md b/scripts/benchmarks_results/README.md deleted file mode 100644 index e67ba6095d..0000000000 --- a/scripts/benchmarks_results/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## How to run the benchmarks - -- To run it locally, check out `crates/examples/push-cdn/README.md`. -- To run it in AWS, take a look at `scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh` and change value of parameters as you'd like, make sure you've installed everything needed in the script and have access to needed servers (and have hotshot on those servers), then start `key-db` in one `tmux` session, in another session, enter `HotShot`, run `./scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh [YOUR_NAME] [REMOTE_BROKER_HOST_PUBLIC_IP_1] [REMOTE_BROKER_HOST_PUBLIC_IP_2] ...`. If you want to run leaders on GPU, for the last step, run `./scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh [YOUR_NAME] [REMOTE_GPU_HOST] [REMOTE_BROKER_HOST_PUBLIC_IP_1] [REMOTE_BROKER_HOST_PUBLIC_IP_2] ...` instead (e.g. `./scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh sishan 11.111.223.224 3.111.223.224`). -- When running on a large group of nodes (1000 etc.), it might take too long for all nodes to post "start", you can use `manual_start` when you think there're enough nodes connected: -``` -export ORCHESTRATOR_MANUAL_START_PASSWORD=password -curl -X POST http://172.31.8.82:4444/v0/api/manual_start -d 'password' -``` - -## How to view the results - -- Three ways to gather the results - - (recommended) check out`scripts/benchmarks_results/results.csv` on EC2, where you should have organized overall results. - - check out Datadog under `host:/hotshot/` where you have stats for each individual validator but it's hard to track since they’re distributed. - - wait for the output of orchestrator in local terminal, where the results are not that organized if you do multiple runs, also hard to track. - -- Explanation on confusing arguments - - `partial_results`: Whether the results are partially collected. It's set to "One" when the results are collected for one node; "HalfDA" when the results are collected for the number equals to DA_committee_number / 2; "Half" when the results are collected for half running nodes; "Full" if the results are successfully collected from all nodes. The reason is sometimes we'll get high throughput however not all the nodes can terminate successfully (I suspect the reason is that some of them fall behind when fetching large transactions). \ No newline at end of file diff --git a/scripts/benchmarks_results/results_upload.csv b/scripts/benchmarks_results/results_upload.csv deleted file mode 100644 index 892d93df07..0000000000 --- a/scripts/benchmarks_results/results_upload.csv +++ /dev/null @@ -1,160 +0,0 @@ -commit_sha,total_nodes,da_committee_size,transactions_per_round,transaction_size,rounds,leader_election_type,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -test_large_tx_local,10,5,1,2000016,10,hotshot::traits::election::static_committee::StaticElectionConfig,25,11,44,977785,22,45,9,0 -commit_sha,total_nodes,da_committee_size,transactions_per_round,transaction_size,rounds,leader_election_type,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -nginx_script_test,10,5,1,1000016,20,hotshot::traits::election::static_committee::StaticElectionConfig,36,6,83,1317668,112,85,19,0 -commit_sha,total_nodes,da_committee_size,transactions_per_round,transaction_size,rounds,leader_election_type,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -test50nodes,50,5,1,1000016,20,hotshot::traits::election::static_committee::StaticElectionConfig,26,6,58,2372919,140,60,19,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,10,5,1,1,1000016,50,fixed-leader-election,Partial,19,1,63,4859452,311,64,50,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,10,5,1,1,1000016,50,fixed-leader-election,Complete,14,1,58,4929903,281,57,50,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,100,10,1,1,1000016,50,fixed-leader-election,Half,48,1,132,8137534,1066,131,50,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,100,10,1,1,1000016,100,fixed-leader-election,HalfDA,87,2,254,6949606,2064,297,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,50,10,1,1,1000016,50,fixed-leader-election,Partial,48,3,128,7640747,978,128,50,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,50,10,1,1,1000016,100,fixed-leader-election,Partial,49,1,142,4759732,1109,233,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,100,10,1,10,1000016,100,fixed-leader-election,HalfDA,59,7,161,3139208,970,310,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,100,10,1,1,10000016,100,fixed-leader-election,Half,17,8,33,449102,15,335,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,200,10,1,1,1000016,100,fixed-leader-election,Half,97,4,280,7056362,2258,325,100,2 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,500,10,1,1,1000016,100,fixed-leader-election,HalfDA,95,8,269,3871646,1417,366,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_no_gpu,1000,10,1,1,1000016,100,fixed-leader-election,Half,142,17,423,1908157,1080,626,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,10,5,1,1,100016,100,fixed-leader-election,Full,1,0,7,1843152,516,29,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,10,10,1,10,100016,100,fixed-leader-election,Full,1,0,2,3097269,960,31,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,50,5,1,1,100016,100,fixed-leader-election,Half,17,1,60,6179321,3707,60,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,50,10,1,10,100016,100,fixed-leader-election,HalfDA,38,1,109,3492698,7997,229,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,50,10,1,1,1000016,100,fixed-leader-election,HalfDA,52,1,155,5478535,1271,232,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,1,1000016,100,fixed-leader-election,HalfDA,80,2,236,6540455,1864,285,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,200,10,1,1,1000016,100,fixed-leader-election,HalfDA,99,3,291,7201334,2362,328,102,2 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,500,10,1,1,1000016,100,fixed-leader-election,HalfDA,110,8,319,4476755,1728,386,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,1000,10,1,1,1000016,100,fixed-leader-election,Half,147,13,371,2234382,1249,560,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,1,1000016,100,fixed-leader-election,HalfDA,102,2,284,7881746,2530,321,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,1,100016,100,fixed-leader-election,Half,29,1,84,5523502,4639,84,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,1,10000016,100,fixed-leader-election,Half,16,7,32,425532,16,376,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,10,1000016,100,fixed-leader-election,Half,18,8,36,382506,153,400,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,10,100016,100,fixed-leader-election,Half,47,2,137,3823773,10399,273,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,100,1,10,100016,100,fixed-leader-election,Full,2,2,5,1477159,960,65,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_random_tx,100,10,1,100,100016,100,fixed-leader-election,Half,9,4,11,170343,700,411,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,500,10,1,1,1000016,100,fixed-leader-election,Half,93,8,257,5577012,1885,338,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,200,10,1,1,1000016,100,fixed-leader-election,Half,101,3,279,9168400,2888,419,100,8 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,100,10,1,1,1000016,100,fixed-leader-election,Half,81,2,240,8896939,2500,283,102,4 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,50,10,1,1,100016,100,fixed-leader-election,Full,12,0,29,6032337,3076,51,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,50,10,1,1,1000016,100,fixed-leader-election,Half,45,1,130,5372813,1182,220,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,10,10,1,10,1000016,100,fixed-leader-election,Full,8,3,14,4481203,950,213,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,10,10,1,10,100016,100,fixed-leader-election,Full,1,0,3,3097269,960,31,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,1000,10,1,1,1000016,100,fixed-leader-election,Half,140,13,363,2636913,1416,537,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -real_2_broker,100,10,1,1,10000016,100,fixed-leader-election,Half,27,11,68,585586,26,444,102,2 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,10,5,1,1,1000016,100,fixed-leader-election,Full,1,0,4,3310397,96,29,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_simple_builder_fixed_leader,10,5,1,1,1000016,100,fixed-leader-election,Full,1,1,6,2341500,96,41,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,50,10,1,1,1000016,100,fixed-leader-election,Full,1,1,6,2666709,96,37,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,50,10,1,10,1000016,100,fixed-leader-election,Full,4,1,6,8275994,960,117,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,50,10,5,10,1000016,100,fixed-leader-election,Full,8,4,13,4363706,960,221,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,1,1000016,100,fixed-leader-election,Full,1,1,6,1920030,96,50,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,1,10000016,100,fixed-leader-election,Full,5,5,9,6442963,96,149,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,1,10000016,100,fixed-leader-election,Full,5,5,9,6442963,96,149,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,10,1000016,100,fixed-leader-election,Full,6,5,8,6400102,960,150,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,200,10,1,1,1000016,100,fixed-leader-election,Full,2,2,7,1280020,96,75,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,200,10,1,1,10000016,100,fixed-leader-election,Full,11,8,14,3428576,96,281,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,1,15000016,100,fixed-leader-election,Half,10,9,11,5691705,96,253,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,1,20000016,100,fixed-leader-election,Half,12,12,14,6018813,96,319,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_with_gpu,100,10,1,10,2000016,100,fixed-leader-election,Half,12,8,15,6241092,958,307,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_simple_builder_fixed_leader,100,10,1,1,1000016,100,fixed-leader-election,Half,1,1,7,1882383,96,51,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -cdn_simple_builder_fixed_leader,100,10,1,10,2000016,50,fixed-leader-election,Full,13,9,15,5380160,460,171,50,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,10,5,1,1,1000016,100,static-leader-selection,Full,1,1,5,3057191,107,35,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,100,10,1,1,1000016,100,static-leader-selection,Full,1,1,7,1515649,97,64,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,100,10,1,10,1000016,100,static-leader-selection,HalfDA,7,3,43,6770294,1090,161,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,100,10,1,1,20000016,100,static-leader-selection,Half,13,8,21,6815292,107,314,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,100,10,1,1,10000016,100,static-leader-selection,Half,7,5,49,7023820,118,168,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,200,10,1,1,10000016,100,static-leader-selection,Half,12,9,18,4857150,136,280,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,500,10,1,1,1000016,100,static-leader-selection,Full,8,7,18,430500,96,223,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,1000,10,1,1,1000016,100,static-leader-selection,Half,20,18,26,188238,96,510,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,500,10,1,1,5000016,100,static-leader-selection,Half,14,9,63,1839768,124,337,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,1000,10,1,1,5000016,100,static-leader-selection,Half,35,19,90,552660,106,959,117,17 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -builder_fix,100,10,1,10,2000016,100,static-leader-selection,HalfDA,21,7,89,7342677,1318,359,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker,500,10,1,1,10000016,100,static-leader-selection,Half,20,15,77,3225063,139,431,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker,500,10,1,1,5000016,100,static-leader-selection,Half,10,9,15,1946571,102,262,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3brokers_my,500,10,1,1,10000016,100,static-leader-selection,Half,23,12,90,2453187,131,534,104,4 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3brokers,500,10,1,1,10000016,100,static-leader-selection,Half,22,13,86,2351742,115,490,104,4 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker,100,10,1,1,10000016,100,static-leader-selection,HalfDA,10,4,105,5670112,110,194,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker,100,10,1,1,20000016,100,static-leader-selection,Half,16,7,96,6749231,109,323,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker,500,10,1,1,10000016,100,static-leader-selection,Half,15,11,33,3202104,122,381,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -4broker,1000,10,1,1,10000016,100,static-leader-selection,HalfDA,148,19,1059,323529,66,2040,157,57 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker_cdn_improve,500,10,1,1,20000016,100,static-leader-selection,HalfDA,40,14,284,3643619,137,752,112,12 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker_cdn_improve,1000,10,1,1,10000016,100,static-leader-selection,Half,39,19,175,1528928,111,727,104,4 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -3broker_cdn_improve,500,10,1,1,10000016,100,static-leader-selection,Half,13,10,77,3260874,105,322,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -,10,10,1,1,100016,100,static-leader-selection,Full,1,0,30,181510,98,54,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -ag_latest_main,100,10,1,1,10000016,100,static-leader-selection,Full,8,7,39,4292042,97,226,100,0 -commit_sha,total_nodes,da_committee_size,fixed_leader_for_gpuvid,transactions_per_round,transaction_size,rounds,leader_election_type,partial_results,avg_latency_in_sec,minimum_latency_in_sec,maximum_latency_in_sec,throughput_bytes_per_sec,total_transactions_committed,total_time_elapsed_in_sec,total_num_views,failed_num_views -549d41f,10,5,1,1,1000016,100,static-leader-selection,Full,2,1,32,1447784,97,68,100,0 \ No newline at end of file diff --git a/scripts/count_fds.sh b/scripts/count_fds.sh deleted file mode 100755 index 0e9e52131e..0000000000 --- a/scripts/count_fds.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -# USAGE: periodically print number of file descriptors in use -# will work with gnu coreutils - -for i in {0..100000} -do - echo NUM FDS: $(lsof | grep -i "udp" | grep -i "libp2p" | tr -s ' ' | cut -d" " -f11 | sort | uniq | wc -l) - sleep 0.2s -done diff --git a/scripts/nix_bump_pr_changes.py b/scripts/nix_bump_pr_changes.py deleted file mode 100755 index 0d9f2296cd..0000000000 --- a/scripts/nix_bump_pr_changes.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 - -# Utility script to show the github commit diff when a `update_flake_lock_action` PR is made. -# -# To run, pipe the contents of the `Flake lock file updates:` into this file -# -# e.g. `cat updates.txt | ./scripts/nix_bump_pr_changes.py` -# -# The output of this script should be pasted as a reply to that PR - -import sys -import re - -name_commit_regex = re.compile(r"'github:([^\/]+\/[^\/]+)\/([^']+)") -prev = '' - -for line in sys.stdin: - line = line.rstrip() - if line.startswith(" 'github:"): - prev = line - if line.startswith(" → 'github:"): - - [_, repo, start_commit, _] = name_commit_regex.split(prev) - [_, _, end_commit, _] = name_commit_regex.split(line) - print("- [ ] " + repo + ": [repo](https://github.com/" + repo + ") | [commits this PR](https://github.com/" + repo + "/compare/" + start_commit + ".." + end_commit + ")") - - diff --git a/scripts/runfail.sh b/scripts/runfail.sh deleted file mode 100755 index 38e882b6d6..0000000000 --- a/scripts/runfail.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Runs a command until it fails. -# Useful for running overnight to see if tests don't fail sporadically. -# -# Usage: -# `./scripts/runfail.sh cargo test --profile=release-lto --features=full-ci --manifest-path testing/Cargo.toml` - -while [ $? -eq 0 ]; do - $@ -done