diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 71de48ee..5f8fd5f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -124,6 +124,11 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} + # Needed for hwloc + - name: Install automake (macOS) + run: brew install automake + if: runner.os == 'macOS' + # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll - name: Remove msys64 run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse @@ -145,10 +150,15 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - name: Build app + - name: Build app (Linux and Windows) run: cargo build --locked -Z build-std --target ${{ matrix.build.target }} --profile production + if: runner.os != 'macOS' + + - name: Build app (macOS) + run: cargo build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --no-default-features + if: runner.os == 'macOS' - # TODO: Package Linux and macOS + # TODO: Package macOS - name: Install cargo-wix (Windows) uses: taiki-e/cache-cargo-install-action@1b76958d032c4d048c599f9fdfa48abe804d6319 # v1.2.2 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7ccda64d..63dccd6b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -142,6 +142,11 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} + # Needed for hwloc + - name: Install automake (macOS) + run: brew install automake + if: runner.os == 'macOS' + # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll - name: Remove msys64 run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse @@ -252,6 +257,11 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} + # Needed for hwloc + - name: Install automake (macOS) + run: brew install automake + if: runner.os == 'macOS' + # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll - name: Remove msys64 run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse diff --git a/Cargo.lock b/Cargo.lock index e19c6dd0..5328cdb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -971,6 +971,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "autotools" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8da1805e028a172334c3b680f93e71126f2327622faef2ec3d893c0a4ad77" +dependencies = [ + "cc", +] + [[package]] name = "backoff" version = "0.4.0" @@ -1638,6 +1647,15 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -1954,7 +1972,7 @@ checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" [[package]] name = "cross-domain-message-gossip" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "futures", "parity-scale-codec", @@ -2082,6 +2100,12 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + [[package]] name = "curve25519-dalek" version = "2.1.3" @@ -2565,7 +2589,7 @@ dependencies = [ [[package]] name = "domain-block-preprocessor" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "async-trait", "domain-runtime-primitives", @@ -2591,7 +2615,7 @@ dependencies = [ [[package]] name = "domain-runtime-primitives" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "parity-scale-codec", "scale-info", @@ -4262,6 +4286,35 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hwlocality" +version = "1.0.0-alpha.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6020affad7f95b46f12607a8714aa70bd02c8df3b3abf9ef5c8cd2f7ae57a033" +dependencies = [ + "arrayvec 0.7.4", + "bitflags 2.4.1", + "derive_more", + "errno", + "hwlocality-sys", + "libc", + "num_enum", + "thiserror", +] + +[[package]] +name = "hwlocality-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "381b203a23b41c29be64454e9cee8d16360606d7e871f5d22532796b6095f164" +dependencies = [ + "autotools", + "cmake", + "libc", + "pkg-config", + "windows-sys 0.52.0", +] + [[package]] name = "hyper" version = "0.14.27" @@ -4849,6 +4902,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3979b5c37ece694f1f5e51e7ecc871fdb0f517ed04ee45f88d15d6d553cb9664" dependencies = [ "cc", + "cty", "libc", ] @@ -6568,6 +6622,26 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683751d591e6d81200c39fb0d1032608b77724f34114db54f571ff1317b337c0" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c11e44798ad209ccdd91fc192f0526a369a01234f7373e1b141c96d7cee4f0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "objc" version = "0.2.7" @@ -6679,7 +6753,7 @@ dependencies = [ [[package]] name = "orml-vesting" version = "0.4.1-dev" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "frame-support", "frame-system", @@ -6753,7 +6827,7 @@ dependencies = [ [[package]] name = "pallet-domains" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "domain-runtime-primitives", "frame-benchmarking", @@ -6777,7 +6851,7 @@ dependencies = [ [[package]] name = "pallet-messenger" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "frame-benchmarking", "frame-support", @@ -6796,7 +6870,7 @@ dependencies = [ [[package]] name = "pallet-offences-subspace" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "frame-support", "frame-system", @@ -6810,7 +6884,7 @@ dependencies = [ [[package]] name = "pallet-rewards" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "frame-support", "frame-system", @@ -6822,7 +6896,7 @@ dependencies = [ [[package]] name = "pallet-runtime-configs" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "frame-support", "frame-system", @@ -6835,7 +6909,7 @@ dependencies = [ [[package]] name = "pallet-subspace" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "frame-benchmarking", "frame-support", @@ -6893,7 +6967,7 @@ dependencies = [ [[package]] name = "pallet-transaction-fees" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "frame-support", "frame-system", @@ -6949,7 +7023,7 @@ dependencies = [ [[package]] name = "pallet-transporter" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "domain-runtime-primitives", "frame-benchmarking", @@ -8546,7 +8620,7 @@ dependencies = [ [[package]] name = "sc-consensus-subspace" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "async-trait", "futures", @@ -8586,7 +8660,7 @@ dependencies = [ [[package]] name = "sc-consensus-subspace-rpc" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "async-oneshot", "futures", @@ -8903,7 +8977,7 @@ dependencies = [ [[package]] name = "sc-proof-of-time" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "atomic", "core_affinity", @@ -9123,7 +9197,7 @@ dependencies = [ [[package]] name = "sc-subspace-block-relay" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "async-channel", "async-trait", @@ -9148,7 +9222,7 @@ dependencies = [ [[package]] name = "sc-subspace-chain-specs" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "sc-chain-spec", "sc-service", @@ -9959,7 +10033,7 @@ dependencies = [ [[package]] name = "sp-consensus-subspace" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "async-trait", "log", @@ -10074,7 +10148,7 @@ dependencies = [ [[package]] name = "sp-domain-digests" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -10083,7 +10157,7 @@ dependencies = [ [[package]] name = "sp-domains" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "blake2", "domain-runtime-primitives", @@ -10114,7 +10188,7 @@ dependencies = [ [[package]] name = "sp-domains-fraud-proof" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "domain-block-preprocessor", "domain-runtime-primitives", @@ -10146,7 +10220,7 @@ dependencies = [ [[package]] name = "sp-executive" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "async-trait", "parity-scale-codec", @@ -10238,7 +10312,7 @@ dependencies = [ [[package]] name = "sp-messenger" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "frame-support", "hash-db", @@ -10268,7 +10342,7 @@ dependencies = [ [[package]] name = "sp-objects" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "sp-api", "sp-std", @@ -10580,7 +10654,7 @@ dependencies = [ [[package]] name = "space-acres" -version = "0.0.13" +version = "0.0.14" dependencies = [ "anyhow", "arc-swap", @@ -10792,7 +10866,7 @@ dependencies = [ [[package]] name = "subspace-archiving" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "parity-scale-codec", "rayon", @@ -10805,7 +10879,7 @@ dependencies = [ [[package]] name = "subspace-core-primitives" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "blake3", "derive_more", @@ -10828,7 +10902,7 @@ dependencies = [ [[package]] name = "subspace-erasure-coding" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "kzg", "rust-kzg-blst", @@ -10838,7 +10912,7 @@ dependencies = [ [[package]] name = "subspace-farmer" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "anyhow", "async-lock 2.8.0", @@ -10856,9 +10930,13 @@ dependencies = [ "fs4 0.7.0", "futures", "hex", + "hwlocality", "jsonrpsee", + "libc", + "libmimalloc-sys", "lru 0.11.1", "mimalloc", + "num_cpus", "parity-scale-codec", "parking_lot 0.12.1", "prometheus-client 0.22.0", @@ -10885,13 +10963,14 @@ dependencies = [ "tracing", "tracing-subscriber 0.3.18", "ulid", + "windows-sys 0.52.0", "zeroize", ] [[package]] name = "subspace-farmer-components" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "async-lock 2.8.0", "async-trait", @@ -10921,7 +11000,7 @@ dependencies = [ [[package]] name = "subspace-metrics" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "actix-web", "parking_lot 0.12.1", @@ -10933,7 +11012,7 @@ dependencies = [ [[package]] name = "subspace-networking" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "actix-web", "async-mutex", @@ -10972,7 +11051,7 @@ dependencies = [ [[package]] name = "subspace-proof-of-space" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "chacha20", "derive_more", @@ -10985,7 +11064,7 @@ dependencies = [ [[package]] name = "subspace-proof-of-time" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "aes 0.8.3", "subspace-core-primitives", @@ -10995,7 +11074,7 @@ dependencies = [ [[package]] name = "subspace-rpc-primitives" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "hex", "serde", @@ -11007,7 +11086,7 @@ dependencies = [ [[package]] name = "subspace-runtime" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "domain-runtime-primitives", "frame-benchmarking", @@ -11059,7 +11138,7 @@ dependencies = [ [[package]] name = "subspace-runtime-primitives" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "pallet-transaction-payment", "serde", @@ -11072,7 +11151,7 @@ dependencies = [ [[package]] name = "subspace-service" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "async-trait", "atomic", @@ -11142,7 +11221,7 @@ dependencies = [ [[package]] name = "subspace-verification" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=196a0086f4240541cf3f48ee26db1c3251128d5a#196a0086f4240541cf3f48ee26db1c3251128d5a" dependencies = [ "parity-scale-codec", "schnorrkel", diff --git a/Cargo.toml b/Cargo.toml index f913225c..c98a99c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "space-acres" description = "Space Acres is an opinionated unofficial GUI application for farming on Subspace Network" license = "0BSD" -version = "0.0.13" +version = "0.0.14" authors = ["Nazar Mokrynskyi "] repository = "https://github.com/nazar-pc/space-acres" edition = "2021" @@ -62,23 +62,23 @@ sc-informant = { git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b sc-network = { git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", default-features = false } sc-service = { git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", default-features = false } sc-storage-monitor = { git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", default-features = false } -sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } serde = { version = "1.0.193", features = ["derive"]} serde_json = "1.0.108" sp-core = { git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", default-features = false } -sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -sp-domains-fraud-proof = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } +sp-domains-fraud-proof = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } sp-runtime = { git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", default-features = false } -subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-networking = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-service = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } +subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } +subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a", default-features = false } +subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } +subspace-networking = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } +subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } +subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } +subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } +subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } +subspace-service = { git = "https://github.com/subspace/subspace", rev = "196a0086f4240541cf3f48ee26db1c3251128d5a" } supports-color = "2.0.0" thiserror = "1.0.50" tokio = { version = "1.34.0", features = ["fs", "time"] } @@ -88,6 +88,12 @@ tracing-subscriber = "0.3.18" [target.'cfg(windows)'.build-dependencies] winres = "0.1.12" +[features] +default = ["numa"] +numa = [ + "subspace-farmer/numa" +] + # The list of dependencies below (which can be both direct and indirect dependencies) are crates # that are suspected to be CPU-intensive, and that are unlikely to require debugging (as some of # their debug info might be missing) or to require to be frequently recompiled. We compile these diff --git a/src/backend/farmer.rs b/src/backend/farmer.rs index 4ec169e4..6394b801 100644 --- a/src/backend/farmer.rs +++ b/src/backend/farmer.rs @@ -27,7 +27,10 @@ use subspace_farmer::single_disk_farm::{ use subspace_farmer::utils::farmer_piece_getter::FarmerPieceGetter; use subspace_farmer::utils::piece_validator::SegmentCommitmentPieceValidator; use subspace_farmer::utils::readers_and_pieces::ReadersAndPieces; -use subspace_farmer::utils::run_future_in_dedicated_thread; +use subspace_farmer::utils::{ + all_cpu_cores, create_tokio_thread_pool_manager_for_pinned_nodes, + run_future_in_dedicated_thread, thread_pool_core_indices, +}; use subspace_farmer::{NodeClient, NodeRpcClient}; use subspace_farmer_components::plotting::PlottedSector; use subspace_networking::utils::piece_provider::PieceProvider; @@ -137,23 +140,12 @@ impl fmt::Debug for Farmer { } } -fn available_parallelism() -> usize { - match std::thread::available_parallelism() { - Ok(parallelism) => parallelism.get(), - Err(error) => { - warn!( - %error, - "Unable to identify available parallelism, you might want to configure thread pool sizes with CLI \ - options manually" - ); - - 0 - } - } -} - fn should_farm_during_initial_plotting() -> bool { - available_parallelism() > 8 + let total_cpu_cores = all_cpu_cores() + .iter() + .flat_map(|set| set.cpu_cores()) + .count(); + total_cpu_cores > 8 } #[derive(Debug, Clone)] @@ -204,14 +196,6 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu } } - let sector_downloading_concurrency = NonZeroUsize::new(2).expect("Not zero; qed"); - let sector_encoding_concurrency = NonZeroUsize::new(1).expect("Not zero; qed"); - let farm_during_initial_plotting = should_farm_during_initial_plotting(); - let available_parallelism = available_parallelism(); - let farming_thread_pool_size = available_parallelism; - let plotting_thread_pool_size = available_parallelism; - let replotting_thread_pool_size = available_parallelism / 2; - let farmer_app_info = node_client .farmer_app_info() .await @@ -252,8 +236,48 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu let mut single_disk_farms = Vec::with_capacity(disk_farms.len()); - let downloading_semaphore = Arc::new(Semaphore::new(sector_downloading_concurrency.get())); - let encoding_semaphore = Arc::new(Semaphore::new(sector_encoding_concurrency.get())); + let farm_during_initial_plotting = should_farm_during_initial_plotting(); + let plotting_thread_pool_core_indices = thread_pool_core_indices(None, None); + let replotting_thread_pool_core_indices = { + let mut replotting_thread_pool_core_indices = thread_pool_core_indices(None, None); + // The default behavior is to use all CPU cores, but for replotting we just want half + replotting_thread_pool_core_indices + .iter_mut() + .for_each(|set| set.truncate(set.cpu_cores().len() / 2)); + replotting_thread_pool_core_indices + }; + + let downloading_semaphore = + Arc::new(Semaphore::new(plotting_thread_pool_core_indices.len() + 1)); + + let all_cpu_cores = all_cpu_cores(); + let plotting_thread_pool_manager = create_tokio_thread_pool_manager_for_pinned_nodes( + "plotting", + plotting_thread_pool_core_indices, + )?; + let replotting_thread_pool_manager = create_tokio_thread_pool_manager_for_pinned_nodes( + "replotting", + replotting_thread_pool_core_indices, + )?; + let farming_thread_pool_size = all_cpu_cores + .first() + .expect("Not empty according to function description; qed") + .cpu_cores() + .len(); + + // TODO: Expose this in UI somehow, maybe warning if not enough farms are configured too + if all_cpu_cores.len() > 1 { + info!(numa_nodes = %all_cpu_cores.len(), "NUMA system detected"); + + if all_cpu_cores.len() < disk_farms.len() { + warn!( + numa_nodes = %all_cpu_cores.len(), + farms_count = %disk_farms.len(), + "Too few disk farms, CPU will not be utilized fully during plotting, same number \ + of farms as NUMA nodes or more is recommended" + ); + } + } let mut plotting_delay_senders = Vec::with_capacity(disk_farms.len()); @@ -274,11 +298,10 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu piece_getter: piece_getter.clone(), cache_percentage: CACHE_PERCENTAGE, downloading_semaphore: Arc::clone(&downloading_semaphore), - encoding_semaphore: Arc::clone(&encoding_semaphore), farm_during_initial_plotting, farming_thread_pool_size, - plotting_thread_pool_size, - replotting_thread_pool_size, + plotting_thread_pool_manager: plotting_thread_pool_manager.clone(), + replotting_thread_pool_manager: replotting_thread_pool_manager.clone(), plotting_delay: Some(plotting_delay_receiver), }, disk_farm_index,