From c6a7d99614dd299f215d4591b623cf9a90da0bc9 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Tue, 26 Dec 2023 08:35:03 +0200 Subject: [PATCH] Fix docs, CI fixes --- .github/workflows/rust.yml | 14 ++++++-- .github/workflows/snapshot-build.yml | 49 +++++++++++++++++++++++----- Cargo.lock | 10 +++--- Dockerfile-bootstrap-node | 4 ++- Dockerfile-bootstrap-node.aarch64 | 5 ++- Dockerfile-farmer | 4 ++- Dockerfile-farmer.aarch64 | 5 ++- Dockerfile-node | 4 ++- Dockerfile-node.aarch64 | 5 ++- Dockerfile-runtime | 5 ++- crates/subspace-farmer/Cargo.toml | 3 +- crates/subspace-farmer/README.md | 35 ++++++++++++++++++-- crates/subspace-node/README.md | 34 +++++++++++++++++-- docs/development.md | 36 ++------------------ docs/farming.md | 5 +-- 15 files changed, 152 insertions(+), 66 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 64f5081cc3b..8a06facccd7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -32,7 +32,7 @@ env: jobs: cargo-fmt: - runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-22.04') }} + runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }} steps: - name: Checkout @@ -91,6 +91,11 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} + # Needed for hwloc + - name: Install automake (macOS) + run: brew install automake + if: runner.os == 'macOS' + # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll - name: Remove msys64 run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse @@ -113,7 +118,7 @@ jobs: args: --locked --all-targets --features "runtime-benchmarks" -- -D warnings cargo-docs: - runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-22.04') }} + runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }} steps: - name: Checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # @v3.1.0 @@ -170,6 +175,11 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} + # Needed for hwloc + - name: Install automake (macOS) + run: brew install automake + if: runner.os == 'macOS' + # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll - name: Remove msys64 run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse diff --git a/.github/workflows/snapshot-build.yml b/.github/workflows/snapshot-build.yml index 3a3bfff9a20..d664134ffda 100644 --- a/.github/workflows/snapshot-build.yml +++ b/.github/workflows/snapshot-build.yml @@ -18,7 +18,7 @@ env: jobs: container-linux: - runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-22.04') }} + runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }} permissions: contents: write packages: write @@ -84,35 +84,35 @@ jobs: strategy: matrix: build: - - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-20.04') }} + - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} target: x86_64-unknown-linux-gnu suffix: ubuntu-x86_64-skylake-${{ github.ref_name }} rustflags: "-C target-cpu=skylake" - - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-20.04') }} + - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} target: x86_64-unknown-linux-gnu suffix: ubuntu-x86_64-v2-${{ github.ref_name }} rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes" - - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-20.04') }} + - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} target: aarch64-unknown-linux-gnu suffix: ubuntu-aarch64-${{ github.ref_name }} # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate bumps MSRV to # at least 1.61: https://github.com/RustCrypto/block-ciphers/issues/373 rustflags: "-C linker=aarch64-linux-gnu-gcc --cfg aes_armv8" - - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || 'macos-12') }} + - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || '"macos-12"') }} target: aarch64-apple-darwin suffix: macos-aarch64-${{ github.ref_name }} # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate bumps MSRV to # at least 1.61: https://github.com/RustCrypto/block-ciphers/issues/373 rustflags: "--cfg aes_armv8" - - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || 'macos-12') }} + - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || '"macos-12"') }} target: x86_64-apple-darwin suffix: macos-x86_64-${{ github.ref_name }} rustflags: "" - - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || 'windows-2022') }} + - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }} target: x86_64-pc-windows-msvc suffix: windows-x86_64-skylake-${{ github.ref_name }} rustflags: "-C target-cpu=skylake" - - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || 'windows-2022') }} + - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }} target: x86_64-pc-windows-msvc suffix: windows-x86_64-v2-${{ github.ref_name }} rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes" @@ -148,6 +148,11 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} + # Needed for hwloc + - name: Install automake (macOS) + run: brew install automake + if: runner.os == 'macOS' + # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll - name: Remove msys64 run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse @@ -156,7 +161,33 @@ jobs: continue-on-error: true - name: AArch64 cross-compile packages - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends g++-aarch64-linux-gnu gcc-aarch64-linux-gnu libc6-dev-arm64-cross + run: | + FLAVOR="$(lsb_release -sc)" + + sudo tee /etc/apt/sources.list.d/arm64.list <> $GITHUB_ENV if: matrix.build.target == 'aarch64-unknown-linux-gnu' - name: Build farmer diff --git a/Cargo.lock b/Cargo.lock index 8de73c39401..41b02a582e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4788,8 +4788,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwlocality" -version = "1.0.0" -source = "git+https://github.com/HadrienG2/hwlocality?rev=3141847b0a463f38adcf623a2d720931757a38ae#3141847b0a463f38adcf623a2d720931757a38ae" +version = "1.0.0-alpha.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6020affad7f95b46f12607a8714aa70bd02c8df3b3abf9ef5c8cd2f7ae57a033" dependencies = [ "arrayvec 0.7.4", "bitflags 2.4.0", @@ -4803,8 +4804,9 @@ dependencies = [ [[package]] name = "hwlocality-sys" -version = "0.2.0" -source = "git+https://github.com/HadrienG2/hwlocality?rev=3141847b0a463f38adcf623a2d720931757a38ae#3141847b0a463f38adcf623a2d720931757a38ae" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "381b203a23b41c29be64454e9cee8d16360606d7e871f5d22532796b6095f164" dependencies = [ "autotools", "cmake", diff --git a/Dockerfile-bootstrap-node b/Dockerfile-bootstrap-node index b5c9e62f6ba..85e5109e7fd 100644 --- a/Dockerfile-bootstrap-node +++ b/Dockerfile-bootstrap-node @@ -19,7 +19,9 @@ RUN \ git \ llvm \ clang \ - cmake \ + automake \ + libtool \ + pkg-config \ make && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION diff --git a/Dockerfile-bootstrap-node.aarch64 b/Dockerfile-bootstrap-node.aarch64 index e338397b70c..6b0902da885 100644 --- a/Dockerfile-bootstrap-node.aarch64 +++ b/Dockerfile-bootstrap-node.aarch64 @@ -19,7 +19,9 @@ RUN \ git \ llvm \ clang \ - cmake \ + automake \ + libtool \ + pkg-config \ make && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION @@ -39,6 +41,7 @@ COPY test /code/test # Up until this line all Rust images in this repo should be the same to share the same layers ENV RUSTFLAGS="${RUSTFLAGS} -C linker=aarch64-linux-gnu-gcc" +ENV PKG_CONFIG_ALLOW_CROSS=true # Dependencies necessary for successful cross-compilation RUN \ diff --git a/Dockerfile-farmer b/Dockerfile-farmer index 186be745655..dd32f45ab07 100644 --- a/Dockerfile-farmer +++ b/Dockerfile-farmer @@ -19,7 +19,9 @@ RUN \ git \ llvm \ clang \ - cmake \ + automake \ + libtool \ + pkg-config \ make && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION diff --git a/Dockerfile-farmer.aarch64 b/Dockerfile-farmer.aarch64 index 42a0dc0e337..11046baa2c4 100644 --- a/Dockerfile-farmer.aarch64 +++ b/Dockerfile-farmer.aarch64 @@ -19,7 +19,9 @@ RUN \ git \ llvm \ clang \ - cmake \ + automake \ + libtool \ + pkg-config \ make && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION @@ -39,6 +41,7 @@ COPY test /code/test # Up until this line all Rust images in this repo should be the same to share the same layers ENV RUSTFLAGS="${RUSTFLAGS} -C linker=aarch64-linux-gnu-gcc" +ENV PKG_CONFIG_ALLOW_CROSS=true # Dependencies necessary for successful cross-compilation RUN \ diff --git a/Dockerfile-node b/Dockerfile-node index dd50db59ae6..a3775ac1fae 100644 --- a/Dockerfile-node +++ b/Dockerfile-node @@ -19,7 +19,9 @@ RUN \ git \ llvm \ clang \ - cmake \ + automake \ + libtool \ + pkg-config \ make && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION diff --git a/Dockerfile-node.aarch64 b/Dockerfile-node.aarch64 index 55693d3816c..0dd70f8822a 100644 --- a/Dockerfile-node.aarch64 +++ b/Dockerfile-node.aarch64 @@ -19,7 +19,9 @@ RUN \ git \ llvm \ clang \ - cmake \ + automake \ + libtool \ + pkg-config \ make && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION @@ -39,6 +41,7 @@ COPY test /code/test # Up until this line all Rust images in this repo should be the same to share the same layers ENV RUSTFLAGS="${RUSTFLAGS} -C linker=aarch64-linux-gnu-gcc" +ENV PKG_CONFIG_ALLOW_CROSS=true # Dependencies necessary for successful cross-compilation RUN \ diff --git a/Dockerfile-runtime b/Dockerfile-runtime index 6be061e62c7..3bd02e65fa6 100644 --- a/Dockerfile-runtime +++ b/Dockerfile-runtime @@ -14,11 +14,14 @@ RUN \ apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ + protobuf-compiler \ curl \ git \ llvm \ clang \ - cmake \ + automake \ + libtool \ + pkg-config \ make && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION diff --git a/crates/subspace-farmer/Cargo.toml b/crates/subspace-farmer/Cargo.toml index e2940cd8a90..ef7317bfe3c 100644 --- a/crates/subspace-farmer/Cargo.toml +++ b/crates/subspace-farmer/Cargo.toml @@ -28,8 +28,7 @@ fdlimit = "0.3.0" fs4 = "0.7.0" futures = "0.3.29" hex = { version = "0.4.3", features = ["serde"] } -# TODO: Upgrade to crates.io version once it is published -hwlocality = { git = "https://github.com/HadrienG2/hwlocality", rev = "3141847b0a463f38adcf623a2d720931757a38ae", features = ["bundled"], optional = true } +hwlocality = { version = "1.0.0-alpha.1", features = ["vendored"], optional = true } jsonrpsee = { version = "0.16.3", features = ["client"] } lru = "0.11.0" mimalloc = "0.1.39" diff --git a/crates/subspace-farmer/README.md b/crates/subspace-farmer/README.md index 707e5ba20f0..be6e8c2f534 100644 --- a/crates/subspace-farmer/README.md +++ b/crates/subspace-farmer/README.md @@ -10,11 +10,42 @@ It is recommended to follow general farming instructions that explain how to run Rust toolchain is expected to be installed for anything in this repository to compile, but there are some extra dependencies for farmer specifically. -Autotools and `libtool` on Linux/macOS or CMake on Windows for `hwlocality-sys` (if `numa` features is enabled, it is by default), also LLVM/Clang is necessary: +`automake`,`libtool` and `pkg-config` on Linux/macOS or CMake on Windows for `hwlocality-sys` (if `numa` features is enabled, it is by default), also LLVM/Clang is necessary. + +### Ubuntu + +```bash +sudo apt-get install automake libtool pkg-config llvm clang +``` + +### macOS + +1. Install via Homebrew: + +```bash +brew install automake libtool llvm@15 clang +``` + +2. Add `llvm` to your `~/.zshrc` or `~/.bashrc`: + +```bash +export PATH="/opt/homebrew/opt/llvm@15/bin:$PATH" +``` + +3. Activate the changes: + ```bash -sudo apt-get install llvm clang libtool +source ~/.zshrc ``` +4. Verify that `llvm` is installed: + +```bash +llvm-config --version +``` + +### Build + Then build the farmer using Cargo: ``` cargo build --profile production --bin subspace-farmer diff --git a/crates/subspace-node/README.md b/crates/subspace-node/README.md index af9d1988c5c..dc8667bbe7d 100644 --- a/crates/subspace-node/README.md +++ b/crates/subspace-node/README.md @@ -14,12 +14,42 @@ It is recommended to follow general farming instructions that explain how to run Rust toolchain is expected to be installed for anything in this repository to compile, but there are some extra dependencies for farmer specifically. -Prost library from libp2p dependency needs CMake, also LLVM/Clang and `make` are necessary: +### Ubuntu + +LLVM/Clang and `make` are necessary: ```bash sudo apt-get install llvm clang cmake make ``` -Then build the farmer using Cargo: +### macOS + +1. Install via Homebrew: + +```bash +brew install llvm@15 clang cmake make +``` + +2. Add `llvm` to your `~/.zshrc` or `~/.bashrc`: + +```bash +export PATH="/opt/homebrew/opt/llvm@15/bin:$PATH" +``` + +3. Activate the changes: + +```bash +source ~/.zshrc +``` + +4. Verify that `llvm` is installed: + +```bash +llvm-config --version +``` + +### Build + +Then build the node using Cargo: ``` cargo build --profile production --bin subspace-node target/production/subspace-node --version diff --git a/docs/development.md b/docs/development.md index f55fd3ef333..596dfb54c9a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -2,39 +2,7 @@ You'll have to have [Rust toolchain](https://rustup.rs/) installed as well as LLVM, Clang and CMake in addition to usual developer tooling. -Below are some examples of how to install these dependencies on different operating systems. - -### Ubuntu - -```bash -sudo apt-get install llvm clang cmake -``` - -### macOS - -1. Install via Homebrew: - -```bash -brew install llvm@15 clang cmake -``` - -2. Add `llvm` to your `~/.zshrc` or `~/.bashrc`: - -```bash -export PATH="/opt/homebrew/opt/llvm@15/bin:$PATH" -``` - -3. Activate the changes: - -```bash -source ~/.zshrc -``` - -4. Verify that `llvm` is installed: - -```bash -llvm-config --version -``` +Check [crates/subspace-node](../crates/subspace-node/README.md) and [crates/subspace-farmer](../crates/subspace-farmer/README.md) for required dependencies. ## To Farm By Yourself (Offline) @@ -43,7 +11,7 @@ llvm-config --version **Linux/MacOS:** -1. Make them executable: `chmod +x subspace-farmer-x86_64-*-snapshot subspace-node-x86_64-*-snapshot` +1. Make files executable: `chmod +x subspace-farmer-x86_64-*-snapshot subspace-node-x86_64-*-snapshot` 2. Run the node: `./subspace-node-x86_64-*-snapshot --dev --tmp` 3. In macOS, it may prompt that this app is not verified. Click on `cancel` instead of moving it to trash. To allow execution, go to `System Preferences -> Security & Privacy -> General`, and click on `allow`. diff --git a/docs/farming.md b/docs/farming.md index 3c65bf98af4..6c9c3c60453 100644 --- a/docs/farming.md +++ b/docs/farming.md @@ -349,10 +349,7 @@ If you're running unsupported Linux distribution or CPU architecture, you may tr NOTE: This is primarily targeted at tech-savvy users and not recommended unless you know what you're doing. Please try to find answer to your question online before reaching out to maintainers. -You'll have to have [Rust toolchain](https://rustup.rs/) installed as well as LLVM, Clang and CMake in addition to usual developer tooling (Ubuntu example): -```bash -sudo apt-get install llvm clang cmake -``` +Check [crates/subspace-node](../crates/subspace-node/README.md) and [crates/subspace-farmer](../crates/subspace-farmer/README.md) for required dependencies. Now clone the source and build snapshot `snapshot-2022-apr-29` (replace occurrences with the snapshot you want to build): ```bash