Fix docs, switch to off-the-shelf-runners for testing purposes

.github/workflows/rust.yml

@@ -32,7 +32,7 @@ env:
 
 jobs:
   cargo-fmt:
-    runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-22.04') }}
+    runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }}
 
     steps:
       - name: Checkout
@@ -60,7 +60,7 @@ jobs:
   cargo-clippy:
     strategy:
       matrix:
-        os: ${{ fromJson(github.repository_owner == 'subspace' && '[["self-hosted", "ubuntu-20.04-x86-64"], ["self-hosted", "macos-12-arm64"], ["self-hosted", "windows-server-2022-x86-64"]]' || '["ubuntu-22.04", "macos-12", "windows-2022"]') }}
+        os: ${{ fromJson(github.repository_owner == 'fake' && '[["self-hosted", "ubuntu-20.04-x86-64"], ["self-hosted", "macos-12-arm64"], ["self-hosted", "windows-server-2022-x86-64"]]' || '["ubuntu-22.04", "macos-12", "windows-2022"]') }}
 
     runs-on: ${{ matrix.os }}
 
@@ -91,6 +91,11 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
+      # Needed for hwloc
+      - name: Install automake (macOS)
+        run: brew install automake
+        if: runner.os == 'macOS'
+
       # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll
       - name: Remove msys64
         run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse
@@ -113,7 +118,7 @@ jobs:
           args: --locked --all-targets --features "runtime-benchmarks" -- -D warnings
 
   cargo-docs:
-    runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-22.04') }}
+    runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }}
     steps:
       - name: Checkout
         uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # @v3.1.0
@@ -139,7 +144,7 @@ jobs:
   cargo-test:
     strategy:
       matrix:
-        os: ${{ fromJson(github.repository_owner == 'subspace' && '[["self-hosted", "ubuntu-20.04-x86-64"], ["self-hosted", "macos-12-arm64"], ["self-hosted", "windows-server-2022-x86-64"]]' || '["ubuntu-22.04", "macos-12", "windows-2022"]') }}
+        os: ${{ fromJson(github.repository_owner == 'fake' && '[["self-hosted", "ubuntu-20.04-x86-64"], ["self-hosted", "macos-12-arm64"], ["self-hosted", "windows-server-2022-x86-64"]]' || '["ubuntu-22.04", "macos-12", "windows-2022"]') }}
 
     runs-on: ${{ matrix.os }}
 
@@ -170,6 +175,11 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
+      # Needed for hwloc
+      - name: Install automake (macOS)
+        run: brew install automake
+        if: runner.os == 'macOS'
+
       # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll
       - name: Remove msys64
         run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse

.github/workflows/snapshot-build.yml

@@ -18,7 +18,7 @@ env:
 
 jobs:
   container-linux:
-    runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-22.04') }}
+    runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }}
     permissions:
       contents: write
       packages: write
@@ -84,35 +84,35 @@ jobs:
     strategy:
       matrix:
         build:
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-20.04') }}
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
             target: x86_64-unknown-linux-gnu
             suffix: ubuntu-x86_64-skylake-${{ github.ref_name }}
             rustflags: "-C target-cpu=skylake"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-20.04') }}
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
             target: x86_64-unknown-linux-gnu
             suffix: ubuntu-x86_64-v2-${{ github.ref_name }}
             rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-20.04') }}
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
             target: aarch64-unknown-linux-gnu
             suffix: ubuntu-aarch64-${{ github.ref_name }}
             # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate bumps MSRV to
             #  at least 1.61: https://github.com/RustCrypto/block-ciphers/issues/373
             rustflags: "-C linker=aarch64-linux-gnu-gcc --cfg aes_armv8"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || 'macos-12') }}
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || '"macos-12"') }}
             target: aarch64-apple-darwin
             suffix: macos-aarch64-${{ github.ref_name }}
             # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate bumps MSRV to
             #  at least 1.61: https://github.com/RustCrypto/block-ciphers/issues/373
             rustflags: "--cfg aes_armv8"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || 'macos-12') }}
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || '"macos-12"') }}
             target: x86_64-apple-darwin
             suffix: macos-x86_64-${{ github.ref_name }}
             rustflags: ""
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || 'windows-2022') }}
+          - os: ${{ fromJson(github.repository_owner == 'fake' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }}
             target: x86_64-pc-windows-msvc
             suffix: windows-x86_64-skylake-${{ github.ref_name }}
             rustflags: "-C target-cpu=skylake"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || 'windows-2022') }}
+          - os: ${{ fromJson(github.repository_owner == 'fake' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }}
             target: x86_64-pc-windows-msvc
             suffix: windows-x86_64-v2-${{ github.ref_name }}
             rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes"
@@ -148,6 +148,11 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
+      # Needed for hwloc
+      - name: Install automake (macOS)
+        run: brew install automake
+        if: runner.os == 'macOS'
+
       # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll
       - name: Remove msys64
         run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse
@@ -156,7 +161,9 @@ jobs:
         continue-on-error: true
 
       - name: AArch64 cross-compile packages
-        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends g++-aarch64-linux-gnu gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+        run: |
+          sudo apt-get update && sudo apt-get install -y --no-install-recommends g++-aarch64-linux-gnu gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+          echo "PKG_CONFIG_ALLOW_CROSS=true" >> $GITHUB_ENV
         if: matrix.build.target == 'aarch64-unknown-linux-gnu'
 
       - name: Build farmer

Dockerfile-bootstrap-node

@@ -19,7 +19,9 @@ RUN \
         git \
         llvm \
         clang \
-        cmake \
+        automake \
+        libtool \
+        pkg-config \
         make && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION
 

Dockerfile-bootstrap-node.aarch64

@@ -19,7 +19,9 @@ RUN \
         git \
         llvm \
         clang \
-        cmake \
+        automake \
+        libtool \
+        pkg-config \
         make && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION
 
@@ -39,6 +41,7 @@ COPY test /code/test
 # Up until this line all Rust images in this repo should be the same to share the same layers
 
 ENV RUSTFLAGS="${RUSTFLAGS} -C linker=aarch64-linux-gnu-gcc"
+ENV PKG_CONFIG_ALLOW_CROSS=true
 
 # Dependencies necessary for successful cross-compilation
 RUN \

Dockerfile-farmer

@@ -19,7 +19,9 @@ RUN \
         git \
         llvm \
         clang \
-        cmake \
+        automake \
+        libtool \
+        pkg-config \
         make && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION
 

Dockerfile-farmer.aarch64

@@ -19,7 +19,9 @@ RUN \
         git \
         llvm \
         clang \
-        cmake \
+        automake \
+        libtool \
+        pkg-config \
         make && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION
 
@@ -39,6 +41,7 @@ COPY test /code/test
 # Up until this line all Rust images in this repo should be the same to share the same layers
 
 ENV RUSTFLAGS="${RUSTFLAGS} -C linker=aarch64-linux-gnu-gcc"
+ENV PKG_CONFIG_ALLOW_CROSS=true
 
 # Dependencies necessary for successful cross-compilation
 RUN \

Dockerfile-node

@@ -19,7 +19,9 @@ RUN \
         git \
         llvm \
         clang \
-        cmake \
+        automake \
+        libtool \
+        pkg-config \
         make && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION
 

Dockerfile-node.aarch64

@@ -19,7 +19,9 @@ RUN \
         git \
         llvm \
         clang \
-        cmake \
+        automake \
+        libtool \
+        pkg-config \
         make && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION
 
@@ -39,6 +41,7 @@ COPY test /code/test
 # Up until this line all Rust images in this repo should be the same to share the same layers
 
 ENV RUSTFLAGS="${RUSTFLAGS} -C linker=aarch64-linux-gnu-gcc"
+ENV PKG_CONFIG_ALLOW_CROSS=true
 
 # Dependencies necessary for successful cross-compilation
 RUN \

Dockerfile-runtime

@@ -14,11 +14,14 @@ RUN \
     apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         ca-certificates \
+        protobuf-compiler \
         curl \
         git \
         llvm \
         clang \
-        cmake \
+        automake \
+        libtool \
+        pkg-config \
         make && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION
 

crates/subspace-farmer/Cargo.toml

@@ -29,7 +29,7 @@ fs4 = "0.7.0"
 futures = "0.3.29"
 hex = { version = "0.4.3", features = ["serde"] }
 # TODO: Upgrade to crates.io version once it is published
-hwlocality = { git = "https://github.com/HadrienG2/hwlocality", rev = "3141847b0a463f38adcf623a2d720931757a38ae", features = ["bundled"], optional = true }
+hwlocality = { git = "https://github.com/HadrienG2/hwlocality", rev = "b57ac2d45cadae33fef4bd427f40d2902d78ee4e", features = ["bundled"], optional = true }
 jsonrpsee = { version = "0.16.3", features = ["client"] }
 lru = "0.11.0"
 mimalloc = "0.1.39"

crates/subspace-farmer/README.md

@@ -10,9 +10,37 @@ It is recommended to follow general farming instructions that explain how to run
 
 Rust toolchain is expected to be installed for anything in this repository to compile, but there are some extra dependencies for farmer specifically.
 
-Autotools and `libtool` on Linux/macOS or CMake on Windows for `hwlocality-sys` (if `numa` features is enabled, it is by default), also LLVM/Clang is necessary:
+### Ubuntu
+
+`automake`,`libtool` and `pkg-config` on Linux/macOS or CMake on Windows for `hwlocality-sys` (if `numa` features is enabled, it is by default), also LLVM/Clang is necessary:
+```bash
+sudo apt-get install automake libtool pkg-config llvm clang
+```
+
+### macOS
+
+1. Install via Homebrew:
+
+```bash
+brew install automake llvm@15 clang cmake
+```
+
+2. Add `llvm` to your `~/.zshrc` or `~/.bashrc`:
+
+```bash
+export PATH="/opt/homebrew/opt/llvm@15/bin:$PATH"
+```
+
+3. Activate the changes:
+
+```bash
+source ~/.zshrc
+```
+
+4. Verify that `llvm` is installed:
+
 ```bash
-sudo apt-get install llvm clang libtool
+llvm-config --version
 ```
 
 Then build the farmer using Cargo:

crates/subspace-node/README.md

@@ -14,12 +14,39 @@ It is recommended to follow general farming instructions that explain how to run
 
 Rust toolchain is expected to be installed for anything in this repository to compile, but there are some extra dependencies for farmer specifically.
 
-Prost library from libp2p dependency needs CMake, also LLVM/Clang and `make` are necessary:
+### Ubuntu
+
+LLVM/Clang and `make` are necessary:
 ```bash
 sudo apt-get install llvm clang cmake make
 ```
 
-Then build the farmer using Cargo:
+1. Install via Homebrew:
+
+```bash
+brew install llvm@15 clang cmake make
+```
+
+2. Add `llvm` to your `~/.zshrc` or `~/.bashrc`:
+
+```bash
+export PATH="/opt/homebrew/opt/llvm@15/bin:$PATH"
+```
+
+3. Activate the changes:
+
+```bash
+source ~/.zshrc
+```
+
+4. Verify that `llvm` is installed:
+
+```bash
+llvm-config --version
+```
+
+
+Then build the node using Cargo:
 ```
 cargo build --profile production --bin subspace-node
 target/production/subspace-node --version

docs/development.md

@@ -2,39 +2,7 @@
 
 You'll have to have [Rust toolchain](https://rustup.rs/) installed as well as LLVM, Clang and CMake in addition to usual developer tooling.
 
-Below are some examples of how to install these dependencies on different operating systems.
-
-### Ubuntu
-
-```bash
-sudo apt-get install llvm clang cmake
-```
-
-### macOS
-
-1. Install via Homebrew:
-
-```bash
-brew install llvm@15 clang cmake
-```
-
-2. Add `llvm` to your `~/.zshrc` or `~/.bashrc`:
-
-```bash
-export PATH="/opt/homebrew/opt/llvm@15/bin:$PATH"
-```
-
-3. Activate the changes:
-
-```bash
-source ~/.zshrc
-```
-
-4. Verify that `llvm` is installed:
-
-```bash
-llvm-config --version
-```
+Check [crates/subspace-node](../crates/subspace-node/README.md) and [crates/subspace-farmer](../crates/subspace-farmer/README.md) for required dependencies.
 
 ## To Farm By Yourself (Offline)
 
@@ -43,7 +11,7 @@ llvm-config --version
 
 **Linux/MacOS:**
 
-1. Make them executable: `chmod +x subspace-farmer-x86_64-*-snapshot subspace-node-x86_64-*-snapshot`
+1. Make files executable: `chmod +x subspace-farmer-x86_64-*-snapshot subspace-node-x86_64-*-snapshot`
 2. Run the node: `./subspace-node-x86_64-*-snapshot --dev --tmp`
 3. In macOS, it may prompt that this app is not verified. Click on `cancel` instead of moving it to trash.
    To allow execution, go to `System Preferences -> Security & Privacy -> General`, and click on `allow`.