diff --git a/.github/workflows/c-bench.yml b/.github/workflows/c-bench.yml new file mode 100644 index 000000000..9398a57db --- /dev/null +++ b/.github/workflows/c-bench.yml @@ -0,0 +1,55 @@ +name: Benchmark C + +on: + workflow_dispatch: + merge_group: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmark: + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-latest + - windows-latest + + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v4 + + - name: 🔨 Build libcrux-ml-kem/c + working-directory: libcrux-ml-kem/c + run: | + cmake -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build --config Release + + # FIXME: Benchmarks on Windows CI are not working right now. + # - name: 🏃🏻‍♀️ Benchmark + # working-directory: libcrux-ml-kem/c + # run: ./build/Release/ml_kem_bench + # if: ${{ matrix.os == 'windows-latest' }} + + - name: 🏃🏻‍♀️ Benchmark (libcrux-ml-kem/c) + working-directory: libcrux-ml-kem/c + run: ./build/ml_kem_bench + if: ${{ matrix.os != 'windows-latest' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }} + + - name: 🔨 Build libcrux-ml-kem/cg + working-directory: libcrux-ml-kem/cg + run: | + cmake -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build --config Release + + - name: 🏃🏻‍♀️ Benchmark libcrux-ml-kem/cg + working-directory: libcrux-ml-kem/cg + run: ./build/ml_kem_bench + if: ${{ matrix.os != 'windows-latest' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }} diff --git a/.github/workflows/c.yml b/.github/workflows/c.yml index c734bf0de..285eac24a 100644 --- a/.github/workflows/c.yml +++ b/.github/workflows/c.yml @@ -14,7 +14,6 @@ concurrency: jobs: extract: - if: ${{ github.event_name != 'merge_group' }} runs-on: ubuntu-latest container: franziskus/libcrux-c:latest defaults: @@ -36,7 +35,6 @@ jobs: if-no-files-found: error extract-header-only: - if: ${{ github.event_name != 'merge_group' }} runs-on: ubuntu-latest container: franziskus/libcrux-c:latest defaults: @@ -59,7 +57,6 @@ jobs: diff: needs: [extract] - if: ${{ github.event_name != 'merge_group' }} runs-on: ubuntu-latest defaults: run: @@ -80,7 +77,6 @@ jobs: diff-header-only: needs: [extract-header-only] - if: ${{ github.event_name != 'merge_group' }} runs-on: ubuntu-latest defaults: run: @@ -101,7 +97,6 @@ jobs: build: needs: [extract] - if: ${{ github.event_name != 'merge_group' }} strategy: fail-fast: false matrix: @@ -141,7 +136,6 @@ jobs: build-header-only: needs: [extract-header-only] - if: ${{ github.event_name != 'merge_group' }} strategy: fail-fast: false matrix: @@ -174,50 +168,3 @@ jobs: - name: 🏃🏻‍♀️ Test run: ./build/ml_kem_test if: ${{ matrix.os != 'windows-latest' }} - - benchmark: - if: ${{ github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' }} - strategy: - fail-fast: false - matrix: - os: - - macos-latest - - ubuntu-latest - - windows-latest - - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - working-directory: libcrux-ml-kem/c - - steps: - - uses: actions/checkout@v4 - - - name: 🔨 Build Release - run: | - cmake -B build -DCMAKE_BUILD_TYPE=Release - cmake --build build --config Release - - # FIXME: Benchmarks on Windows CI are not working right now. - # - name: 🏃🏻‍♀️ Benchmark - # run: ./build/Release/ml_kem_bench - # if: ${{ matrix.os == 'windows-latest' }} - - - name: 🏃🏻‍♀️ Benchmark (c) - run: ./build/ml_kem_bench - if: ${{ matrix.os != 'windows-latest' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }} - - - name: 🏃🏻‍♀️ Benchmark (cg) - working-directory: libcrux-ml-kem/cg - run: ./build/ml_kem_bench - if: ${{ matrix.os != 'windows-latest' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }} - - mq_status: - if: | - always() && - (github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group') - needs: [benchmark] - uses: cryspen/actions/.github/workflows/merge-queue-status.yml@jonas/merge-queue-status - with: - needs_json: "${{toJSON(needs)}}" diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5fab6fabd..0438fb798 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -6,6 +6,7 @@ on: pull_request: branches: ["main", "dev"] workflow_dispatch: + merge_group: env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/ecdh.yml b/.github/workflows/ecdh.yml index feee99cfa..438386e60 100644 --- a/.github/workflows/ecdh.yml +++ b/.github/workflows/ecdh.yml @@ -6,6 +6,7 @@ on: pull_request: branches: ["main", "dev", "*"] workflow_dispatch: + merge_group: env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/hax.yml b/.github/workflows/hax.yml index bac3854d5..d324bff1c 100644 --- a/.github/workflows/hax.yml +++ b/.github/workflows/hax.yml @@ -11,6 +11,7 @@ on: - cron: "0 0 * * *" workflow_dispatch: + merge_group: env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/kem.yml b/.github/workflows/kem.yml index ca3c7f23f..d4ada3cdf 100644 --- a/.github/workflows/kem.yml +++ b/.github/workflows/kem.yml @@ -6,6 +6,7 @@ on: pull_request: branches: ["main", "dev", "*"] workflow_dispatch: + merge_group: env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/mldsa.yml b/.github/workflows/mldsa.yml index 8c46feafd..a05d1e6c3 100644 --- a/.github/workflows/mldsa.yml +++ b/.github/workflows/mldsa.yml @@ -6,6 +6,7 @@ on: pull_request: branches: ["main", "dev", "*"] workflow_dispatch: + merge_group: env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/mlkem-bench.yml b/.github/workflows/mlkem-bench.yml new file mode 100644 index 000000000..b5c825c09 --- /dev/null +++ b/.github/workflows/mlkem-bench.yml @@ -0,0 +1,103 @@ +name: Benchmark ML-KEM + +on: + workflow_dispatch: + merge_group: + +env: + CARGO_TERM_COLOR: always + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmark: + strategy: + fail-fast: true + matrix: + bits: [32, 64] + os: + - macos-13 + - macos-latest + - ubuntu-latest + - windows-latest + exclude: + # There's no such thing as 32-bit macOS + - bits: 32 + os: "macos-latest" + - bits: 32 + os: "macos-13" + + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + working-directory: libcrux-ml-kem + + steps: + - uses: actions/checkout@v4 + + - name: Update dependencies + run: cargo update + + - run: echo "RUST_TARGET_FLAG=" > $GITHUB_ENV + if: ${{ matrix.bits == 64 }} + + - name: 🛠️ Setup Ubuntu x86 + if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} + run: | + rustup target add i686-unknown-linux-gnu + sudo apt-get update + sudo apt-get install -y gcc-multilib g++-multilib + + - name: 🛠️ Setup Ubuntu x64 + if: ${{ matrix.bits == 64 && matrix.os == 'ubuntu-latest' }} + run: | + rustup target add aarch64-unknown-linux-gnu + + - name: 🛠️ Setup macOS + if: ${{ matrix.os == 'macos-latest' }} + run: | + rustup target add aarch64-apple-darwin + + # Set up 32 bit systems + + - name: 🛠️ Config Windows x86 + run: echo "RUST_TARGET_FLAG=--target=i686-pc-windows-msvc" > $GITHUB_ENV + if: ${{ matrix.bits == 32 && matrix.os == 'windows-latest' }} + + - name: 🛠️ Config Linux x86 + run: | + echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu" > $GITHUB_ENV + if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} + + # - name: 🔨 Build + # run: cargo build --benches + + # - name: ⬆ Upload build + # uses: ./.github/actions/upload_artifacts + # with: + # name: benchmarks_${{ matrix.os }}_${{ matrix.bits }} + + # Benchmarks ... + + - name: 🏃🏻‍♀️ Benchmarks + run: cargo bench --verbose $RUST_TARGET_FLAG -- --output-format bencher | tee bench.txt + + - name: 🏃🏻‍♀️ Benchmarks Portable + run: | + cargo clean + LIBCRUX_DISABLE_SIMD128=1 LIBCRUX_DISABLE_SIMD256=1 cargo bench --verbose $RUST_TARGET_FLAG -- --output-format bencher | sed 's/^test \(.*\) \.\.\. bench/test portable \1 ... bench/' | tee -a bench.txt + - name: Store benchmarks + uses: benchmark-action/github-action-benchmark@v1 + with: + name: ML-KEM Benchmark + tool: 'cargo' + output-file-path: libcrux-ml-kem/bench.txt + benchmark-data-dir-path: dev/bench/mlkem + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + + + diff --git a/.github/workflows/mlkem.yml b/.github/workflows/mlkem.yml index 6e7c59209..575339c5d 100644 --- a/.github/workflows/mlkem.yml +++ b/.github/workflows/mlkem.yml @@ -17,7 +17,6 @@ concurrency: jobs: build: - if: ${{ github.event_name != 'merge_group' }} strategy: fail-fast: false matrix: @@ -174,96 +173,3 @@ jobs: run: | cargo clean cargo hack test --each-feature $EXCLUDE_FEATURES --verbose $RUST_TARGET_FLAG - - benchmarks: - if: ${{ github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' }} - strategy: - fail-fast: true - matrix: - bits: [32, 64] - os: - - macos-13 - - macos-latest - - ubuntu-latest - - windows-latest - exclude: - # There's no such thing as 32-bit macOS - - bits: 32 - os: "macos-latest" - - bits: 32 - os: "macos-13" - - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - working-directory: libcrux-ml-kem - - steps: - - uses: actions/checkout@v4 - - - name: Update dependencies - run: cargo update - - - run: echo "RUST_TARGET_FLAG=" > $GITHUB_ENV - if: ${{ matrix.bits == 64 }} - - - name: 🛠️ Setup Ubuntu x86 - if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} - run: | - rustup target add i686-unknown-linux-gnu - sudo apt-get update - sudo apt-get install -y gcc-multilib g++-multilib - - - name: 🛠️ Setup Ubuntu x64 - if: ${{ matrix.bits == 64 && matrix.os == 'ubuntu-latest' }} - run: | - rustup target add aarch64-unknown-linux-gnu - - - name: 🛠️ Setup macOS - if: ${{ matrix.os == 'macos-latest' }} - run: | - rustup target add aarch64-apple-darwin - - # Set up 32 bit systems - - - name: 🛠️ Config Windows x86 - run: echo "RUST_TARGET_FLAG=--target=i686-pc-windows-msvc" > $GITHUB_ENV - if: ${{ matrix.bits == 32 && matrix.os == 'windows-latest' }} - - - name: 🛠️ Config Linux x86 - run: | - echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu" > $GITHUB_ENV - if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} - - # - name: 🔨 Build - # run: cargo build --benches - - # - name: ⬆ Upload build - # uses: ./.github/actions/upload_artifacts - # with: - # name: benchmarks_${{ matrix.os }}_${{ matrix.bits }} - - # Benchmarks ... - - - name: 🏃🏻‍♀️ Benchmarks - run: cargo bench --verbose $RUST_TARGET_FLAG - - - name: 🏃🏻‍♀️ Benchmarks Portable - run: | - cargo clean - LIBCRUX_DISABLE_SIMD128=1 LIBCRUX_DISABLE_SIMD256=1 cargo bench --verbose $RUST_TARGET_FLAG - - mq_status: - if: | - always() && - github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' - needs: [benchmarks] - runs-on: ubuntu-latest - steps: - - name: Successful - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - name: Failing - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index aa893300c..f852b95a3 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -4,6 +4,7 @@ on: push: branches: [main, dev] pull_request: + merge_group: jobs: nix: diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index 3415ee2f3..074dea37d 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -6,6 +6,7 @@ on: pull_request: branches: ["main", "dev", "*"] workflow_dispatch: + merge_group: env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/rust-bench.yml b/.github/workflows/rust-bench.yml new file mode 100644 index 000000000..20644290d --- /dev/null +++ b/.github/workflows/rust-bench.yml @@ -0,0 +1,98 @@ +name: Benchmark + +on: + workflow_dispatch: + merge_group: + +env: + CARGO_TERM_COLOR: always + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmark: + strategy: + fail-fast: true + matrix: + bits: [32, 64] + os: + - macos-latest + - ubuntu-latest + - windows-latest + exclude: + # There's no such thing as 32-bit macOS + - bits: 32 + os: "macos-latest" + # FIXME: Linking isn't working here yet for hacl #42 + - bits: 32 + os: "windows-latest" + + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v4 + + - run: echo "RUST_TARGET_FLAG=" > $GITHUB_ENV + if: ${{ matrix.bits == 64 }} + + - name: ⚙️ Setup Ubuntu x86 + if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} + run: | + rustup target add i686-unknown-linux-gnu + sudo apt-get update + sudo apt-get install -y gcc-multilib g++-multilib + + - name: ⚙️ Setup Ubuntu x64 + if: ${{ matrix.bits == 64 && matrix.os == 'ubuntu-latest' }} + run: | + rustup target add aarch64-unknown-linux-gnu + + - name: ⚙️ Setup macOS + if: ${{ matrix.os == 'macos-latest' }} + run: | + rustup target add aarch64-apple-darwin + + # Set up 32 bit systems + + - name: 🛠️ Config Windows x86 + run: echo "RUST_TARGET_FLAG=--target=i686-pc-windows-msvc" > $GITHUB_ENV + if: ${{ matrix.bits == 32 && matrix.os == 'windows-latest' }} + + - name: 🛠️ Config Linux x86 + run: | + echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu" > $GITHUB_ENV + if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} + + # Set up windows + + - name: ⚙️ Setup Windows x86 + if: ${{ matrix.bits == 32 && matrix.os == 'windows-latest' }} + shell: pwsh + run: | + echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append + vcpkg install openssl:x86-windows-static-md + + - name: ⚙️ Setup Windows x64 + if: ${{ matrix.bits == 64 && matrix.os == 'windows-latest' }} + shell: pwsh + run: | + echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append + vcpkg install openssl:x64-windows-static-md + + # Benchmarks ... + + - name: 🏃🏻‍♀️ Benchmarks + run: cargo bench --verbose $RUST_TARGET_FLAG -p benchmarks -- --output-format bencher | tee bench.txt + - name: Store Benchmarks + uses: benchmark-action/github-action-benchmark@v1 + with: + name: ML-KEM Benchmark + tool: 'cargo' + output-file-path: bench.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7b4324a5c..e519051ea 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -17,7 +17,6 @@ concurrency: jobs: build: - if: ${{ github.event_name != 'merge_group' }} strategy: fail-fast: false matrix: @@ -141,90 +140,4 @@ jobs: - name: 🏃🏻‍♀️ Test run: CC=emcc AR=emar wasm-pack test --node --features wasm - benchmarks: - if: ${{ github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' }} - strategy: - fail-fast: true - matrix: - bits: [32, 64] - os: - - macos-latest - - ubuntu-latest - - windows-latest - exclude: - # There's no such thing as 32-bit macOS - - bits: 32 - os: "macos-latest" - # FIXME: Linking isn't working here yet for hacl #42 - - bits: 32 - os: "windows-latest" - - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - - steps: - - uses: actions/checkout@v4 - - - run: echo "RUST_TARGET_FLAG=" > $GITHUB_ENV - if: ${{ matrix.bits == 64 }} - - - name: ⚙️ Setup Ubuntu x86 - if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} - run: | - rustup target add i686-unknown-linux-gnu - sudo apt-get update - sudo apt-get install -y gcc-multilib g++-multilib - - - name: ⚙️ Setup Ubuntu x64 - if: ${{ matrix.bits == 64 && matrix.os == 'ubuntu-latest' }} - run: | - rustup target add aarch64-unknown-linux-gnu - - - name: ⚙️ Setup macOS - if: ${{ matrix.os == 'macos-latest' }} - run: | - rustup target add aarch64-apple-darwin - - # Set up 32 bit systems - - - name: 🛠️ Config Windows x86 - run: echo "RUST_TARGET_FLAG=--target=i686-pc-windows-msvc" > $GITHUB_ENV - if: ${{ matrix.bits == 32 && matrix.os == 'windows-latest' }} - - - name: 🛠️ Config Linux x86 - run: | - echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu" > $GITHUB_ENV - if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} - - # Set up windows - - - name: ⚙️ Setup Windows x86 - if: ${{ matrix.bits == 32 && matrix.os == 'windows-latest' }} - shell: pwsh - run: | - echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append - vcpkg install openssl:x86-windows-static-md - - - name: ⚙️ Setup Windows x64 - if: ${{ matrix.bits == 64 && matrix.os == 'windows-latest' }} - shell: pwsh - run: | - echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append - vcpkg install openssl:x64-windows-static-md - - # Benchmarks ... - - - name: 🏃🏻‍♀️ Benchmarks - run: cargo bench --verbose $RUST_TARGET_FLAG -p benchmarks - - mq_status: - if: | - always() && - (github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group') - needs: [benchmarks] - uses: cryspen/actions/.github/workflows/merge-queue-status.yml@jonas/merge-queue-status - with: - needs_json: "${{toJSON(needs)}}" diff --git a/.github/workflows/skip-benches-in-prs.yml b/.github/workflows/skip-benches-in-prs.yml new file mode 100644 index 000000000..fd259b985 --- /dev/null +++ b/.github/workflows/skip-benches-in-prs.yml @@ -0,0 +1,14 @@ +name: Skip Benchmarks in PRs + +on: [ pull_request ] +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - run: true + benchmarks: + runs-on: ubuntu-latest + steps: + - run: true + + diff --git a/.github/workflows/specs.yml b/.github/workflows/specs.yml index 83a79e802..383d45cdf 100644 --- a/.github/workflows/specs.yml +++ b/.github/workflows/specs.yml @@ -5,6 +5,7 @@ on: branches: [ "main", "dev" ] pull_request: branches: [ "main", "dev" ] + merge_group: env: CARGO_TERM_COLOR: always diff --git a/Cargo.toml b/Cargo.toml index b2e2765e3..869a771a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ exclude = [ [lib] crate-type = ["staticlib", "cdylib", "lib"] +bench = false # so libtest doesn't eat the arguments for criterion [build-dependencies] libcrux-platform = { version = "=0.0.2-alpha.3", path = "sys/platform" } diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 3e9f3b781..21a2c3dc3 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -9,6 +9,9 @@ repository.workspace = true readme.workspace = true publish = false +[lib] +bench = false # so libtest doesn't eat the arguments for criterion + [dependencies] [dev-dependencies] diff --git a/libcrux-ml-kem/Cargo.toml b/libcrux-ml-kem/Cargo.toml index c967a416b..1deaa591c 100644 --- a/libcrux-ml-kem/Cargo.toml +++ b/libcrux-ml-kem/Cargo.toml @@ -18,6 +18,9 @@ exclude = [ "/hax.py", ] +[lib] +bench = false # so libtest doesn't eat the arguments to criterion + [dependencies] rand_core = { version = "0.6" } libcrux-platform = { version = "0.0.2-alpha.3", path = "../sys/platform" }