Skip to content

Blocks Running Workflow #2730

Blocks Running Workflow

Blocks Running Workflow #2730

Workflow file for this run

name: Bench
on:
pull_request:
branches: [main]
merge_group:
types: [checks_requested]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
jobs:
bench-hyperfine:
name: Hyperfine
runs-on: ubuntu-24.04
env:
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/
TABLEGEN_190_PREFIX: /usr/lib/llvm-19/
LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/
steps:
- uses: actions/checkout@v4
- name: check and free hdd space left
run: |
echo "Listing 20 largest packages"
dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20
df -h
sudo apt-get update
sudo apt-get remove -y '^llvm-.*'
sudo apt-get remove -y 'php.*'
sudo apt-get remove -y '^dotnet-.*'
sudo apt-get remove -y '^temurin-.*'
sudo apt-get remove -y azure-cli microsoft-edge-stable google-chrome-stable firefox mono-devel
sudo apt-get autoremove -y
sudo apt-get clean
df -h
echo "Removing large directories"
# deleting 15GB
sudo rm -rf /usr/share/dotnet/
sudo rm -rf /usr/local/lib/android
df -h
- uses: dtolnay/[email protected]
with:
components: clippy
- uses: Swatinem/rust-cache@v2
- name: add llvm deb repository
uses: myci-actions/add-deb-repo@11
with:
repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main
repo-name: llvm-repo
keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key
- name: Install LLVM
run: sudo apt-get install llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools
- name: Install hyperfine
uses: taiki-e/install-action@v2
with:
tool: [email protected]
- name: Install deps
run: make deps
- name: Build project
run: make build
- name: Build runtime subproject
run: make runtime-ci
- name: Run benchmarks
run: ./scripts/bench-hyperfine.sh programs/benches/*.cairo
- name: Create markdown file
run: bash .github/scripts/merge-benches.sh
- name: Find Bench Comment
continue-on-error: true
uses: peter-evans/find-comment@v3
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
body-includes: Benchmarking
- name: Create or update bench comment
continue-on-error: true
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body-path: bench-hyperfine.md
edit-mode: replace
build-binaries:
env:
MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/
LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/
TABLEGEN_190_PREFIX: /usr/lib/llvm-19/
RUST_LOG: cairo_native=debug,cairo_native_test=debug
strategy:
matrix:
branch: [ base, head ]
name: Build cairo-native-run for ${{ matrix.branch }}
runs-on: ubuntu-24.04
steps:
- name: Cache binary
uses: actions/cache@v3
id: cache-binary
with:
path: bin/cairo-native-run-${{ matrix.branch }}
key: binary-${{ github.event.pull_request[matrix.branch].sha }}
- name: Cache runtime-library
uses: actions/cache@v3
id: cache-library
with:
path: lib/libcairo-native-runtime-${{ matrix.branch }}.a
key: library-${{ github.event.pull_request[matrix.branch].sha }}
- name: check and free hdd space left
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' }}
run: |
echo "Listing 20 largest packages"
dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20
df -h
sudo apt-get update
sudo apt-get remove -y '^llvm-.*'
sudo apt-get remove -y 'php.*'
sudo apt-get remove -y '^dotnet-.*'
sudo apt-get remove -y '^temurin-.*'
sudo apt-get remove -y azure-cli microsoft-edge-stable google-chrome-stable firefox mono-devel
sudo apt-get autoremove -y
sudo apt-get clean
df -h
echo "Removing large directories"
# deleting 15GB
sudo rm -rf /usr/share/dotnet/
sudo rm -rf /usr/local/lib/android
df -h
- name: Install Rust
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
uses: dtolnay/[email protected]
- name: add llvm deb repository
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
uses: myci-actions/add-deb-repo@11
with:
repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main
repo-name: llvm-repo
keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key
- name: Install LLVM
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
run: sudo apt-get update && sudo apt-get install llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools
- name: Checkout base commit
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request[matrix.branch].sha }}
- name: Fetch Rust cache
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
uses: Swatinem/rust-cache@v2
- name: Build binary
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
run: |
make build
mkdir lib bin
cp target/release/cairo-native-run bin/cairo-native-run-${{ matrix.branch }}
cp target/release/libcairo_native_runtime.a lib/libcairo-native-runtime-${{ matrix.branch }}.a
hyperfine-prs:
name: Bench PR (linux, amd64)
needs: [ build-binaries ]
runs-on: ubuntu-24.04
env:
PROGRAM: fib_2M
OUTPUT_DIR: bench-outputs
steps:
- uses: actions/checkout@v4
- name: Install Hyperfine
uses: taiki-e/install-action@v2
with:
tool: [email protected]
- name: Fetch base binary
uses: actions/cache/restore@v3
with:
path: bin/cairo-native-run-base
key: binary-${{ github.event.pull_request.base.sha }}
- name: Fetch HEAD binary
uses: actions/cache/restore@v3
with:
path: bin/cairo-native-run-head
key: binary-${{ github.event.pull_request.head.sha }}
- name: Fetch base runtime-library
uses: actions/cache/restore@v3
with:
path: lib/libcairo-native-runtime-base.a
key: library-${{ github.event.pull_request.base.sha }}
- name: Fetch head runtime-library
uses: actions/cache/restore@v3
with:
path: lib/libcairo-native-runtime-head.a
key: library-${{ github.event.pull_request.head.sha }}
- name: Build Dependencies
run: make deps
- name: Run Program
run: |
mkdir $OUTPUT_DIR
cp programs/benches/*.cairo $OUTPUT_DIR
for f in $OUTPUT_DIR/*.cairo; do
hyperfine \
--export-markdown "${f%.*}.md" \
--export-json "${f%.*}.json" \
--warmup 3 \
-n "head $(basename $f) (JIT)" "./bin/cairo-native-run-head --run-mode=jit -s $f --opt-level 3 --available-gas 19446744073709551615" \
-n "base $(basename $f) (JIT)" "./bin/cairo-native-run-base --run-mode=jit -s $f --opt-level 3 --available-gas 19446744073709551615" \
-n "head $(basename $f) (AOT)" "CAIRO_NATIVE_RUNTIME_LIBRARY=lib/libcairo-native-runtime-head.a ./bin/cairo-native-run-head --run-mode=aot -s $f --opt-level 3 --available-gas 19446744073709551615" \
-n "base $(basename $f) (AOT)" "CAIRO_NATIVE_RUNTIME_LIBRARY=lib/libcairo-native-runtime-base.a ./bin/cairo-native-run-base --run-mode=aot -s $f --opt-level 3 --available-gas 19446744073709551615" \
>> /dev/stderr
done
- name: Print tables
run: |
{
echo "${bold}Benchmark results Main vs HEAD.${normal}"
for f in $OUTPUT_DIR/*.md; do
echo
cat $f
done
} | tee -a comment_body.md
- name: Find Bench Comment
continue-on-error: true
uses: peter-evans/find-comment@v3
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
body-includes: Benchmark results Main vs HEAD
- name: Create or update bench comment
continue-on-error: true
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body-path: comment_body.md
edit-mode: replace