From 7794e0a667454d7c7efdac531e2895772cec9d01 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:12:05 +0100 Subject: [PATCH] [WIP] Add CI benchmarking --- .github/workflows/bench.yml | 65 ++++++++++ benchmarker/Cargo.toml | 13 ++ benchmarker/src/main.rs | 238 ++++++++++++++++++++++++++++++++++++ zlib_benchmarks.json | 10 ++ 4 files changed, 326 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 benchmarker/Cargo.toml create mode 100644 benchmarker/src/main.rs create mode 100644 zlib_benchmarks.json diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..1d63c64 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,65 @@ +name: Benchmark + +permissions: + contents: read + +on: + push: + workflow_dispatch: + inputs: + ref: + description: "The commit or branch to benchmark" + required: true + type: string + merge_group: + branches: + - main + +jobs: + bench: + name: "Benchmark ${{ matrix.name }}" + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + matrix: + include: + - name: linux-x86 + os: [benchmark, X64] + target: "x86_64-unknown-linux-gnu" + - name: macos-arm64 + os: [benchmark, ARM64, macOS] + target: "aarch64-apple-darwin" + steps: + - name: Checkout sources + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + persist-credentials: false + ref: "${{inputs.ref}}" + - name: cargo build + run: | + . "$HOME/.cargo/env" + cargo build --target ${{matrix.target}} -p test-libz-rs-sys --release --examples + cd benchmarker && cargo build --release + - name: Benchmark + run: | + cp target/${{matrix.target}}/release/examples/blogpost-compress . + benchmarker/target/release/benchmarker "./blogpost-compress 9 rs silesia-small.tar" "./blogpost-compress 9 ng silesia-small.tar" > bench_results.json + - name: Upload benchmark results to artifacts + uses: actions/upload-artifact@v4 + with: + name: "benchmark-results-${{matrix.name}}" + path: bench_results.json + - name: Upload benchmark results to bench repo + if: github.event_name == 'push' + run: | + mkdir -p ~/.ssh + echo "${{ secrets.BENCH_DATA_DEPLOY_KEY }}" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + chmod 700 ~/.ssh + + git clone --depth 1 git@github.com:trifectatechfoundation/zlib-rs-bench.git + cat bench_results.json >> zlib-rs-bench/metrics-${{matrix.name}}.json + cd zlib-rs-bench + git add . + git -c user.name="Perf bot" -c user.email=perf-bot@trifectatech.org commit --message 📈 + git push origin main diff --git a/benchmarker/Cargo.toml b/benchmarker/Cargo.toml new file mode 100644 index 0000000..9181812 --- /dev/null +++ b/benchmarker/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "benchmarker" +version = "0.0.0" +edition = "2021" +license = "Apache-2.0 OR MIT" +publish = false + +[workspace] + +[dependencies] +libc = "0.2.168" +serde = { version = "1.0.216", features = ["derive"] } +serde_json = "1.0.133" diff --git a/benchmarker/src/main.rs b/benchmarker/src/main.rs new file mode 100644 index 0000000..806899a --- /dev/null +++ b/benchmarker/src/main.rs @@ -0,0 +1,238 @@ +use std::collections::BTreeMap; +use std::process::Command; +use std::time::SystemTime; +use std::{env, fs}; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +struct Commands(BTreeMap>); + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +struct PerfData { + event: String, + counter_value: String, + unit: String, +} + +#[derive(Debug, Serialize)] +struct BenchData { + // What and when are we benchmarking + commit_hash: String, + timestamp: SystemTime, + + // Where are we benchmarking it on + arch: String, + os: String, + runner: String, + cpu_model: String, + + // The actual results for benchmarks + bench_groups: Vec, +} + +#[derive(Debug, Serialize)] +struct BenchGroup { + group_name: String, + results: Vec, +} + +#[derive(Debug, Serialize)] +struct SingleBench { + cmd: Vec, + counters: BTreeMap, +} + +#[derive(Debug, Serialize)] +struct BenchCounter { + value: String, + unit: String, +} + +impl BenchData { + fn render_markdown(&self) -> String { + use std::fmt::Write; + + let mut md = String::new(); + + writeln!( + md, + "## [`{commit}`](https://github.com/trifectatechfoundation/zlib-rs/commit/{commit}) \ + (on {cpu})", + commit = self.commit_hash, + cpu = self.cpu_model + ) + .unwrap(); + writeln!(md, "").unwrap(); + + for bench_group in &self.bench_groups { + writeln!(md, "### {}", bench_group.group_name).unwrap(); + writeln!(md, "").unwrap(); + for bench in &bench_group.results { + writeln!(md, "#### `{}`", bench.cmd.join(" ")).unwrap(); + writeln!(md, "").unwrap(); + writeln!(md, "|metric|value|").unwrap(); + writeln!(md, "|------|-----|").unwrap(); + for (name, data) in bench.counters.iter() { + writeln!(md, "|{name}|`{}` {}|", data.value, data.unit).unwrap(); + } + writeln!(md, "").unwrap(); + } + } + + md + } +} + +fn get_cpu_model() -> String { + if !cfg!(target_os = "linux") { + return "".to_owned(); + } + + serde_json::from_slice::( + &Command::new("lscpu").arg("-J").output().unwrap().stdout, + ) + .unwrap()["lscpu"] + .as_array() + .unwrap() + .iter() + .find(|entry| entry["field"] == "Model name:") + .unwrap()["data"] + .as_str() + .unwrap() + .to_owned() +} + +fn bench_single_cmd(cmd: Vec) -> SingleBench { + if cfg!(target_os = "linux") { + bench_single_cmd_perf(cmd) + } else { + bench_single_cmd_getrusage(cmd) + } +} + +fn bench_single_cmd_perf(cmd: Vec) -> SingleBench { + let mut perf_stat_cmd = Command::new("perf"); + perf_stat_cmd + .arg("stat") + .arg("-j") + .arg("-e") + .arg("task-clock,cycles,instructions") + .arg("--repeat") + .arg("1") // FIXME 20 + .arg("--"); + perf_stat_cmd.args(&cmd); + + let output = perf_stat_cmd.output().unwrap(); + assert!( + output.status.success(), + "`{:?}` failed with {:?}:=== stdout ===\n{}\n\n=== stderr ===\n{}", + perf_stat_cmd, + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + + let counters = String::from_utf8(output.stderr) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .map(|counter| { + ( + counter.event, + BenchCounter { + value: counter.counter_value, + unit: counter.unit, + }, + ) + }) + .collect::>(); + + SingleBench { cmd, counters } +} + +fn bench_single_cmd_getrusage(cmd: Vec) -> SingleBench { + use std::mem; + use std::time::Duration; + + fn get_cpu_times() -> Duration { + use libc::{getrusage, rusage, RUSAGE_CHILDREN}; + + let result: rusage = unsafe { + let mut buf = mem::zeroed(); + let success = getrusage(RUSAGE_CHILDREN, &mut buf); + assert_eq!(0, success); + buf + }; + + Duration::new( + result.ru_utime.tv_sec as _, + (result.ru_utime.tv_usec * 1000) as _, + ) + } + + let mut bench_cmd = Command::new(cmd.get(0).unwrap()); + bench_cmd.args(&cmd[1..]); + + let start_cpu = get_cpu_times(); + let output = bench_cmd.output().unwrap(); + let user_time = get_cpu_times() - start_cpu; + assert!( + output.status.success(), + "`{:?}` failed with {:?}:\n=== stdout ===\n{}\n\n=== stderr ===\n{}", + bench_cmd, + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + + SingleBench { + cmd, + counters: BTreeMap::from_iter([( + "user-time".to_owned(), + BenchCounter { + value: format!("{:.06}", user_time.as_secs_f64() * 1000.0), + unit: "msec".to_owned(), + }, + )]), + } +} + +fn main() { + let mut bench_data = BenchData { + commit_hash: env::var("GITHUB_SHA").unwrap_or_default(), + timestamp: SystemTime::now(), + + arch: env::var("RUNNER_ARCH").unwrap_or_default(), + os: env::var("RUNNER_OS").unwrap_or_default(), + runner: env::var("RUNNER_NAME").unwrap_or_else(|_| "".to_owned()), + cpu_model: get_cpu_model(), + + bench_groups: vec![], + }; + + let commands: Commands = + serde_json::from_slice(&fs::read(env::args().nth(1).unwrap()).unwrap()).unwrap(); + + for (group_name, benches) in commands.0 { + let mut group = BenchGroup { + group_name, + results: vec![], + }; + for cmd in benches { + group.results.push(bench_single_cmd( + cmd.split(" ").map(|arg| arg.to_owned()).collect(), + )); + } + bench_data.bench_groups.push(group); + } + + println!("{}", serde_json::to_string(&bench_data).unwrap()); + + eprintln!("{}", bench_data.render_markdown()); + if let Ok(path) = env::var("GITHUB_STEP_SUMMARY") { + fs::write(path, bench_data.render_markdown()).unwrap(); + } +} diff --git a/zlib_benchmarks.json b/zlib_benchmarks.json new file mode 100644 index 0000000..badc7c6 --- /dev/null +++ b/zlib_benchmarks.json @@ -0,0 +1,10 @@ +{ + "blogpost-compress-rs": [ + "./blogpost-compress 1 rs silesia-small.tar", + "./blogpost-compress 9 rs silesia-small.tar" + ], + "blogpost-compress-ng": [ + "./blogpost-compress 1 ng silesia-small.tar", + "./blogpost-compress 9 ng silesia-small.tar" + ] +}