From 1d3ee8355531f77ea86b795f63cdc4fb9a2dbc10 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:12:05 +0100 Subject: [PATCH] [WIP] Add CI benchmarking --- .github/workflows/bench.yml | 61 ++++++++++++++ benchmarker/Cargo.toml | 12 +++ benchmarker/src/main.rs | 155 ++++++++++++++++++++++++++++++++++++ 3 files changed, 228 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 benchmarker/Cargo.toml create mode 100644 benchmarker/src/main.rs diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..3b95aef --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,61 @@ +name: Benchmark + +permissions: + contents: read + +on: + push: + workflow_dispatch: + inputs: + ref: + description: "The commit or branch to benchmark" + required: true + type: string + merge_group: + branches: + - main + +jobs: + bench: + name: Benchmark + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + matrix: + include: + - rust: stable + os: [benchmark, X64] + target: "x86_64-unknown-linux-gnu" + steps: + - name: Checkout sources + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + persist-credentials: false + ref: "${{inputs.ref}}" + - name: cargo build + run: | + . "$HOME/.cargo/env" + cargo build --target ${{matrix.target}} -p test-libz-rs-sys --release --examples + cd benchmarker && cargo build --release + - name: Benchmark + run: | + benchmarker/target/release/benchmarker "target/${{matrix.target}}/release/examples/blogpost-compress 9 rs silesia-small.tar" "target/${{matrix.target}}/release/examples/blogpost-compress 9 ng silesia-small.tar" > bench_results.json + - name: Upload benchmark results to artifacts + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: bench_results.json + - name: Upload benchmark results to bench repo + if: github.event_name == "push" + run: | + mkdir -p ~/.ssh + echo "${{ secrets.BENCH_DATA_DEPLOY_KEY }}" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + chmod 700 ~/.ssh + + git clone --depth 1 git@github.com:trifectatechfoundation/zlib-rs-bench.git + cat bench_results.json >> zlib-rs-bench/metrics.json + cd zlib-rs-bench + git add . + git -c user.name="Folkert wants a better name for this bot" -c user.email=perf-bot@trifectatech.org commit --message 📈 + git push origin main diff --git a/benchmarker/Cargo.toml b/benchmarker/Cargo.toml new file mode 100644 index 0000000..5c73f9a --- /dev/null +++ b/benchmarker/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "benchmarker" +version = "0.0.0" +edition = "2021" +license = "Apache-2.0 OR MIT" +publish = false + +[workspace] + +[dependencies] +serde = { version = "1.0.216", features = ["derive"] } +serde_json = "1.0.133" diff --git a/benchmarker/src/main.rs b/benchmarker/src/main.rs new file mode 100644 index 0000000..da1127f --- /dev/null +++ b/benchmarker/src/main.rs @@ -0,0 +1,155 @@ +use std::collections::BTreeMap; +use std::process::Command; +use std::time::SystemTime; +use std::{env, fs}; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +struct PerfData { + event: String, + counter_value: String, + unit: String, +} + +#[derive(Debug, Serialize)] +struct BenchData { + // What and when are we benchmarking + commit_hash: String, + timestamp: SystemTime, + + // Where are we benchmarking it on + arch: String, + os: String, + runner: String, + cpu_model: String, + + // The actual results for individual benchmarks + results: Vec, +} + +#[derive(Debug, Serialize)] +struct SingleBench { + cmd: Vec, + counters: BTreeMap, +} + +#[derive(Debug, Serialize)] +struct BenchCounter { + value: String, + unit: String, +} + +impl BenchData { + fn render_markdown(&self) -> String { + use std::fmt::Write; + + let mut md = String::new(); + + writeln!( + md, + "## [`{commit}`](https://github.com/trifectatechfoundation/zlib-rs/commit/{commit}) \ + (on {cpu})", + commit = self.commit_hash, + cpu = self.cpu_model + ) + .unwrap(); + writeln!(md, "").unwrap(); + + for bench in &self.results { + writeln!(md, "### `{}`", bench.cmd.join(" ")).unwrap(); + writeln!(md, "").unwrap(); + writeln!(md, "|metric|value|").unwrap(); + writeln!(md, "|------|-----|").unwrap(); + for (name, data) in bench.counters.iter() { + writeln!(md, "|{name}|`{}` {}|", data.value, data.unit).unwrap(); + } + writeln!(md, "").unwrap(); + } + + md + } +} + +fn get_cpu_model() -> String { + serde_json::from_slice::( + &Command::new("lscpu").arg("-J").output().unwrap().stdout, + ) + .unwrap()["lscpu"] + .as_array() + .unwrap() + .iter() + .find(|entry| entry["field"] == "Model name:") + .unwrap()["data"] + .as_str() + .unwrap() + .to_owned() +} + +fn bench_single_cmd(cmd: Vec) -> SingleBench { + let mut perf_stat_cmd = Command::new("perf"); + perf_stat_cmd + .arg("stat") + .arg("-j") + .arg("-e") + .arg("task-clock,cycles,instructions") + .arg("--repeat") + .arg("1") // FIXME 20 + .arg("--"); + perf_stat_cmd.args(&cmd); + + let output = perf_stat_cmd.output().unwrap(); + assert!( + output.status.success(), + "`{:?}` failed with {:?}:=== stdout ===\n{}\n\n=== stderr ===\n{}", + perf_stat_cmd, + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + + let counters = String::from_utf8(output.stderr) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .map(|counter| { + ( + counter.event, + BenchCounter { + value: counter.counter_value, + unit: counter.unit, + }, + ) + }) + .collect::>(); + + SingleBench { cmd, counters } +} + +fn main() { + let mut bench_data = BenchData { + commit_hash: env::var("GITHUB_SHA").unwrap_or_default(), + timestamp: SystemTime::now(), + + arch: env::var("RUNNER_ARCH").unwrap_or_default(), + os: env::var("RUNNER_OS").unwrap_or_default(), + runner: env::var("RUNNER_NAME").unwrap_or_else(|_| "".to_owned()), + cpu_model: get_cpu_model(), + + results: vec![], + }; + + for cmd in env::args().skip(1) { + bench_data.results.push(bench_single_cmd( + cmd.split(" ").map(|arg| arg.to_owned()).collect(), + )); + } + + println!("{}", serde_json::to_string(&bench_data).unwrap()); + + eprintln!("{}", bench_data.render_markdown()); + if let Ok(path) = env::var("GITHUB_STEP_SUMMARY") { + fs::write(path, bench_data.render_markdown()).unwrap(); + } +}