[WIP] Add CI benchmarking

.github/workflows/bench.yml

@@ -0,0 +1,49 @@
+name: checks
+
+permissions:
+  contents: read
+
+on:
+  push:
+  merge_group:
+    branches:
+      - main
+
+jobs:
+  bench:
+    name: Benchmark
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        include:
+          - rust: stable
+            os: [benchmark, X64]
+            target: "x86_64-unknown-linux-gnu"
+          # - rust: "stable"
+          #   os: macos-latest
+          #   target: "x86_64-apple-darwin"
+          # - rust: "stable"
+          #   os: macos-14
+          #   target: "aarch64-apple-darwin"
+          # - rust: stable-x86_64-gnu
+          #   os: windows-2022
+          #   target: "x86_64-pc-windows-gnu"
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        with:
+          persist-credentials: false
+      - name: cargo build
+        run: |
+          . "$HOME/.cargo/env"
+          cargo build --target ${{matrix.target}} -p test-libz-rs-sys --release --examples
+          cd benchmarker && cargo build --release
+      - name: Benchmark
+        run: |
+          benchmarker/target/release/benchmarker "target/${{matrix.target}}/release/examples/blogpost-compress 9 rs silesia-small.tar" "target/${{matrix.target}}/release/examples/blogpost-compress 9 ng silesia-small.tar" > bench_results.json
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: bench_results.json

benchmarker/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "benchmarker"
+version = "0.0.0"
+edition = "2021"
+license = "Apache-2.0 OR MIT"
+publish = false
+
+[workspace]
+
+[dependencies]
+serde = { version = "1.0.216", features = ["derive"] }
+serde_json = "1.0.133"

benchmarker/src/main.rs

@@ -0,0 +1,155 @@
+use std::collections::BTreeMap;
+use std::process::Command;
+use std::time::SystemTime;
+use std::{env, fs};
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+struct PerfData {
+    event: String,
+    counter_value: String,
+    unit: String,
+}
+
+#[derive(Debug, Serialize)]
+struct BenchData {
+    // What and when are we benchmarking
+    commit_hash: String,
+    timestamp: SystemTime,
+
+    // Where are we benchmarking it on
+    arch: String,
+    os: String,
+    runner: String,
+    cpu_model: String,
+
+    // The actual results for individual benchmarks
+    results: Vec<SingleBench>,
+}
+
+#[derive(Debug, Serialize)]
+struct SingleBench {
+    cmd: Vec<String>,
+    counters: BTreeMap<String, BenchCounter>,
+}
+
+#[derive(Debug, Serialize)]
+struct BenchCounter {
+    value: String,
+    unit: String,
+}
+
+impl BenchData {
+    fn render_markdown(&self) -> String {
+        use std::fmt::Write;
+
+        let mut md = String::new();
+
+        writeln!(
+            md,
+            "## [`{commit}`](https://github.com/trifectatechfoundation/zlib-rs/commit/{commit}) \
+                (on {cpu})",
+            commit = self.commit_hash,
+            cpu = self.cpu_model
+        )
+        .unwrap();
+        writeln!(md, "").unwrap();
+
+        for bench in &self.results {
+            writeln!(md, "### `{}`", bench.cmd.join(" ")).unwrap();
+            writeln!(md, "").unwrap();
+            writeln!(md, "|metric|value|").unwrap();
+            writeln!(md, "|------|-----|").unwrap();
+            for (name, data) in bench.counters.iter() {
+                writeln!(md, "|{name}|`{}` {}|", data.value, data.unit).unwrap();
+            }
+            writeln!(md, "").unwrap();
+        }
+
+        md
+    }
+}
+
+fn get_cpu_model() -> String {
+    serde_json::from_slice::<serde_json::Value>(
+        &Command::new("lscpu").arg("-J").output().unwrap().stdout,
+    )
+    .unwrap()["lscpu"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .find(|entry| entry["field"] == "Model name:")
+        .unwrap()["data"]
+        .as_str()
+        .unwrap()
+        .to_owned()
+}
+
+fn bench_single_cmd(cmd: Vec<String>) -> SingleBench {
+    let mut perf_stat_cmd = Command::new("perf");
+    perf_stat_cmd
+        .arg("stat")
+        .arg("-j")
+        .arg("-e")
+        .arg("task-clock,cycles,instructions")
+        .arg("--repeat")
+        .arg("20")
+        .arg("--");
+    perf_stat_cmd.args(&cmd);
+
+    let output = perf_stat_cmd.output().unwrap();
+    assert!(
+        output.status.success(),
+        "`{:?}` failed with {:?}:=== stdout ===\n{}\n\n=== stderr ===\n{}",
+        perf_stat_cmd,
+        output.status,
+        String::from_utf8_lossy(&output.stdout),
+        String::from_utf8_lossy(&output.stderr),
+    );
+
+    let counters = String::from_utf8(output.stderr)
+        .unwrap()
+        .lines()
+        .map(|line| serde_json::from_str::<PerfData>(line).unwrap())
+        .map(|counter| {
+            (
+                counter.event,
+                BenchCounter {
+                    value: counter.counter_value,
+                    unit: counter.unit,
+                },
+            )
+        })
+        .collect::<BTreeMap<_, _>>();
+
+    SingleBench { cmd, counters }
+}
+
+fn main() {
+    let mut bench_data = BenchData {
+        commit_hash: env::var("GITHUB_SHA").unwrap_or_default(),
+        timestamp: SystemTime::now(),
+
+        arch: env::var("RUNNER_ARCH").unwrap_or_default(),
+        os: env::var("RUNNER_OS").unwrap_or_default(),
+        runner: env::var("RUNNER_NAME").unwrap_or_else(|_| "<local bench>".to_owned()),
+        cpu_model: get_cpu_model(),
+
+        results: vec![],
+    };
+
+    for cmd in env::args().skip(1) {
+        bench_data.results.push(bench_single_cmd(
+            cmd.split(" ").map(|arg| arg.to_owned()).collect(),
+        ));
+    }
+
+    println!("{}", serde_json::to_string(&bench_data).unwrap());
+
+    eprintln!("{}", bench_data.render_markdown());
+    if let Ok(path) = env::var("GITHUB_STEP_SUMMARY") {
+        fs::write(path, bench_data.render_markdown()).unwrap();
+    }
+}