Skip to content

Commit

Permalink
Merge pull request #808 from jprendes/benchmark-memory
Browse files Browse the repository at this point in the history
Benchmark memory
  • Loading branch information
Mossaka authored Jan 22, 2025
2 parents afe5c9c + ddcaca3 commit 827fbd4
Showing 3 changed files with 123 additions and 4 deletions.
48 changes: 44 additions & 4 deletions .github/workflows/benchmarks.yml
Original file line number Diff line number Diff line change
@@ -17,8 +17,6 @@ jobs:

steps:
- uses: actions/checkout@v4
- name: Fetch submodules
run: git submodule update --init --recursive
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
rustflags: '' #Disable. By default this action sets environment variable is set to -D warnings. We manage this in the Makefile
@@ -54,14 +52,56 @@ jobs:
# Automatically push the benchmark result to gh-pages branch
# See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details
auto-push: ${{ github.event_name == 'schedule' }}

benchmark-mem:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
rustflags: '' #Disable. By default this action sets environment variable is set to -D warnings. We manage this in the Makefile
- name: Setup build environment
shell: bash
run: |
os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]')
./scripts/setup-$os.sh
- name: Build and load shims and wasi-demo-app
shell: bash
run: |
make OPT_PROFILE=release build install test-image load test-image/oci load/oci
- name: Run Benchmarks
shell: bash
run: |
set -euxo pipefail
for RUNTIME in wasmtime wasmedge wasmer wamr; do
sudo ./scripts/benchmark-mem.sh $RUNTIME > bench-mem-$RUNTIME.json
done
cat bench-mem-* | jq -s 'flatten(1)' > bench-mem.json
- name: Store benchmark result
uses: benchmark-action/github-action-benchmark@v1.20.4
with:
name: Criterion.rs Benchmark
tool: 'customSmallerIsBetter'
output-file-path: bench-mem.json
github-token: ${{ secrets.GITHUB_TOKEN }}
# my experimental local benchmarking seems to have a 20% margin of error.
# So I set the alert threshold to 130% of the previous benchmark result.
# If the current benchmark result is more than 130% of the previous benchmark result, it will fail.
alert-threshold: '130%'
fail-on-alert: ${{ github.event_name == 'schedule' }}
alert-comment-cc-users: '@runwasi-committers'
# Enable Job Summary
summary-always: true
# Automatically push the benchmark result to gh-pages branch
# See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details
auto-push: ${{ github.event_name == 'schedule' }}

benchmark-http:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- name: Fetch submodules
run: git submodule update --init --recursive
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
rustflags: '' #Disable. By default this action sets environment variable is set to -D warnings. We manage this in the Makefile
39 changes: 39 additions & 0 deletions crates/containerd-shim-wasm/src/sandbox/cli.rs
Original file line number Diff line number Diff line change
@@ -32,6 +32,42 @@ macro_rules! revision {
};
}

#[cfg(target_os = "linux")]
fn get_mem(pid: u32) -> (usize, usize) {
let mut rss = 0;
let mut total = 0;
for line in std::fs::read_to_string(format!("/proc/{pid}/status"))
.unwrap()
.lines()
{
let line = line.trim();
// VmPeak is the maximum total virtual memory used so far.
// VmHWM (high water mark) is the maximum resident set memory used so far.
// See: https://man7.org/linux/man-pages/man5/proc_pid_status.5.html
if let Some(rest) = line.strip_prefix("VmPeak:") {
if let Some(rest) = rest.strip_suffix("kB") {
total = rest.trim().parse().unwrap_or(0);
}
} else if let Some(rest) = line.strip_prefix("VmHWM:") {
if let Some(rest) = rest.strip_suffix("kB") {
rss = rest.trim().parse().unwrap_or(0);
}
}
}
(rss, total)
}

#[cfg(target_os = "linux")]
fn log_mem() {
let pid = std::process::id();
let (rss, tot) = get_mem(pid);
log::info!("Shim peak memory usage was: peak resident set {rss} kB, peak total {tot} kB");

let pid = zygote::Zygote::global().run(|_| std::process::id(), ());
let (rss, tot) = get_mem(pid);
log::info!("Zygote peak memory usage was: peak resident set {rss} kB, peak total {tot} kB");
}

/// Main entry point for the shim.
///
/// If the `opentelemetry` feature is enabled, this function will start the shim with OpenTelemetry tracing.
@@ -70,6 +106,9 @@ pub fn shim_main<'a, I>(
{
shim_main_inner::<I>(name, version, revision, shim_version, config);
}

#[cfg(target_os = "linux")]
log_mem();
}

#[cfg_attr(feature = "tracing", tracing::instrument(parent = tracing::Span::current(), skip_all, level = "Info"))]
40 changes: 40 additions & 0 deletions scripts/benchmark-mem.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/bin/bash

set -euxo pipefail

# Parse CLI arguments
RUNTIME=${1:-"wasmtime"}; shift || true
IMAGE=${1:-"ghcr.io/containerd/runwasi/wasi-demo-app:latest"}; shift || true

if [ $IMAGE == "ghcr.io/containerd/runwasi/wasi-demo-app:latest" ] && [ "$#" == "0" ]; then
set -- /wasi-demo-app.wasm echo 'hello'
fi

# Run the shim and collect logs
LOG_FILE=$(mktemp)
journalctl -fn 0 -u containerd | timeout -k 16s 15s grep -m 2 'peak memory usage was' > $LOG_FILE &
ctr run --null-io --rm --runtime=io.containerd.$RUNTIME.v1 "$IMAGE" testwasm "$@"

# Parse the logs
wait
SHIM_MEM=$(cat $LOG_FILE | grep 'Shim peak memory usage was' | sed -E 's/.*peak resident set ([0-9]+) kB.*/\1/')
ZYGOTE_MEM=$(cat $LOG_FILE | grep 'Zygote peak memory usage was' | sed -E 's/.*peak resident set ([0-9]+) kB.*/\1/')
rm $LOG_FILE

if [ "$SHIM_MEM" == "" ] || [ "$ZYGOTE_MEM" == "" ]; then
exit 1
fi

TOTAL_MEM=$(( $SHIM_MEM + $ZYGOTE_MEM ))

# Print the JSON for the benchmark report
cat <<EOF
[
{
"name": "$RUNTIME/memory-usage",
"unit": "kB",
"value": $TOTAL_MEM,
"extra": "shim: $SHIM_MEM kB\nzygote: $ZYGOTE_MEM kB"
}
]
EOF

0 comments on commit 827fbd4

Please sign in to comment.