diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000000..8a9b05b86b --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,67 @@ +name: 'Benchmark' + +on: + push: + paths: + - '**.f90' + - '**.fpp' + - '**.py' + - '**.yml' + - 'mfc.sh' + - 'CMakeLists.txt' + - 'requirements.txt' + + pull_request: + +jobs: + self: + name: Georgia Tech | Phoenix (NVHPC) + if: github.repository == 'MFlowCode/MFC' + strategy: + matrix: + device: ['cpu', 'gpu'] + runs-on: + group: phoenix + labels: self-hosted + steps: + - name: Clone - PR + uses: actions/checkout@v3 + + - name: Bench - PR + run: | + bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }} + mv bench-${{ matrix.device }}.out ~/bench-${{ matrix.device }}-pr.out + mv bench-${{ matrix.device }}.yaml ~/bench-${{ matrix.device }}-pr.yaml + + - name: Clone - Master + uses: actions/checkout@v3 + with: + repository: henryleberre/MFC + ref: master + + - name: Bench - Master + run: | + bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }} + mv bench-${{ matrix.device }}.out ~/bench-${{ matrix.device }}-master.out + mv bench-${{ matrix.device }}.yaml ~/bench-${{ matrix.device }}-master.yaml + + - name: Post Comment + run: | + PR_ID=$(echo "${{ github.event_name }}" | jq --raw-output .pull_request.number) + PR_COMMENT=`python3 .github/workflows/phoenix/compare.py ~/bench-${{ matrix.device }}-master.yaml ~/bench-${{ matrix.device }}-pr.yaml` + + echo "Posting comment on PR #$PR_ID: $PR_COMMENT" + + curl -X POST \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/issues/$PR_ID/comments" \ + -d "{\"body\":\"$PR_COMMENT\"}" + + - name: Archive Logs + uses: actions/upload-artifact@v3 + if: always() + with: + name: logs + path: | + ~/bench-${{ matrix.device }}-* diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4e3431ab2a..ac27cf6dc3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,7 +22,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Build & Publish thereto - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v3 with: file: toolchain/Dockerfile push: true diff --git a/.github/workflows/phoenix/bench.sh b/.github/workflows/phoenix/bench.sh new file mode 100644 index 0000000000..55c07dea22 --- /dev/null +++ b/.github/workflows/phoenix/bench.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +n_ranks=12 + +if [ "$job_device" == "gpu" ]; then + n_ranks=$(nvidia-smi -L | wc -l) # number of GPUs on node + gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1 + device_opts="--gpu -g $gpu_ids" +fi + +./mfc.sh bench "$job_slug.yaml" -j $(nproc) -b mpirun $device_opts -n $n_ranks diff --git a/.github/workflows/phoenix/compare.py b/.github/workflows/phoenix/compare.py new file mode 100644 index 0000000000..90be62996e --- /dev/null +++ b/.github/workflows/phoenix/compare.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import argparse + +import yaml + +parser = argparse.ArgumentParser() +parser.add_argument('master', metavar="MASTER", type=str) +parser.add_argument('pr', metavar="PR", type=str) + +args = parser.parse_args() + +def load_cases(filepath): + return { case["name"]: case for case in yaml.safe_load(open(filepath))["cases"] } + +master, pr = load_cases(args.master), load_cases(args.pr) + +master_keys = set(master.keys()) +pr_keys = set(pr.keys()) + +missing_cases = master_keys.symmetric_difference(pr_keys) + +if len(missing_cases) > 0: + print("**Warning:** The following cases are **missing** from master or this PR:\n") + + for case in missing_cases: + print(f" - {case}.") + + print("") + +speedups = {} + +for case in master_keys.intersection(pr_keys): + speedups[case] = { + "pre_proess": pr[case]["pre_process"] / master[case]["pre_process"], + "simulation": pr[case]["simulation"] / master[case]["simulation"], + } + +avg_speedup = sum([ speedups[case]["simulation"] for case in speedups ]) / len(speedups) + +print(f"""\ +**[Benchmark Results]** Compared to Master, this PR's `simulation` is on average **~{avg_speedup:0.2f}x faster**. + +| **Case** | **Master** | **PR** | **Speedup** | +| -------- | ---------- | ------ | ----------- |\ +""") + +for case in sorted(speedups.keys()): + speedup = speedups[case] + + print(f"| {case} | {master[case]['simulation']:0.2f}s | {pr[case]['simulation']:0.2f}s | {speedups[case]['simulation']:0.2f}x |") diff --git a/.github/workflows/phoenix/submit.sh b/.github/workflows/phoenix/submit.sh new file mode 100644 index 0000000000..4ba2c5d97b --- /dev/null +++ b/.github/workflows/phoenix/submit.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 [script.sh] [cpu|gpu]" +} + +if [ ! -z "$1" ]; then + sbatch_script_contents=`cat $1` +else + usage + exit 1 +fi + +sbatch_cpu_opts="\ +#SBATCH --ntasks-per-node=12 # Number of cores per node required +#SBATCH --mem-per-cpu=2G # Memory per core\ +" + +sbatch_gpu_opts="\ +#SBATCH -CV100-16GB +#SBATCH -G2\ +" + +if [ "$2" == "cpu" ]; then + sbatch_device_opts="$sbatch_cpu_opts" +elif [ "$2" == "gpu" ]; then + sbatch_device_opts="$sbatch_gpu_opts" +else + usage + exit 1 +fi + +job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2" + +sbatch <