#49: GitHub CI Benchmarking

.github/workflows/bench.yml

@@ -0,0 +1,50 @@
+name: 'Benchmark'
+
+on:
+  pull_request:
+
+jobs:
+  self:
+    name: Georgia Tech | Phoenix (NVHPC)
+    if: github.repository == 'MFlowCode/MFC'
+    strategy:
+      matrix:
+        device: ['cpu', 'gpu']
+    runs-on:
+      group:  phoenix
+      labels: self-hosted
+    steps:
+      - name: Clone - PR
+        uses: actions/checkout@v3
+        with:
+          path: pr
+
+      - name: Clone - Master
+        uses: actions/checkout@v3
+        with:
+          repository: henryleberre/MFC
+          ref: master
+          path: master
+
+      - name: Bench (Master v. PR)
+        run: |
+          (cd pr     && bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}) &
+          (cd master && bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}) &
+          wait %1 && wait %2
+
+      - name: Generate Comment
+        run: |
+          COMMENT_MSG=`python3 .github/workflows/phoenix/compare.py master/bench-${{ matrix.device }}.yaml pr/bench-${{ matrix.device }}.yaml`
+          echo "COMMENT_MSG=\"$COMMENT_MSG\"" >> $GITHUB_ENV
+
+      - name: Post Comment
+        run:  echo "$COMMENT_MSG"
+
+      - name: Archive Logs
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: logs-${{ matrix.device }}
+          path: |
+            pr/bench-${{ matrix.device }}-*
+            master/bench-${{ matrix.device }}-*

.github/workflows/docker.yml

@@ -22,9 +22,8 @@ jobs:
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - name: Build & Publish thereto
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v3
       with:
         file: toolchain/Dockerfile
         push: true
         tags: ${{ secrets.DOCKER_USERNAME }}/mfc:latest
-

.github/workflows/docs.yml

@@ -11,7 +11,11 @@ jobs:
   docs:
     name:    Build & Publish
     runs-on: ubuntu-latest
-
+    if:      github.repository == 'MFlowCode/MFC'
+    concurrency:
+      group: docs-publish
+      cancel-in-progress: true
+
     steps:
     - uses: actions/checkout@v3
 
@@ -46,4 +50,4 @@ jobs:
         git -C ../www push
 
 # DOC_PUSH_URL should be of the format:
-# --> https://<username>:<token>@github.com/<username>/<repository>
+# --> https://<username>:<token>@github.com/<username>/<repository>

.github/workflows/lint.yml

@@ -0,0 +1,19 @@
+name: Lint
+
+on:
+  push:
+
+  pull_request:
+
+  workflow_dispatch:
+
+jobs:
+  docs:
+    name:    Lint
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Lint
+      run: ./mfc.sh lint

.github/workflows/phoenix/bench.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+n_ranks=$(nproc)
+
+if [ "$job_device" == "gpu" ]; then
+    n_ranks=$(nvidia-smi -L | wc -l)        # number of GPUs on node
+    gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1
+    device_opts="--gpu -g $gpu_ids"
+fi
+
+./mfc.sh bench -j $(nproc) -o "$job_slug.yaml" -- -b mpirun $device_opts -n $n_ranks

.github/workflows/phoenix/compare.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+import argparse
+
+import yaml
+
+parser = argparse.ArgumentParser()
+parser.add_argument('master', metavar="MASTER", type=str)
+parser.add_argument('pr',     metavar="PR",     type=str)
+
+args = parser.parse_args()
+
+def load_cases(filepath):
+    raw = yaml.safe_load(open(filepath))
+
+    cases = { case["slug"]: case for case in raw["cases"] }
+
+    for target in ["pre_process", "simulation"]:
+        for case in cases:
+            cases[case][target] = cases[case][target]["runtime"][target]
+
+    return cases
+
+master, pr = load_cases(args.master), load_cases(args.pr)
+
+master_keys = set(master.keys())
+pr_keys     = set(pr.keys())
+
+missing_cases = master_keys.symmetric_difference(pr_keys)
+
+if len(missing_cases) > 0:
+    print("**Warning:** The following cases are **missing** from master or this PR:\n")
+
+    for case in missing_cases:
+        print(f" - {case}.")
+
+    print("")
+
+speedups = {}
+
+for case in master_keys.intersection(pr_keys):
+    speedups[case] = {
+        "pre_proess": pr[case]["pre_process"] / master[case]["pre_process"],
+        "simulation": pr[case]["simulation"] / master[case]["simulation"],
+    }
+
+avg_speedup = sum([ speedups[case]["simulation"] for case in speedups ]) / len(speedups)
+
+print(f"""\
+**[Benchmark Results]** Compared to Master, this PR's `simulation` is on average **~{avg_speedup:0.2f}x faster**.
+
+| **Case** | **Master** | **PR** | **Speedup** |
+| -------- | ---------- | ------ | ----------- |\
+""")
+
+for case in sorted(speedups.keys()):
+    speedup = speedups[case]
+
+    print(f"| {case} | {master[case]['simulation']:0.2f}s | {pr[case]['simulation']:0.2f}s | {speedups[case]['simulation']:0.2f}x |")

.github/workflows/phoenix/submit.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -e
+
+usage() {
+    echo "Usage: $0 [script.sh] [cpu|gpu]"
+}
+
+if [ ! -z "$1" ]; then
+    sbatch_script_contents=`cat $1`
+else
+    usage
+    exit 1
+fi
+
+sbatch_cpu_opts="\
+#SBATCH -p cpu-small               # partition
+#SBATCH --ntasks-per-node=24       # Number of cores per node required
+#SBATCH --mem-per-cpu=2G           # Memory per core\
+"
+
+sbatch_gpu_opts="\
+#SBATCH -CV100-16GB
+#SBATCH -G2\
+"
+
+if [ "$2" == "cpu" ]; then
+    sbatch_device_opts="$sbatch_cpu_opts"
+elif [ "$2" == "gpu" ]; then
+    sbatch_device_opts="$sbatch_gpu_opts"
+else
+    usage
+    exit 1
+fi
+
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
+
+sbatch <<EOT
+#!/bin/bash
+#SBATCH -Jshb-$job_slug            # Job name
+#SBATCH --account=gts-sbryngelson3 # charge account
+#SBATCH -N1                        # Number of nodes required
+$sbatch_device_opts
+#SBATCH -t 04:00:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -q embers                  # QOS Name
+#SBATCH -o$job_slug.out            # Combined output and error messages file
+#SBATCH -W                         # Do not exit until the submitted job terminates.
+
+set -e
+set -x
+
+cd "\$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
+
+job_slug="$job_slug"
+job_device="$2"
+
+. ./mfc.sh load -c p -m $2
+
+$sbatch_script_contents
+
+EOT

.github/workflows/phoenix/test.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+build_opts=""
+if [ "$job_device" == "gpu" ]; then
+    build_opts="--gpu"
+fi
+
+./mfc.sh build -j $(nproc) $build_opts
+
+n_test_threads=$(nproc)
+
+if [ "$job_device" == "gpu" ]; then
+    gpu_count=$(nvidia-smi -L | wc -l)        # number of GPUs on node
+    gpu_ids=$(seq -s ' ' 0 $(($gpu_count-1))) # 0,1,2,...,gpu_count-1
+    device_opts="-g $gpu_ids"
+    n_test_threads=`expr $gpu_count \* 2`
+fi
+
+./mfc.sh test -a -j $n_test_threads $device_opts -- -b mpirun -f --bind-to none

.github/workflows/ci.yml → .github/workflows/test.yml

@@ -114,19 +114,12 @@ jobs:
       - name: Clone
         uses: actions/checkout@v3
 
-      - name: Build
-        run:  |
-          . ./mfc.sh load -c p -m gpu
-          ./mfc.sh build -j 2 $(if [ '${{ matrix.device }}' == 'gpu' ]; then echo '--gpu'; fi)
+      - name: Build & Test
+        run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }}
 
-      - name: Test
-        run: |
-          . ./mfc.sh load -c p -m gpu
-          mv misc/run-phoenix-release-${{ matrix.device }}.sh ./
-          sbatch run-phoenix-release-${{ matrix.device }}.sh
-
-      - name: Print
+      - name: Archive Logs
+        uses: actions/upload-artifact@v3
         if: always()
-        run: |
-          cat test.out
-
+        with:
+          name: logs
+          path: test-${{ matrix.device }}.out

README.md

@@ -9,7 +9,7 @@
     <img src="https://zenodo.org/badge/doi/10.1016/j.cpc.2020.107396.svg" />
   </a>
   <a href="https://github.com/MFlowCode/MFC/actions">
-    <img src="https://github.com/MFlowCode/MFC/actions/workflows/ci.yml/badge.svg" />
+    <img src="https://github.com/MFlowCode/MFC/actions/workflows/test.yml/badge.svg" />
   </a>
   <a href="https://lbesson.mit-license.org/">
     <img src="https://img.shields.io/badge/License-MIT-blue.svg" />

mfc.sh

@@ -439,10 +439,23 @@ if ! cmp "$(pwd)/toolchain/requirements.txt" "$(pwd)/build/requirements.txt" > /
 fi
 
 
-# Run the main.py bootstrap script
-python3 "$(pwd)/toolchain/mfc.py" "$@"
-code=$?
+if [ "$1" == "lint" ]; then
+    shift
+
+    if ! command -v pylint > /dev/null 2>&1; then
+        error "Couldn't find$MAGENTA pylint$COLOR_RESET. Please ensure it is discoverable."
+
+        exit 1
+    fi
+
+    log "(venv) Running$MAGENTA pylint$COLOR_RESET on$MAGENTA MFC$COLOR_RESET."
 
+    pylint -d R1722,W0718,C0301,C0116,C0115,C0114,C0410,W0622,W0640 toolchain/
+else
+    # Run the main.py bootstrap script
+    python3 "$(pwd)/toolchain/mfc.py" "$@"
+    code=$?
+fi
 
 # Deactivate the Python virtualenv in case the user "source"'d this script
 log "(venv) Exiting the$MAGENTA Python$COLOR_RESET virtual environment."

toolchain/bench.yaml

@@ -0,0 +1,9 @@
+- slug: 3D_shockdroplet
+  path: examples/3D_shockdroplet/case.py
+  args: []
+- slug: 3D_sphbubcollapse
+  path: examples/3D_sphbubcollapse/case.py
+  args: []
+- slug: 3D_turb_mixing
+  path: examples/3D_turb_mixing/case.py
+  args: []