benchmark.yml

name: benchmark

on:
  push:
    branches:
      - master
    paths-ignore:
      - "docs/**"
      - "papers/**"
      - "rfcs/**"
      - "*.md"

jobs:
  callgrind:
    strategy:
      matrix:
        os: [ubuntu-22.04]
        benchResultsRepo:
          - { name: "luau-lang/benchmark-data", branch: "main" }

    runs-on: ${{ matrix.os }}
    steps:
      - name: Checkout Luau repository
        uses: actions/checkout@v3

      - name: Install valgrind
        run: |
          sudo apt-get update
          sudo apt-get install valgrind

      - name: Build Luau (gcc)
        run: |
          CXX=g++ make config=profile luau
          cp luau luau-gcc

      - name: Build Luau (codegen)
        run: |
          make config=profile clean
          CXX=clang++ make config=profile native=1 luau
          cp luau luau-codegen

      - name: Build Luau (clang)
        run: |
          make config=profile clean
          CXX=clang++ make config=profile luau luau-analyze luau-compile

      - name: Run benchmark (bench-gcc)
        run: |
          python bench/bench.py --callgrind --vm "./luau-gcc -O2" | tee -a bench-gcc-output.txt

      - name: Run benchmark (bench)
        run: |
          python bench/bench.py --callgrind --vm "./luau -O2" | tee -a bench-output.txt

      - name: Run benchmark (bench-codegen)
        run: |
          python bench/bench.py --callgrind --vm "./luau-codegen --codegen -O2" | tee -a bench-codegen-output.txt

      - name: Run benchmark (analyze)
        run: |
          filter() {
            awk '/.*I\s+refs:\s+[0-9,]+/ {gsub(",", "", $4); X=$4} END {print "SUCCESS: '$1' : " X/1e7 "ms +/- 0% on luau-analyze"}'
          }
          valgrind --tool=callgrind ./luau-analyze --mode=nonstrict bench/other/LuauPolyfillMap.lua 2>&1 | filter map-nonstrict | tee -a analyze-output.txt
          valgrind --tool=callgrind ./luau-analyze --mode=strict bench/other/LuauPolyfillMap.lua 2>&1 | filter map-strict | tee -a analyze-output.txt
          valgrind --tool=callgrind ./luau-analyze --mode=strict --fflags=DebugLuauDeferredConstraintResolution bench/other/LuauPolyfillMap.lua 2>&1 | filter map-dcr | tee -a analyze-output.txt
          valgrind --tool=callgrind ./luau-analyze --mode=nonstrict bench/other/regex.lua 2>&1 | filter regex-nonstrict | tee -a analyze-output.txt
          valgrind --tool=callgrind ./luau-analyze --mode=strict bench/other/regex.lua 2>&1 | filter regex-strict | tee -a analyze-output.txt
          valgrind --tool=callgrind ./luau-analyze --mode=strict --fflags=DebugLuauDeferredConstraintResolution bench/other/regex.lua 2>&1 | filter regex-dcr | tee -a analyze-output.txt

      - name: Run benchmark (compile)
        run: |
          filter() {
            awk '/.*I\s+refs:\s+[0-9,]+/ {gsub(",", "", $4); X=$4} END {print "SUCCESS: '$1' : " X/1e7 "ms +/- 0% on luau-compile"}'
          }
          valgrind --tool=callgrind ./luau-compile --null -O0 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O0 | tee -a compile-output.txt
          valgrind --tool=callgrind ./luau-compile --null -O1 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O1 | tee -a compile-output.txt
          valgrind --tool=callgrind ./luau-compile --null -O2 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O2 | tee -a compile-output.txt
          valgrind --tool=callgrind ./luau-compile --codegennull -O2 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O2-codegen | tee -a compile-output.txt
          valgrind --tool=callgrind ./luau-compile --null -O0 bench/other/regex.lua 2>&1 | filter regex-O0 | tee -a compile-output.txt
          valgrind --tool=callgrind ./luau-compile --null -O1 bench/other/regex.lua 2>&1 | filter regex-O1 | tee -a compile-output.txt
          valgrind --tool=callgrind ./luau-compile --null -O2 bench/other/regex.lua 2>&1 | filter regex-O2 | tee -a compile-output.txt
          valgrind --tool=callgrind ./luau-compile --codegennull -O2 bench/other/regex.lua 2>&1 | filter regex-O2-codegen | tee -a compile-output.txt

      - name: Checkout benchmark results
        uses: actions/checkout@v3
        with:
          repository: ${{ matrix.benchResultsRepo.name }}
          ref: ${{ matrix.benchResultsRepo.branch }}
          token: ${{ secrets.BENCH_GITHUB_TOKEN }}
          path: "./gh-pages"

      - name: Store results (bench)
        uses: Roblox/rhysd-github-action-benchmark@v-luau
        with:
          name: callgrind clang
          tool: "benchmarkluau"
          output-file-path: ./bench-output.txt
          external-data-json-path: ./gh-pages/bench.json

      - name: Store results (bench-codegen)
        uses: Roblox/rhysd-github-action-benchmark@v-luau
        with:
          name: callgrind codegen
          tool: "benchmarkluau"
          output-file-path: ./bench-codegen-output.txt
          external-data-json-path: ./gh-pages/bench-codegen.json

      - name: Store results (bench-gcc)
        uses: Roblox/rhysd-github-action-benchmark@v-luau
        with:
          name: callgrind gcc
          tool: "benchmarkluau"
          output-file-path: ./bench-gcc-output.txt
          external-data-json-path: ./gh-pages/bench-gcc.json

      - name: Store results (analyze)
        uses: Roblox/rhysd-github-action-benchmark@v-luau
        with:
          name: luau-analyze
          tool: "benchmarkluau"
          output-file-path: ./analyze-output.txt
          external-data-json-path: ./gh-pages/analyze.json

      - name: Store results (compile)
        uses: Roblox/rhysd-github-action-benchmark@v-luau
        with:
          name: luau-compile
          tool: "benchmarkluau"
          output-file-path: ./compile-output.txt
          external-data-json-path: ./gh-pages/compile.json

      - name: Push benchmark results
        if: github.event_name == 'push'
        run: |
          echo "Pushing benchmark results..."
          cd gh-pages
          git config user.name github-actions
          git config user.email github@users.noreply.github.com
          git add *.json
          git commit -m "Add benchmarks results for ${{ github.sha }}"
          git push
          cd ..