-
Notifications
You must be signed in to change notification settings - Fork 384
145 lines (126 loc) · 6.25 KB
/
benchmark.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
name: benchmark
on:
push:
branches:
- master
paths-ignore:
- "docs/**"
- "papers/**"
- "rfcs/**"
- "*.md"
jobs:
callgrind:
strategy:
matrix:
os: [ubuntu-22.04]
benchResultsRepo:
- { name: "luau-lang/benchmark-data", branch: "main" }
runs-on: ${{ matrix.os }}
steps:
- name: Checkout Luau repository
uses: actions/checkout@v3
- name: Install valgrind
run: |
sudo apt-get update
sudo apt-get install valgrind
- name: Build Luau (gcc)
run: |
CXX=g++ make config=profile luau
cp luau luau-gcc
- name: Build Luau (codegen)
run: |
make config=profile clean
CXX=clang++ make config=profile native=1 luau
cp luau luau-codegen
- name: Build Luau (clang)
run: |
make config=profile clean
CXX=clang++ make config=profile luau luau-analyze luau-compile
- name: Run benchmark (bench-gcc)
run: |
python bench/bench.py --callgrind --vm "./luau-gcc -O2" | tee -a bench-gcc-output.txt
- name: Run benchmark (bench)
run: |
python bench/bench.py --callgrind --vm "./luau -O2" | tee -a bench-output.txt
- name: Run benchmark (bench-codegen)
run: |
python bench/bench.py --callgrind --vm "./luau-codegen --codegen -O2" | tee -a bench-codegen-output.txt
- name: Run benchmark (analyze)
run: |
filter() {
awk '/.*I\s+refs:\s+[0-9,]+/ {gsub(",", "", $4); X=$4} END {print "SUCCESS: '$1' : " X/1e7 "ms +/- 0% on luau-analyze"}'
}
valgrind --tool=callgrind ./luau-analyze --mode=nonstrict bench/other/LuauPolyfillMap.lua 2>&1 | filter map-nonstrict | tee -a analyze-output.txt
valgrind --tool=callgrind ./luau-analyze --mode=strict bench/other/LuauPolyfillMap.lua 2>&1 | filter map-strict | tee -a analyze-output.txt
valgrind --tool=callgrind ./luau-analyze --mode=strict --fflags=LuauSolverV2 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-dcr | tee -a analyze-output.txt
valgrind --tool=callgrind ./luau-analyze --mode=nonstrict bench/other/regex.lua 2>&1 | filter regex-nonstrict | tee -a analyze-output.txt
valgrind --tool=callgrind ./luau-analyze --mode=strict bench/other/regex.lua 2>&1 | filter regex-strict | tee -a analyze-output.txt
valgrind --tool=callgrind ./luau-analyze --mode=strict --fflags=LuauSolverV2 bench/other/regex.lua 2>&1 | filter regex-dcr | tee -a analyze-output.txt
- name: Run benchmark (compile)
run: |
filter() {
awk '/.*I\s+refs:\s+[0-9,]+/ {gsub(",", "", $4); X=$4} END {print "SUCCESS: '$1' : " X/1e7 "ms +/- 0% on luau-compile"}'
}
valgrind --tool=callgrind ./luau-compile --null -O0 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O0 | tee -a compile-output.txt
valgrind --tool=callgrind ./luau-compile --null -O1 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O1 | tee -a compile-output.txt
valgrind --tool=callgrind ./luau-compile --null -O2 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O2 | tee -a compile-output.txt
valgrind --tool=callgrind ./luau-compile --codegennull -O2 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O2-codegen | tee -a compile-output.txt
valgrind --tool=callgrind ./luau-compile --codegennull -O2 -t1 bench/other/LuauPolyfillMap.lua 2>&1 | filter map-O2-t1-codegen | tee -a compile-output.txt
valgrind --tool=callgrind ./luau-compile --null -O0 bench/other/regex.lua 2>&1 | filter regex-O0 | tee -a compile-output.txt
valgrind --tool=callgrind ./luau-compile --null -O1 bench/other/regex.lua 2>&1 | filter regex-O1 | tee -a compile-output.txt
valgrind --tool=callgrind ./luau-compile --null -O2 bench/other/regex.lua 2>&1 | filter regex-O2 | tee -a compile-output.txt
valgrind --tool=callgrind ./luau-compile --codegennull -O2 bench/other/regex.lua 2>&1 | filter regex-O2-codegen | tee -a compile-output.txt
valgrind --tool=callgrind ./luau-compile --codegennull -O2 -t1 bench/other/regex.lua 2>&1 | filter regex-O2-t1-codegen | tee -a compile-output.txt
- name: Checkout benchmark results
uses: actions/checkout@v3
with:
repository: ${{ matrix.benchResultsRepo.name }}
ref: ${{ matrix.benchResultsRepo.branch }}
token: ${{ secrets.BENCH_GITHUB_TOKEN }}
path: "./gh-pages"
- name: Store results (bench)
uses: Roblox/rhysd-github-action-benchmark@v-luau
with:
name: callgrind clang
tool: "benchmarkluau"
output-file-path: ./bench-output.txt
external-data-json-path: ./gh-pages/bench.json
- name: Store results (bench-codegen)
uses: Roblox/rhysd-github-action-benchmark@v-luau
with:
name: callgrind codegen
tool: "benchmarkluau"
output-file-path: ./bench-codegen-output.txt
external-data-json-path: ./gh-pages/bench-codegen.json
- name: Store results (bench-gcc)
uses: Roblox/rhysd-github-action-benchmark@v-luau
with:
name: callgrind gcc
tool: "benchmarkluau"
output-file-path: ./bench-gcc-output.txt
external-data-json-path: ./gh-pages/bench-gcc.json
- name: Store results (analyze)
uses: Roblox/rhysd-github-action-benchmark@v-luau
with:
name: luau-analyze
tool: "benchmarkluau"
output-file-path: ./analyze-output.txt
external-data-json-path: ./gh-pages/analyze.json
- name: Store results (compile)
uses: Roblox/rhysd-github-action-benchmark@v-luau
with:
name: luau-compile
tool: "benchmarkluau"
output-file-path: ./compile-output.txt
external-data-json-path: ./gh-pages/compile.json
- name: Push benchmark results
if: github.event_name == 'push'
run: |
echo "Pushing benchmark results..."
cd gh-pages
git config user.name github-actions
git config user.email [email protected]
git add *.json
git commit -m "Add benchmarks results for ${{ github.sha }}"
git push
cd ..