ci

commaai · Oct 2, 2024 · 8b46d01 · 8b46d01
1 parent c2edba2
commit 8b46d01
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 5 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -106,6 +106,9 @@ jobs:
   mutation:
     name: Mutation tests
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        mutation_mode: ['DIFF_COVERAGE', 'SAFETY_ONLY']
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@v2
@@ -115,7 +118,7 @@ jobs:
         run: eval "$BUILD"
       - name: Mutation tests
         timeout-minutes: 5
-        run: ${{ env.RUN }} "GIT_REF=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.event.before || 'origin/master' }} cd tests/safety && ./mutation.sh"
+        run: ${{ env.RUN }} "GIT_REF=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.event.before || 'origin/master' }} MUTATION_MODE=${{ matrix.mutation_mode }} cd tests/safety && ./mutation.sh"
 
   static_analysis:
     name: static analysis

diff --git a/tests/safety/mutation.sh b/tests/safety/mutation.sh
@@ -11,9 +11,9 @@ MUTATION_OPS="mutators: [cxx_increment, cxx_decrement, cxx_comparison, cxx_bound
 # TODO: add more files from board/safety
 MUTATION_SAFETY_FILES=( safety_body.h safety_defaults.h safety_elm327.h )
 
-# MODE_SAFETY_ONLY -> verify mutations on safety_xx.h with test_xx.py
-# MODE_DIFF_COVERAGE -> verify mutations with test_xx.py on the intersection between its code coverage and the current git diff
-MUTATION_MODE="${MUTATION_MODE:-MODE_DIFF_COVERAGE}"
+# SAFETY_ONLY -> verify mutations on safety_xx.h with test_xx.py
+# DIFF_COVERAGE -> verify mutations with test_xx.py on the intersection between its code coverage and the current git diff
+MUTATION_MODE="${MUTATION_MODE:-DIFF_COVERAGE}"
 
 GIT_REF="${GIT_REF:-origin/master}"
 GIT_ROOT=$(git rev-parse --show-toplevel)
@@ -30,7 +30,7 @@ scons --mutation -j$(nproc) -D
 
 SAFETY_TESTS=$(find * | grep "^test_.*\.py")
 for SAFETY_TEST in ${SAFETY_TESTS[@]}; do
-  if [[ $MUTATION_MODE == "MODE_SAFETY_ONLY" ]]; then
+  if [[ $MUTATION_MODE == "SAFETY_ONLY" ]]; then
     SAFETY_MODE=$(echo $SAFETY_TEST | sed -e 's/test_/safety_/g' | sed -e 's/\.py/\.h/g')
     if [[ ! " ${MUTATION_SAFETY_FILES[*]} " =~ [[:space:]]${SAFETY_MODE}[[:space:]] ]]; then
       continue