refactor: use the same workflow file for building, testing and linting

.github/workflows/ci.yml

@@ -1,7 +1,10 @@
 name: LAPIS-SILO
 
 on:
+  pull_request:
   push:
+    branches:
+      - main
 
 concurrency:
   group: ci-${{ github.ref }}
@@ -12,48 +15,42 @@ env:
   DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo
 
 jobs:
-  formatting-check:
-    name: Formatting Check
+  dockerImageUnitTests:
+    name: Build Docker Image and Run Unit Tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        path:
-          - check: 'src'
-          - check: 'include'
-            exclude: '(PerfEvent.hpp)'
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run clang-format style check
-        uses: jidicula/[email protected]
-        with:
-          clang-format-version: '17'
-          check-path: ${{ matrix.path['check'] }}
-          exclude-regex: ${{ matrix.path['exclude'] }}
-
-  dependencyImage:
-    name: Build Docker Image Dependencies
-    runs-on: ubuntu-latest
+        platform:
+          ['linux/amd64', 'linux/arm64']
     permissions:
       packages: write
     steps:
       - uses: actions/checkout@v4
 
+      - name: Determine HEAD_SHA
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
+          fi
+
       - name: Generate dependency files hash
         id: files-hash
         run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }})
+          DIR_HASH=$(echo -n ${{ hashFiles(matrix.platform, 'conanfile.py', 'conanprofile.docker', './Dockerfile_dependencies') }})
           echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
 
       - name: Docker metadata
-        id: dockerMetadata
+        id: dockerMetadataDependencies
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}
           tags: |
             type=ref,event=branch
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
-            type=raw,value=${{ env.DIR_HASH }}
-            type=sha,format=long,prefix=commit-
+            type=raw,value=filehash-${{ env.DIR_HASH }}
+            type=raw,value=commit-${{ env.HEAD_SHA }}
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
@@ -63,71 +60,58 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Check if image exists
-        id: check-image
         run: |
           EXISTS=$(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
           echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
 
+      - name: Determine platform build necessity
+        id: platform-check
+        run: |
+          if [[ "${{ matrix.platform }}" == "linux/amd64" || "${{ github.ref }}" == "refs/heads/main" ]]; then
+            echo "PLATFORM_BUILD=true" >> $GITHUB_ENV
+          else
+            echo "Skipping build for this platform"
+            echo "PLATFORM_BUILD=false" >> $GITHUB_ENV
+          fi
+        shell: bash
+
       - name: Set up Docker Buildx
-        if: env.CACHE_HIT == 'false'
+        if: env.PLATFORM_BUILD == 'true'
         uses: docker/setup-buildx-action@v3
 
       - name: Build and push image if input files changed
-        if: env.CACHE_HIT == 'false'
+        if: env.CACHE_HIT == 'false' && env.PLATFORM_BUILD == 'true'
         uses: docker/build-push-action@v6
         with:
           context: .
           file: Dockerfile_dependencies
           push: true
-          tags: ${{ steps.dockerMetadata.outputs.tags }}
+          tags: ${{ steps.dockerMetadataDependencies.outputs.tags }}
           cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
           cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
-          platforms: linux/amd64,linux/arm64
+          platforms: ${{ matrix.platform }}
 
       - name: Retag and push existing image if cache hit
-        if: env.CACHE_HIT == 'true'
+        if: env.CACHE_HIT == 'true' && env.PLATFORM_BUILD == 'true'
         run: |
-          TAGS=(${{ steps.dockerMetadata.outputs.tags }})
+          TAGS=(${{ steps.dockerMetadataDependencies.outputs.tags }})
           for TAG in "${TAGS[@]}"; do
-            docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }}
+            docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}
           done
 
-  dockerImageUnitTests:
-    name: Build Docker Image and Run Unit Tests
-    runs-on: ubuntu-latest
-    needs: dependencyImage
-    permissions:
-      packages: write
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Generate dependency files hash
-        id: files-hash
-        run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }})
-          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
-
       - name: Docker metadata
-        id: dockerMetadata
+        if: env.PLATFORM_BUILD == 'true'
+        id: dockerMetadataImage
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_IMAGE_NAME }}
           tags: |
             type=ref,event=branch
             type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
-            type=sha,format=long,prefix=commit-
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+            type=raw,value=commit-${{ env.HEAD_SHA }}
 
       - name: Build unit test image
+        if: env.PLATFORM_BUILD == 'true'
         uses: docker/build-push-action@v6
         with:
           context: .
@@ -137,37 +121,41 @@ jobs:
           cache-from: type=gha,ref=${{ github.ref_name }}-image-cache
           cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache
           build-args: |
-            DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:${{ env.DIR_HASH }}
+            DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}
 
       - name: Run tests
+        if: env.PLATFORM_BUILD == 'true'
         uses: addnab/docker-run-action@v3
         with:
           image: builder
           run: ./silo_test
 
       - name: Build and push production image
-        if: ${{ github.event_name == 'push' }}
+        if: env.PLATFORM_BUILD == 'true'
         uses: docker/build-push-action@v6
         with:
           context: .
           push: true
           platforms: ${{ github.ref == 'refs/heads/main' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
-          tags: ${{ steps.dockerMetadata.outputs.tags }}
+          tags: ${{ steps.dockerMetadataImage.outputs.tags }}
           build-args: |
-            DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:${{ env.DIR_HASH }}
+            DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}
 
   endToEndTests:
     name: Run End To End Tests
     needs: dockerImageUnitTests
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        preprocessing-docker-compose: [
-          docker-compose-for-tests-preprocessing-from-ndjson.yml
-        ]
     steps:
       - uses: actions/checkout@v4
 
+      - name: Determine HEAD_SHA
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
+          fi
+
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -176,30 +164,67 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}"
 
       - name: npm install
         run: cd endToEndTests && npm ci
 
       - name: Check Format
         run: cd endToEndTests && npm run check-format
 
-      - name: Docker Metadata
-        id: dockerMetadata
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.DOCKER_IMAGE_NAME }}
-          tags: type=ref,event=branch
-
       - name: Start Docker Container and preprocess data
-        run: docker compose -f ${{ matrix.preprocessing-docker-compose }} up
+        run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up
         env:
-          SILO_IMAGE: ${{ steps.dockerMetadata.outputs.tags }}
+          SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}
 
       - name: Start Docker Container and run api
         run: docker compose -f docker-compose-for-tests-api.yml up -d --wait
         env:
-          SILO_IMAGE: ${{ steps.dockerMetadata.outputs.tags }}
+          SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}
 
       - name: Run Tests
-        run: cd endToEndTests && SILO_URL=localhost:8080 npm run test
+        run: cd endToEndTests && SILO_URL=localhost:8080 npm run test
+
+  linterChanges:
+    name: Build And Run linter
+    needs: dockerImageUnitTests
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    env:
+      PR_NUMBER: ${{ github.event.number }}
+    container:
+      image: ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ github.event.pull_request.head.sha }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - shell: bash
+        name: Configure and run clang-tidy on changed files
+        run: |
+          mv /src/build .
+          cmake -DBUILD_WITH_CLANG_TIDY=on -D CMAKE_BUILD_TYPE=Debug -B build/Debug
+          echo "Successfully configured cmake"
+          files=""
+          PAGE=1  
+          while true; do
+            page_files=$(curl -s \
+           "https://api.github.com/repos/${{ github.repository }}/pulls/${PR_NUMBER}/files?per_page=100&page=$PAGE" \
+           | jq -r '.[] | select(.status != "removed") | .filename')
+            # If there are no more files, break the loop
+            if [[ -z "$page_files" ]]; then
+              break
+            fi
+            files+="$page_files"$'\n'
+            PAGE=$((PAGE + 1))
+          done
+          
+          echo "Changed files of this PR:"
+          echo "$files"
+          IFS=$'\n'
+          for file in $files; do
+            echo "Check ending for file: $file"
+            if [[ $file == *.cpp ]]; then
+              echo "Now linting the file: $file"
+              echo "cmake --build build/Debug --target ${file%.cpp}.o"
+              cmake --build build/Debug --target ${file%.cpp}.o
+            fi
+          done

.github/workflows/formatting.yml

@@ -0,0 +1,22 @@
+name: Formatting
+on:
+  push:
+
+jobs:
+  formatting-check:
+    name: Formatting Check
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        path:
+          - check: 'src'
+          - check: 'include'
+            exclude: '(PerfEvent.hpp)'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run clang-format style check
+        uses: jidicula/[email protected]
+        with:
+          clang-format-version: '17'
+          check-path: ${{ matrix.path['check'] }}
+          exclude-regex: ${{ matrix.path['exclude'] }}