From 8cf19c5106b14f0037a5a05d901051e6cb704a3c Mon Sep 17 00:00:00 2001 From: Alexander Taepper Date: Thu, 24 Oct 2024 08:57:59 +0200 Subject: [PATCH] ci: use separate github workflows to depend on the same dependency image for tests and linter --- .github/workflows/dependencies.yml | 67 +++++++++++++++++++ .github/workflows/formatting.yml | 22 ++++++ .github/workflows/linter.yml | 70 ++----------------- .github/workflows/{ci.yml => tests.yml} | 89 ++----------------------- 4 files changed, 102 insertions(+), 146 deletions(-) create mode 100644 .github/workflows/dependencies.yml create mode 100644 .github/workflows/formatting.yml rename .github/workflows/{ci.yml => tests.yml} (55%) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 000000000..f26cc4e9d --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,67 @@ +name: Dependency Image + +on: + push: + +jobs: + dependencyImage: + name: Build Docker Image Dependencies + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Generate dependency files hash + id: files-hash + run: | + DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }}) + echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV + + - name: Docker metadata + id: dockerMetadata + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=${{ env.DIR_HASH }} + type=sha,format=long,prefix=commit- + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Check if image exists + id: check-image + run: | + EXISTS=$(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false") + echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV + + - name: Set up Docker Buildx + if: env.CACHE_HIT == 'false' + uses: docker/setup-buildx-action@v3 + + - name: Build and push image if input files changed + if: env.CACHE_HIT == 'false' + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile_dependencies + push: true + tags: ${{ steps.dockerMetadata.outputs.tags }} + cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} + cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} + platforms: linux/amd64,linux/arm64 + + - name: Retag and push existing image if cache hit + if: env.CACHE_HIT == 'true' + run: | + TAGS=(${{ steps.dockerMetadata.outputs.tags }}) + for TAG in "${TAGS[@]}"; do + docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} + done diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml new file mode 100644 index 000000000..c5b26d83f --- /dev/null +++ b/.github/workflows/formatting.yml @@ -0,0 +1,22 @@ +name: Formatting +on: + push: + +jobs: + formatting-check: + name: Formatting Check + runs-on: ubuntu-latest + strategy: + matrix: + path: + - check: 'src' + - check: 'include' + exclude: '(PerfEvent.hpp)' + steps: + - uses: actions/checkout@v4 + - name: Run clang-format style check + uses: jidicula/clang-format-action@v4.13.0 + with: + clang-format-version: '17' + check-path: ${{ matrix.path['check'] }} + exclude-regex: ${{ matrix.path['exclude'] }} diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 533cb7ed5..8a1f86ec6 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -2,84 +2,28 @@ name: Run Linter on: pull_request: + workflow_run: + workflows: [ "First Workflow" ] + types: + - completed concurrency: group: linter-${{ github.ref }} cancel-in-progress: true env: - DOCKER_LINTER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-linter-dependencies + DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies jobs: - linterDependencies: - name: Build linter dependencies - runs-on: ubuntu-latest - permissions: - packages: write - steps: - - uses: actions/checkout@v4 - - - name: Generate dependency files hash - id: files-hash - run: | - DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/linter.yml', './Dockerfile_linter_dependencies') }}) - echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV - - - name: Docker metadata - id: dockerMetadata - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} - type=raw,value=${{ env.DIR_HASH }} - type=sha,format=long,prefix=commit- - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Check if image exists - id: check-image - run: | - EXISTS=$(docker manifest inspect ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false") - echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV - - - name: Set up Docker Buildx - if: env.CACHE_HIT == 'false' - uses: docker/setup-buildx-action@v3 - - - name: Build linter dependencies image - if: env.CACHE_HIT == 'false' - uses: docker/build-push-action@v6 - with: - context: . - push: true - tags: ${{ steps.dockerMetadata.outputs.tags }} - file: ./Dockerfile_linter_dependencies - cache-from: type=gha,ref=linter-dependencies-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_linter') }} - cache-to: type=gha,mode=min,ref=linter-dependencies-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_linter') }} - - - name: Retag and push existing image if cache hit - if: env.CACHE_HIT == 'true' - run: | - TAGS=(${{ steps.dockerMetadata.outputs.tags }}) - for TAG in "${TAGS[@]}"; do - docker buildx imagetools create --tag $TAG ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} - done - linter: name: Build And Run linter needs: linterDependencies runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' && github.event_name == 'pull_request' }} env: PR_NUMBER: ${{ github.event.number }} container: - image: ghcr.io/genspectrum/lapis-silo-linter-dependencies:commit-${{ github.sha }} + image: "${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ github.sha }}" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/tests.yml similarity index 55% rename from .github/workflows/ci.yml rename to .github/workflows/tests.yml index a072a7c49..17e958682 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,11 @@ name: LAPIS-SILO on: - push: + workflow_run: + workflows: [ "Dependency Image" ] + types: + - completed + concurrency: group: ci-${{ github.ref }} @@ -12,90 +16,9 @@ env: DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo jobs: - formatting-check: - name: Formatting Check - runs-on: ubuntu-latest - strategy: - matrix: - path: - - check: 'src' - - check: 'include' - exclude: '(PerfEvent.hpp)' - steps: - - uses: actions/checkout@v4 - - name: Run clang-format style check - uses: jidicula/clang-format-action@v4.13.0 - with: - clang-format-version: '17' - check-path: ${{ matrix.path['check'] }} - exclude-regex: ${{ matrix.path['exclude'] }} - - dependencyImage: - name: Build Docker Image Dependencies - runs-on: ubuntu-latest - permissions: - packages: write - steps: - - uses: actions/checkout@v4 - - - name: Generate dependency files hash - id: files-hash - run: | - DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }}) - echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV - - - name: Docker metadata - id: dockerMetadata - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} - type=raw,value=${{ env.DIR_HASH }} - type=sha,format=long,prefix=commit- - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Check if image exists - id: check-image - run: | - EXISTS=$(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false") - echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV - - - name: Set up Docker Buildx - if: env.CACHE_HIT == 'false' - uses: docker/setup-buildx-action@v3 - - - name: Build and push image if input files changed - if: env.CACHE_HIT == 'false' - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile_dependencies - push: true - tags: ${{ steps.dockerMetadata.outputs.tags }} - cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} - cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} - platforms: linux/amd64,linux/arm64 - - - name: Retag and push existing image if cache hit - if: env.CACHE_HIT == 'true' - run: | - TAGS=(${{ steps.dockerMetadata.outputs.tags }}) - for TAG in "${TAGS[@]}"; do - docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} - done - dockerImageUnitTests: name: Build Docker Image and Run Unit Tests runs-on: ubuntu-latest - needs: dependencyImage permissions: packages: write steps: @@ -176,7 +99,7 @@ jobs: - uses: actions/cache@v4 with: path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}" - name: npm install run: cd endToEndTests && npm ci