Skip to content

Dependency image refactor #2844

Dependency image refactor

Dependency image refactor #2844

Workflow file for this run

name: LAPIS-SILO
on:
pull_request:
push:
branches:
- main
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
env:
DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies
DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo
jobs:
dockerImageUnitTests:
name: Build Docker Image and Run Unit Tests
runs-on: ubuntu-latest
strategy:
matrix:
platform:
['linux/amd64', 'linux/arm64']
permissions:
packages: write
steps:
- uses: actions/checkout@v4
- name: Determine HEAD_SHA
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
else
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
fi
- name: Generate dependency files hash
id: files-hash
run: |
DIR_HASH=$(echo -n ${{ hashFiles(matrix.platform, 'conanfile.py', 'conanprofile.docker', './Dockerfile_dependencies') }})
echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
- name: Docker metadata
id: dockerMetadataDependencies
uses: docker/metadata-action@v5
with:
images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}
tags: |
type=ref,event=branch
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=raw,value=filehash-${{ env.DIR_HASH }}
type=raw,value=commit-${{ env.HEAD_SHA }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Check if image exists
run: |
EXISTS=$(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
- name: Determine platform build necessity
id: platform-check
run: |
if [[ "${{ matrix.platform }}" == "linux/amd64" || "${{ github.ref }}" == "refs/heads/main" ]]; then
echo "PLATFORM_BUILD=true" >> $GITHUB_ENV
else
echo "Skipping build for this platform"
echo "PLATFORM_BUILD=false" >> $GITHUB_ENV
fi
shell: bash
- name: Set up Docker Buildx
if: env.PLATFORM_BUILD == 'true'
uses: docker/setup-buildx-action@v3
- name: Build and push image if input files changed
if: env.CACHE_HIT == 'false' && env.PLATFORM_BUILD == 'true'
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile_dependencies
push: true
tags: ${{ steps.dockerMetadataDependencies.outputs.tags }}
cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
platforms: ${{ matrix.platform }}
- name: Retag and push existing image if cache hit
if: env.CACHE_HIT == 'true' && env.PLATFORM_BUILD == 'true'
run: |
TAGS=(${{ steps.dockerMetadataDependencies.outputs.tags }})
for TAG in "${TAGS[@]}"; do
docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}
done
- name: Docker metadata
if: env.PLATFORM_BUILD == 'true'
id: dockerMetadataImage
uses: docker/metadata-action@v5
with:
images: ${{ env.DOCKER_IMAGE_NAME }}
tags: |
type=ref,event=branch
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
type=raw,value=commit-${{ env.HEAD_SHA }}
- name: Build unit test image
if: env.PLATFORM_BUILD == 'true'
uses: docker/build-push-action@v6
with:
context: .
target: builder
tags: builder
load: true
cache-from: type=gha,ref=${{ github.ref_name }}-image-cache
cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache
build-args: |
DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}
- name: Run tests
if: env.PLATFORM_BUILD == 'true'
uses: addnab/docker-run-action@v3
with:
image: builder
run: ./silo_test
- name: Build and push production image
if: env.PLATFORM_BUILD == 'true'
uses: docker/build-push-action@v6
with:
context: .
push: true
platforms: ${{ github.ref == 'refs/heads/main' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
tags: ${{ steps.dockerMetadataImage.outputs.tags }}
build-args: |
DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}
endToEndTests:
name: Run End To End Tests
needs: dockerImageUnitTests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Determine HEAD_SHA
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
else
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
fi
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: 18.x
- uses: actions/cache@v4
with:
path: ~/.npm
key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}"
- name: npm install
run: cd endToEndTests && npm ci
- name: Check Format
run: cd endToEndTests && npm run check-format
- name: Start Docker Container and preprocess data
run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up
env:
SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}
- name: Start Docker Container and run api
run: docker compose -f docker-compose-for-tests-api.yml up -d --wait
env:
SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}
- name: Run Tests
run: cd endToEndTests && SILO_URL=localhost:8080 npm run test
linterChanges:
name: Build And Run linter
needs: dockerImageUnitTests
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
env:
PR_NUMBER: ${{ github.event.number }}
container:
image: ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ github.event.pull_request.head.sha }}
steps:
- uses: actions/checkout@v4
- shell: bash
name: Configure and run clang-tidy on changed files
run: |
mv /src/build .
cmake -DBUILD_WITH_CLANG_TIDY=on -D CMAKE_BUILD_TYPE=Debug -B build/Debug
echo "Successfully configured cmake"
files=""
PAGE=1
while true; do
page_files=$(curl -s \
"https://api.github.com/repos/${{ github.repository }}/pulls/${PR_NUMBER}/files?per_page=100&page=$PAGE" \
| jq -r '.[] | select(.status != "removed") | .filename')
# If there are no more files, break the loop
if [[ -z "$page_files" ]]; then
break
fi
files+="$page_files"$'\n'
PAGE=$((PAGE + 1))
done
echo "Changed files of this PR:"
echo "$files"
IFS=$'\n'
for file in $files; do
echo "Check ending for file: $file"
if [[ $file == *.cpp ]]; then
echo "Now linting the file: $file"
echo "cmake --build build/Debug --target ${file%.cpp}.o"
cmake --build build/Debug --target ${file%.cpp}.o
fi
done