Dependency image refactor #2844
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: LAPIS-SILO | |
on: | |
pull_request: | |
push: | |
branches: | |
- main | |
concurrency: | |
group: ci-${{ github.ref }} | |
cancel-in-progress: true | |
env: | |
DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies | |
DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo | |
jobs: | |
dockerImageUnitTests: | |
name: Build Docker Image and Run Unit Tests | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
platform: | |
['linux/amd64', 'linux/arm64'] | |
permissions: | |
packages: write | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Determine HEAD_SHA | |
run: | | |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
else | |
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
fi | |
- name: Generate dependency files hash | |
id: files-hash | |
run: | | |
DIR_HASH=$(echo -n ${{ hashFiles(matrix.platform, 'conanfile.py', 'conanprofile.docker', './Dockerfile_dependencies') }}) | |
echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV | |
- name: Docker metadata | |
id: dockerMetadataDependencies | |
uses: docker/metadata-action@v5 | |
with: | |
images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }} | |
tags: | | |
type=ref,event=branch | |
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} | |
type=raw,value=filehash-${{ env.DIR_HASH }} | |
type=raw,value=commit-${{ env.HEAD_SHA }} | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v3 | |
with: | |
registry: ghcr.io | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Check if image exists | |
run: | | |
EXISTS=$(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false") | |
echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV | |
- name: Determine platform build necessity | |
id: platform-check | |
run: | | |
if [[ "${{ matrix.platform }}" == "linux/amd64" || "${{ github.ref }}" == "refs/heads/main" ]]; then | |
echo "PLATFORM_BUILD=true" >> $GITHUB_ENV | |
else | |
echo "Skipping build for this platform" | |
echo "PLATFORM_BUILD=false" >> $GITHUB_ENV | |
fi | |
shell: bash | |
- name: Set up Docker Buildx | |
if: env.PLATFORM_BUILD == 'true' | |
uses: docker/setup-buildx-action@v3 | |
- name: Build and push image if input files changed | |
if: env.CACHE_HIT == 'false' && env.PLATFORM_BUILD == 'true' | |
uses: docker/build-push-action@v6 | |
with: | |
context: . | |
file: Dockerfile_dependencies | |
push: true | |
tags: ${{ steps.dockerMetadataDependencies.outputs.tags }} | |
cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
platforms: ${{ matrix.platform }} | |
- name: Retag and push existing image if cache hit | |
if: env.CACHE_HIT == 'true' && env.PLATFORM_BUILD == 'true' | |
run: | | |
TAGS=(${{ steps.dockerMetadataDependencies.outputs.tags }}) | |
for TAG in "${TAGS[@]}"; do | |
docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }} | |
done | |
- name: Docker metadata | |
if: env.PLATFORM_BUILD == 'true' | |
id: dockerMetadataImage | |
uses: docker/metadata-action@v5 | |
with: | |
images: ${{ env.DOCKER_IMAGE_NAME }} | |
tags: | | |
type=ref,event=branch | |
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} | |
type=raw,value=commit-${{ env.HEAD_SHA }} | |
- name: Build unit test image | |
if: env.PLATFORM_BUILD == 'true' | |
uses: docker/build-push-action@v6 | |
with: | |
context: . | |
target: builder | |
tags: builder | |
load: true | |
cache-from: type=gha,ref=${{ github.ref_name }}-image-cache | |
cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache | |
build-args: | | |
DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }} | |
- name: Run tests | |
if: env.PLATFORM_BUILD == 'true' | |
uses: addnab/docker-run-action@v3 | |
with: | |
image: builder | |
run: ./silo_test | |
- name: Build and push production image | |
if: env.PLATFORM_BUILD == 'true' | |
uses: docker/build-push-action@v6 | |
with: | |
context: . | |
push: true | |
platforms: ${{ github.ref == 'refs/heads/main' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
tags: ${{ steps.dockerMetadataImage.outputs.tags }} | |
build-args: | | |
DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }} | |
endToEndTests: | |
name: Run End To End Tests | |
needs: dockerImageUnitTests | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Determine HEAD_SHA | |
run: | | |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
else | |
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
fi | |
- name: Use Node.js ${{ matrix.node-version }} | |
uses: actions/setup-node@v4 | |
with: | |
node-version: 18.x | |
- uses: actions/cache@v4 | |
with: | |
path: ~/.npm | |
key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}" | |
- name: npm install | |
run: cd endToEndTests && npm ci | |
- name: Check Format | |
run: cd endToEndTests && npm run check-format | |
- name: Start Docker Container and preprocess data | |
run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up | |
env: | |
SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }} | |
- name: Start Docker Container and run api | |
run: docker compose -f docker-compose-for-tests-api.yml up -d --wait | |
env: | |
SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }} | |
- name: Run Tests | |
run: cd endToEndTests && SILO_URL=localhost:8080 npm run test | |
linterChanges: | |
name: Build And Run linter | |
needs: dockerImageUnitTests | |
if: github.event_name == 'pull_request' | |
runs-on: ubuntu-latest | |
env: | |
PR_NUMBER: ${{ github.event.number }} | |
container: | |
image: ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ github.event.pull_request.head.sha }} | |
steps: | |
- uses: actions/checkout@v4 | |
- shell: bash | |
name: Configure and run clang-tidy on changed files | |
run: | | |
mv /src/build . | |
cmake -DBUILD_WITH_CLANG_TIDY=on -D CMAKE_BUILD_TYPE=Debug -B build/Debug | |
echo "Successfully configured cmake" | |
files="" | |
PAGE=1 | |
while true; do | |
page_files=$(curl -s \ | |
"https://api.github.com/repos/${{ github.repository }}/pulls/${PR_NUMBER}/files?per_page=100&page=$PAGE" \ | |
| jq -r '.[] | select(.status != "removed") | .filename') | |
# If there are no more files, break the loop | |
if [[ -z "$page_files" ]]; then | |
break | |
fi | |
files+="$page_files"$'\n' | |
PAGE=$((PAGE + 1)) | |
done | |
echo "Changed files of this PR:" | |
echo "$files" | |
IFS=$'\n' | |
for file in $files; do | |
echo "Check ending for file: $file" | |
if [[ $file == *.cpp ]]; then | |
echo "Now linting the file: $file" | |
echo "cmake --build build/Debug --target ${file%.cpp}.o" | |
cmake --build build/Debug --target ${file%.cpp}.o | |
fi | |
done |