chore(deps): use Ubuntu 24.04 as base image, update dependencies

.github/workflows/ci.yml

@@ -2,6 +2,13 @@ name: LAPIS-SILO
 
 on:
   push:
+  workflow_dispatch:
+    inputs:
+      build_arm:
+        type: boolean
+        description: "Build for ARM as well"
+        default: false
+        required: false
 
 concurrency:
   group: ci-${{ github.ref }}
@@ -11,6 +18,7 @@ env:
   DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies
   DOCKER_LINTER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-linter-dependencies
   DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo
+  BUILD_ARM: ${{ github.event.inputs.build_arm || github.ref == 'refs/heads/main' }}
 
 jobs:
   formatting-check:
@@ -19,15 +27,15 @@ jobs:
     strategy:
       matrix:
         path:
-          - check: 'src'
-          - check: 'include'
-            exclude: '(PerfEvent.hpp)'
+          - check: "src"
+          - check: "include"
+            exclude: "(PerfEvent.hpp)"
     steps:
       - uses: actions/checkout@v4
       - name: Run clang-format style check
         uses: jidicula/[email protected]
         with:
-          clang-format-version: '17'
+          clang-format-version: "17"
           check-path: ${{ matrix.path['check'] }}
           exclude-regex: ${{ matrix.path['exclude'] }}
 
@@ -39,8 +47,17 @@ jobs:
 
       - name: Generate dependency files hash
         id: files-hash
-        run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_linter_dependencies') }})
+        run: |-
+          DIR_HASH=$(
+            echo -n ${{
+              hashFiles(
+                'conanfile.py',
+                'conanprofile.docker',
+                '.github/workflows/ci.yml',
+                './Dockerfile_linter_dependencies'
+              )
+            }}
+          )
           echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
 
       - name: Docker metadata
@@ -91,8 +108,17 @@ jobs:
 
       - name: Generate dependency files hash
         id: files-hash
-        run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_linter_dependencies') }})
+        run: |-
+          DIR_HASH=$(
+            echo -n ${{
+              hashFiles(
+                'conanfile.py',
+                'conanprofile.docker',
+                '.github/workflows/ci.yml',
+                './Dockerfile_linter_dependencies'
+              )
+            }}
+          )
           echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
 
       - name: Set up Docker Buildx
@@ -119,9 +145,22 @@ jobs:
 
       - name: Generate dependency files hash
         id: files-hash
-        run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }})
-          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
+        run: |-
+          DIR_HASH=$(
+            echo -n ${{
+              hashFiles(
+                'conanfile.py',
+                'conanprofile.docker',
+                'conanprofile.docker_arm',
+                '.github/workflows/ci.yml',
+                './Dockerfile_dependencies'
+              )
+            }}
+          )
+          if [ $BUILD_ARM == 'true' ]; then
+            echo "NONARM_HASH=$DIR_HASH" >> $GITHUB_ENV
+          fi
+          echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV
 
       - name: Docker metadata
         id: dockerMetadata
@@ -132,6 +171,7 @@ jobs:
             type=ref,event=branch
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
             type=raw,value=${{ env.DIR_HASH }}
+            type=raw,value=${{ env.NONARM_HASH }},enable=${{ env.BUILD_ARM }}
             type=sha,prefix=commit-
 
       - name: Login to GitHub Container Registry
@@ -161,7 +201,7 @@ jobs:
           tags: ${{ steps.dockerMetadata.outputs.tags }}
           cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
           cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
-          platforms: linux/amd64,linux/arm64
+          platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
 
   dockerImageUnitTests:
     name: Build Docker Image and Run Unit Tests
@@ -174,9 +214,19 @@ jobs:
 
       - name: Generate dependency files hash
         id: files-hash
-        run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }})
-          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
+        run: |-
+          DIR_HASH=$(
+            echo -n ${{
+              hashFiles(
+                'conanfile.py',
+                'conanprofile.docker',
+                'conanprofile.docker_arm',
+                '.github/workflows/ci.yml',
+                './Dockerfile_dependencies'
+              )
+            }}
+          )
+          echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV
 
       - name: Docker metadata
         id: dockerMetadata
@@ -233,10 +283,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        preprocessing-docker-compose: [
-          docker-compose-for-tests-preprocessing-from-tsv.yml,
-          docker-compose-for-tests-preprocessing-from-ndjson.yml
-        ]
+        preprocessing-docker-compose:
+          [
+            docker-compose-for-tests-preprocessing-from-tsv.yml,
+            docker-compose-for-tests-preprocessing-from-ndjson.yml,
+          ]
     steps:
       - uses: actions/checkout@v4
 
@@ -274,4 +325,4 @@ jobs:
           SILO_IMAGE: ${{ steps.dockerMetadata.outputs.tags }}
 
       - name: Run Tests
-        run: cd endToEndTests && SILO_URL=localhost:8080 npm run test
+        run: cd endToEndTests && SILO_URL=localhost:8080 npm run test

.gitignore

@@ -62,6 +62,8 @@ conanprofile
 *.out
 *.app
 
+# Debug files
+debug.duckdb
 
 **/output/
 **/logs/

CMakeLists.txt

@@ -67,7 +67,7 @@ list(REMOVE_ITEM SRC_SILO_API ${SRC_TEST})
 # ---------------------------------------------------------------------------
 
 option(BUILD_WITH_CLANG_TIDY "Build process clang-tidy")
-if (NOT CMAKE_BUILD_TYPE STREQUAL Release AND BUILD_WITH_CLANG_TIDY)
+if (NOT CMAKE_BUILD_TYPE STREQUAL Release AND BUILD_WITH_CLANG_TIDY AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
     find_program(CLANG_TIDY_EXE NAMES clang-tidy-19)
     if (NOT CLANG_TIDY_EXE)
         message(SEND_ERROR "clang-tidy not found, aborting. You can run the build with '-D BUILD_WITH_CLANG_TIDY=OFF' to disable clang-tidy.")

Dockerfile

@@ -4,27 +4,34 @@ FROM $DEPENDENCY_IMAGE AS builder
 
 COPY . ./
 
-RUN  \
-    python ./build_with_conan.py --release --parallel 4\
+RUN python3 ./build_with_conan.py --release --parallel 4 \
     && cp build/silo_test . \
     && cp build/siloApi .
 
 
-FROM alpine:3.18 AS server
+FROM ubuntu:24.04 AS server
+
+LABEL org.opencontainers.image.source="https://github.com/GenSpectrum/LAPIS-SILO"
+LABEL org.opencontainers.image.description="Sequence Indexing engine for Large Order of genomic data"
+
+RUN apt-get update \
+    && apt-get install -y \
+    curl \
+    jq \
+    libtbb12 \
+    && apt-get clean \
+    && apt-get autoremove --purge -y
 
 WORKDIR /app
 COPY docker_default_preprocessing_config.yaml ./default_preprocessing_config.yaml
 COPY docker_runtime_config.yaml ./runtime_config.yaml
 COPY --from=builder /src/siloApi ./
 
-RUN apk update && apk add libtbb=2021.9.0-r0 curl jq
 
-# call /info, extract "seqeunceCount" from the JSON and assert that the value is not 0. If any of those fails, "exit 1".
+# call /info, extract "sequenceCount" from the JSON and assert that the value is not 0. If any of those fails, "exit 1".
+#TODO: This health check wrongly assumes that no sequences means there is a problem.
 HEALTHCHECK --start-period=20s CMD curl --fail --silent localhost:8081/info | jq .sequenceCount | xargs test 0 -ne || exit 1
 
 EXPOSE 8081
 
 ENTRYPOINT ["./siloApi"]
-
-LABEL org.opencontainers.image.source="https://github.com/GenSpectrum/LAPIS-SILO"
-LABEL org.opencontainers.image.description="Sequence Indexing engine for Large Order of genomic data"

Dockerfile_dependencies

@@ -1,23 +1,28 @@
-FROM alpine:3.18
+FROM ubuntu:24.04
 
 ARG TARGETPLATFORM
 
-RUN apk update && apk add --no-cache py3-pip \
-    build-base=0.5-r3 \
-    cmake=3.26.5-r0 \
-    bash=5.2.15-r5 \
-    linux-headers=6.3-r0 \
-    boost-build=1.82.0-r0 \
-    libtbb=2021.9.0-r0
+WORKDIR /src
 
-RUN pip install conan==2.0.17
+RUN apt-get update \
+    && apt-get install -y \
+    cmake \
+    pipx \
+    build-essential \
+    gcc \
+    g++ \
+    && apt-get clean \
+    && apt-get autoremove --purge -y
+
+ENV PATH="/root/.local/bin:$PATH"
+RUN pipx install conan --pip-args="--no-cache-dir" --system-site-packages
 
-WORKDIR /src
 COPY conanfile.py conanprofile.docker conanprofile.docker_arm ./
+
 RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
-        mv conanprofile.docker_arm conanprofile; \
+    mv conanprofile.docker_arm conanprofile; \
     else \
-        mv conanprofile.docker conanprofile; \
+    mv conanprofile.docker conanprofile; \
     fi
 
-RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build
+RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build

Dockerfile_linter_dependencies

@@ -1,22 +1,36 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
+
+ARG TARGETPLATFORM
 
 WORKDIR /src
 
-RUN apt update \
-    && apt install -y \
-    cmake=3.22.1-1ubuntu1.22.04.2 \
-    python3-pip=22.0.2+dfsg-1ubuntu0.4 \
-    software-properties-common=0.99.22.9 \
-    wget=1.21.2-2ubuntu1 \
-    gnupg=2.2.27-3ubuntu2.1 \
-    lsb-release=11.1.0ubuntu4 \
+RUN apt-get update \
+    && apt-get install -y \
+    cmake \
+    build-essential \
+    gcc \
+    g++ \
+    software-properties-common \
+    wget \
+    gnupg \
+    lsb-release \
+    pipx \
     && wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc \
-    && add-apt-repository 'deb http://apt.llvm.org/jammy/  llvm-toolchain-jammy main' \
-    && apt install -y clang-tidy-19
+    && add-apt-repository 'deb http://apt.llvm.org/mantic/  llvm-toolchain-mantic main' \
+    && apt-get update \
+    && apt-get install -y clang-tidy-19 \
+    && apt-get clean \
+    && apt-get autoremove --purge -y
+
+ENV PATH="/root/.local/bin:$PATH"
+RUN pipx install conan --pip-args="--no-cache-dir" --system-site-packages
 
-RUN pip install conan==2.0.17
+COPY conanfile.py conanprofile.docker conanprofile.docker_arm ./
 
-COPY conanfile.py conanprofile.docker ./
-RUN mv conanprofile.docker conanprofile
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
+    mv conanprofile.docker_arm conanprofile; \
+    else \
+    mv conanprofile.docker conanprofile; \
+    fi
 
 RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build -s build_type=Debug

conanfile.py

@@ -5,18 +5,19 @@
 class SiloRecipe(ConanFile):
     settings = "os", "compiler", "build_type", "arch"
 
+    # Update regularly
     requires = [
-        "boost/1.82.0",
-        "duckdb/0.8.1",
-        "poco/1.12.4",
-        "hwloc/2.9.3",
-        "onetbb/2021.10.0",
-        "nlohmann_json/3.11.2",
-        "gtest/cci.20210126",
-        "roaring/1.0.0",
-        "spdlog/1.11.0",
-        "yaml-cpp/0.7.0",
-        "zstd/1.5.5",
+        "boost/1.83.0", # https://conan.io/center/recipes?value=boosta #FIXME: Serialization error with 1.84.0
+        "duckdb/0.8.1", # https://conan.io/center/recipes?value=duckdb #FIXME: test files only compatible with 0.8, need to EXPORT DATABASE in v0.8, then IMPORT DATABASE on the current version of DuckDB
+        "gtest/1.14.0", # https://conan.io/center/recipes?value=gtest
+        "hwloc/2.9.3", # https://conan.io/center/recipes?value=hwloc # BLOCKED: Pinned by onetbb <= 2021.12.0
+        "nlohmann_json/3.11.3", # https://conan.io/center/recipes?value=nlohmann_json
+        "onetbb/2021.12.0", # https://conan.io/center/recipes?value=onetbb
+        "poco/1.13.3", # https://conan.io/center/recipes?value=poco
+        "roaring/4.0.0", # https://conan.io/center/recipes?value=roaring
+        "spdlog/1.11.0", # https://conan.io/center/recipes?value=spdlog #FIXME: 1.12.0 fails with `error: implicit instantiation of undefined template` #TODO: holding back upgrade to 1.14.1
+        "yaml-cpp/0.8.0", # https://conan.io/center/recipes?value=yaml-cpp
+        "zstd/1.5.5", # https://conan.io/center/recipes?value=zstd #BLOCKED: Pinned by boost <=1.85.0
     ]
 
     default_options = {

conanprofile.docker

@@ -3,6 +3,6 @@ os=Linux
 arch=x86_64
 compiler=gcc
 compiler.libcxx=libstdc++11
-compiler.version=9.3
+compiler.version=12.2
 build_type=Release
 [options]

conanprofile.docker_arm

@@ -3,6 +3,6 @@ os=Linux
 arch=armv8
 compiler=gcc
 compiler.libcxx=libstdc++11
-compiler.version=9.3
+compiler.version=12.2
 build_type=Release
 [options]

conanprofile.example → conanprofile.linux_x86

@@ -3,6 +3,6 @@ os=Linux
 arch=x86_64
 compiler=gcc
 compiler.libcxx=libstdc++11
-compiler.version=9.3
+compiler.version=12.2
 build_type=Release
 [options]

conanprofile.mac_arm

@@ -0,0 +1,8 @@
+[settings]
+os=Macos
+arch=armv8
+compiler=gcc
+compiler.libcxx=libstdc++11
+compiler.version=12.2
+build_type=Release
+[options]

include/silo/config/util/abstract_config_source.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstdint>
 #include <optional>
 #include <string>
 #include <string_view>