Merge branch 'main' into dependabot/github_actions/jidicula/clang-for…

…mat-action-4.13.0

.github/workflows/ci.yml

@@ -34,6 +34,8 @@ jobs:
   linterDependencies:
     name: Build linter dependencies
     runs-on: ubuntu-latest
+    permissions:
+      packages: write
     steps:
       - uses: actions/checkout@v4
 
@@ -82,6 +84,14 @@ jobs:
           cache-from: type=gha,ref=linter-dependencies-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_linter') }}
           cache-to: type=gha,mode=min,ref=linter-dependencies-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_linter') }}
 
+      - name: Retag and push existing image if cache hit
+        if: env.CACHE_HIT == 'true'
+        run: |
+          TAGS=(${{ steps.dockerMetadata.outputs.tags }})
+          for TAG in "${TAGS[@]}"; do
+            docker buildx imagetools create --tag $TAG ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }}
+          done
+
   linter:
     name: Build And Run linter
     needs: linterDependencies
@@ -163,6 +173,14 @@ jobs:
           cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
           platforms: linux/amd64,linux/arm64
 
+      - name: Retag and push existing image if cache hit
+        if: env.CACHE_HIT == 'true'
+        run: |
+          TAGS=(${{ steps.dockerMetadata.outputs.tags }})
+          for TAG in "${TAGS[@]}"; do
+            docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }}
+          done
+
   dockerImageUnitTests:
     name: Build Docker Image and Run Unit Tests
     runs-on: ubuntu-latest

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-    ".": "0.2.0"
+    ".": "0.2.5"
 }

CHANGELOG.md

@@ -1,5 +1,49 @@
 # Changelog
 
+## [0.2.5](https://github.com/GenSpectrum/LAPIS-SILO/compare/v0.2.4...v0.2.5) (2024-06-17)
+
+
+### Bug Fixes
+
+* empty input without partitioning ([5fa3c92](https://github.com/GenSpectrum/LAPIS-SILO/commit/5fa3c9267eff1b0e59dda718a2ded0f874b038ae))
+
+## [0.2.4](https://github.com/GenSpectrum/LAPIS-SILO/compare/v0.2.3...v0.2.4) (2024-06-14)
+
+
+### Features
+
+* allow null for sequenceName in insertion contains queries ([6dbe251](https://github.com/GenSpectrum/LAPIS-SILO/commit/6dbe251a03e8a317188de0292f0e637b8ec4c24d))
+
+
+### Bug Fixes
+
+* more efficient ndjson emptiness check ([#481](https://github.com/GenSpectrum/LAPIS-SILO/issues/481)) ([344ec7b](https://github.com/GenSpectrum/LAPIS-SILO/commit/344ec7b20a3d6727e8972334542a610260cdc782))
+
+## [0.2.3](https://github.com/GenSpectrum/LAPIS-SILO/compare/v0.2.2...v0.2.3) (2024-06-10)
+
+
+### Bug Fixes
+
+* be able to start without genes, without nucleotide sequences or with neither ([e878ed5](https://github.com/GenSpectrum/LAPIS-SILO/commit/e878ed5567679f966c615a26b2d4c713f2f045e6))
+* empty ndjson input files more robust ([#473](https://github.com/GenSpectrum/LAPIS-SILO/issues/473)) ([9d4232b](https://github.com/GenSpectrum/LAPIS-SILO/commit/9d4232b01cb366def7e464155867bc6f7a466471))
+* erroneous file created during unit tests should not leak ([1b764af](https://github.com/GenSpectrum/LAPIS-SILO/commit/1b764afbefe13969496bf8b2bf51d50824c495bf))
+* insertions being added at wrong index for large files ([#472](https://github.com/GenSpectrum/LAPIS-SILO/issues/472)) ([e056ed9](https://github.com/GenSpectrum/LAPIS-SILO/commit/e056ed9354306690b51d6d0f829b962420e2be24))
+* remove incorrect compile flags ([b92ee4e](https://github.com/GenSpectrum/LAPIS-SILO/commit/b92ee4e0816f0f5b9ef1bef711b968600620f80d))
+
+## [0.2.2](https://github.com/GenSpectrum/LAPIS-SILO/compare/v0.2.1...v0.2.2) (2024-05-31)
+
+
+### Bug Fixes
+
+* start with empty files without throwing an error ([d407b92](https://github.com/GenSpectrum/LAPIS-SILO/commit/d407b9277ea89294ebbc581a80d23124040e266d))
+
+## [0.2.1](https://github.com/GenSpectrum/LAPIS-SILO/compare/v0.2.0...v0.2.1) (2024-05-28)
+
+
+### Bug Fixes
+
+* allow insertions that start at 0 ([#449](https://github.com/GenSpectrum/LAPIS-SILO/issues/449)) ([f427137](https://github.com/GenSpectrum/LAPIS-SILO/commit/f4271375b743cab0b012ef4337316c8bb3d0c6d8))
+
 ## [0.2.0](https://github.com/GenSpectrum/LAPIS-SILO/compare/v0.1.1...v0.2.0) (2024-05-23)
 
 

CMakeLists.txt

@@ -9,7 +9,7 @@ message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
 
 set(CMAKE_CXX_FLAGS "-Wall")
 set(CMAKE_CXX_FLAGS_DEBUG "-g -fsanitize=address")
-set(CMAKE_CXX_FLAGS_RELEASE "-O3 -static-libstdc++ -static-libgcc")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3")
 
 # Work-around only for MacOS
 if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
@@ -56,11 +56,8 @@ include_directories(SYSTEM
 
 file(GLOB_RECURSE SRC_TEST "src/*.test.cpp")
 
-file(GLOB_RECURSE SRC_SILO_LIB "src/silo/*.cpp")
-list(REMOVE_ITEM SRC_SILO_LIB ${SRC_TEST})
-
-file(GLOB_RECURSE SRC_SILO_API "src/silo_api/*.cpp")
-list(REMOVE_ITEM SRC_SILO_API ${SRC_TEST})
+file(GLOB_RECURSE SRC_SILO "src/*.cpp")
+list(REMOVE_ITEM SRC_SILO ${SRC_TEST})
 
 # ---------------------------------------------------------------------------
 # Linter
@@ -82,9 +79,9 @@ endif ()
 # Targets
 # ---------------------------------------------------------------------------
 
-add_library(silo ${SRC_SILO_LIB})
+add_executable(siloApi src/silo_api/api.cpp ${SRC_SILO})
 target_link_libraries(
-        silo
+        siloApi
         PUBLIC
         ${Boost_LIBRARIES}
         ${duckdb_LIBRARIES}
@@ -94,11 +91,11 @@ target_link_libraries(
         TBB::tbb
         ${yaml-cpp_LIBRARIES}
         zstd::libzstd_static
+        Poco::Net
+        Poco::Util
+        Poco::JSON
 )
 
-add_executable(siloApi src/silo_api/api.cpp ${SRC_SILO_API})
-target_link_libraries(siloApi PUBLIC silo Poco::Net Poco::Util Poco::JSON nlohmann_json::nlohmann_json ${spdlog_LIBRARIES})
-
 # ---------------------------------------------------------------------------
 # Tests
 # ---------------------------------------------------------------------------
@@ -107,12 +104,26 @@ enable_testing()
 find_package(GTest REQUIRED)
 include_directories(${GTest_INCLUDE_DIRS})
 
-set(SRC_SILO_API_WITHOUT_MAIN ${SRC_SILO_API})
-list(REMOVE_ITEM SRC_SILO_API_WITHOUT_MAIN "${CMAKE_SOURCE_DIR}/src/silo_api/api.cpp")
-
-add_executable(silo_test ${SRC_TEST} ${SRC_SILO_API_WITHOUT_MAIN})
+set(SRC_SILO_WITHOUT_MAIN ${SRC_SILO})
+list(REMOVE_ITEM SRC_SILO_WITHOUT_MAIN "${CMAKE_SOURCE_DIR}/src/silo_api/api.cpp")
 
+add_executable(silo_test ${SRC_TEST} ${SRC_SILO_WITHOUT_MAIN})
 if (NOT GTest_LIBRARIES)
     set(GTest_LIBRARIES gtest gmock)
 endif ()
-target_link_libraries(silo_test ${GTest_LIBRARIES} silo Poco::Net Poco::Util Poco::JSON nlohmann_json::nlohmann_json)
+target_link_libraries(
+        silo_test
+        ${GTest_LIBRARIES}
+        ${Boost_LIBRARIES}
+        ${duckdb_LIBRARIES}
+        nlohmann_json::nlohmann_json
+        ${roaring_LIBRARIES}
+        ${spdlog_LIBRARIES}
+        TBB::tbb
+        ${yaml-cpp_LIBRARIES}
+        zstd::libzstd_static
+        Poco::Net
+        Poco::Util
+        Poco::JSON
+        nlohmann_json::nlohmann_json
+)

Dockerfile

@@ -10,14 +10,14 @@ RUN  \
     && cp build/siloApi .
 
 
-FROM alpine:3.18 AS server
+FROM alpine:3.20 AS server
 
 WORKDIR /app
 COPY docker_default_preprocessing_config.yaml ./default_preprocessing_config.yaml
 COPY docker_runtime_config.yaml ./runtime_config.yaml
 COPY --from=builder /src/siloApi ./
 
-RUN apk update && apk add libtbb=2021.9.0-r0 curl jq
+RUN apk update && apk add onetbb=2021.12.0-r0 curl jq
 
 # call /info, extract "seqeunceCount" from the JSON and assert that the value is not 0. If any of those fails, "exit 1".
 HEALTHCHECK --start-period=20s CMD curl --fail --silent localhost:8081/info | jq .sequenceCount | xargs test 0 -ne || exit 1

Dockerfile_dependencies

@@ -1,16 +1,16 @@
-FROM alpine:3.18
+FROM alpine:3.20
 
 ARG TARGETPLATFORM
 
 RUN apk update && apk add --no-cache py3-pip \
     build-base=0.5-r3 \
-    cmake=3.26.5-r0 \
-    bash=5.2.15-r5 \
-    linux-headers=6.3-r0 \
-    boost-build=1.82.0-r0 \
-    libtbb=2021.9.0-r0
+    cmake=3.29.3-r0 \
+    bash=5.2.26-r0 \
+    linux-headers=6.6-r0 \
+    boost-build=1.84.0-r0 \
+    onetbb=2021.12.0-r0
 
-RUN pip install conan==2.0.17
+RUN pip install conan==2.4.1 --break-system-packages
 
 WORKDIR /src
 COPY conanfile.py conanprofile.docker conanprofile.docker_arm ./

Dockerfile_linter_dependencies

@@ -17,6 +17,10 @@ RUN apt update \
 RUN pip install conan==2.0.17
 
 COPY conanfile.py conanprofile.docker ./
-RUN mv conanprofile.docker conanprofile
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
+        mv conanprofile.docker_arm conanprofile; \
+    else \
+        mv conanprofile.docker conanprofile; \
+    fi
 
 RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build -s build_type=Debug

conanfile.py

@@ -14,7 +14,7 @@ class SiloRecipe(ConanFile):
         "nlohmann_json/3.11.2",
         "gtest/cci.20210126",
         "roaring/1.0.0",
-        "spdlog/1.11.0",
+        "spdlog/1.14.1",
         "yaml-cpp/0.7.0",
         "zstd/1.5.5",
     ]

endToEndTests/package.json

@@ -7,7 +7,7 @@
   },
   "dependencies": {
     "chai": "^5.1.1",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.2",
     "supertest": "^7.0.0"
   }
 }

include/silo/common/aa_symbols.h

@@ -47,6 +47,7 @@ class AminoAcid {
    static constexpr std::string_view SYMBOL_NAME_LOWER_CASE = "amino acid";
    static constexpr std::string_view SYMBOL_NAME_UPPER_CASE = "AMINO ACID";
    static constexpr std::string_view SYMBOL_NAME_SHORT = "AA";
+   static constexpr std::string_view PREFIX = "aa_";
 
    static constexpr std::array<Symbol, COUNT> SYMBOLS{
       Symbol::GAP, Symbol::A, Symbol::C, Symbol::D,    Symbol::E, Symbol::F, Symbol::G,

include/silo/common/nucleotide_symbols.h

@@ -39,6 +39,7 @@ class Nucleotide {
    static constexpr std::string_view SYMBOL_NAME_LOWER_CASE = "nucleotide";
    static constexpr std::string_view SYMBOL_NAME_UPPER_CASE = "NUCLEOTIDE";
    static constexpr std::string_view SYMBOL_NAME_SHORT = "NUC";
+   static constexpr std::string_view PREFIX = "nuc_";
 
    static constexpr std::array<Symbol, COUNT> SYMBOLS{
       Symbol::GAP,

include/silo/common/string_utils.h

@@ -12,4 +12,22 @@ std::string removeSymbol(const std::string& value, char symbol);
 
 std::vector<std::string> slice(const std::vector<std::string>& elements, size_t start, size_t end);
 
+std::vector<std::string> prepend(std::string_view prefix, const std::vector<std::string>& elements);
+
+std::vector<std::string> tie(
+   std::string_view prefix,
+   const std::vector<std::string>& elements1,
+   std::string_view delimiter,
+   const std::vector<std::string>& elements2,
+   std::string_view suffix
+);
+
+std::string tieAsString(
+   std::string_view prefix,
+   const std::vector<std::string>& elements1,
+   std::string_view delimiter,
+   const std::vector<std::string>& elements2,
+   std::string_view suffix
+);
+
 }  // namespace silo

include/silo/common/table_reader.h

@@ -13,9 +13,16 @@
 
 namespace silo {
 
-struct ColumnFunction {
+class ColumnFunction {
+   friend class TableReader;
    std::string column_name;
    std::function<void(size_t, const duckdb::Value&)> function;
+
+  public:
+   ColumnFunction(
+      std::string column_name,
+      std::function<void(size_t, const duckdb::Value&)> function
+   );
 };
 
 class TableReader {
@@ -29,12 +36,7 @@ class TableReader {
    std::unique_ptr<duckdb::MaterializedQueryResult> query_result;
    std::unique_ptr<duckdb::DataChunk> current_chunk;
    size_t current_row;
-
-   std::optional<std::string> nextKey();
-
-   std::string getTableQuery();
-
-   void advanceRow();
+   size_t current_row_in_chunk;
 
   public:
    explicit TableReader(
@@ -46,7 +48,14 @@ class TableReader {
       std::string_view order_by_clause
    );
 
-   void read();
+   size_t read();
+
+  private:
+   std::optional<std::string> nextKey();
+
+   std::string getTableQuery();
+
+   void advanceRow();
 
    void loadTable();
 };