feat: insertions no longer in metadata or databaseConfig, instead exp…

…ected for all aligned sequences #372 BREAKING CHANGE: old database_config files might be invalid if they contained insertion columns. Also, we are more prohibitive for ndjson input files, which now MUST contain nucleotide/amino acid insertions for all respective sequences. The insertions action and filter do no longer require a column field.
GenSpectrum · May 23, 2024 · 4e29220 · 4e29220
1 parent decae59
commit 4e29220
Show file tree

Hide file tree

Showing 68 changed files with 1,012 additions and 1,178 deletions.
diff --git a/endToEndTests/test/invalidQueries/insertionContains_empty.json b/endToEndTests/test/invalidQueries/insertionContains_empty.json
@@ -6,7 +6,6 @@
     },
     "filterExpression": {
       "type": "InsertionContains",
-      "column": "nucleotideInsertions",
       "position": 25701,
       "value": ""
     }

diff --git a/endToEndTests/test/invalidQueries/insertionContains_invalidPattern.json b/endToEndTests/test/invalidQueries/insertionContains_invalidPattern.json
@@ -6,7 +6,6 @@
     },
     "filterExpression": {
       "type": "InsertionContains",
-      "column": "nucleotideInsertions",
       "position": 25701,
       "value": "CC+++"
     }

diff --git a/endToEndTests/test/invalidQueries/insertionContains_invalidPattern2.json b/endToEndTests/test/invalidQueries/insertionContains_invalidPattern2.json
@@ -6,7 +6,6 @@
     },
     "filterExpression": {
       "type": "InsertionContains",
-      "column": "nucleotideInsertions",
       "position": 25701,
       "value": "CC..*"
     }

diff --git a/endToEndTests/test/invalidQueries/insertionsAAseparation.json b/endToEndTests/test/invalidQueries/insertionsAAseparation.json
@@ -3,7 +3,6 @@
   "query": {
     "action": {
       "type": "Insertions",
-      "column": "nucleotideInsertions",
       "sequenceName": "S"
     },
     "filterExpression": {

diff --git a/endToEndTests/test/invalidQueries/insertionsInvalidColumn.json b/endToEndTests/test/invalidQueries/insertionsInvalidColumn.json
diff --git a/endToEndTests/test/invalidQueries/insertionsInvalidSequence.json b/endToEndTests/test/invalidQueries/insertionsInvalidSequence.json
@@ -3,7 +3,6 @@
   "query": {
     "action": {
       "type": "Insertions",
-      "column": "nucleotideInsertions",
       "sequenceName": "notAValidSequence"
     },
     "filterExpression": {

diff --git a/endToEndTests/test/queries/aaInsertionsContains.json b/endToEndTests/test/queries/aaInsertionsContains.json
@@ -3,7 +3,7 @@
   "query": {
     "action": {
       "type": "Details",
-      "fields": ["aminoAcidInsertions", "gisaid_epi_isl"],
+      "fields": ["gisaid_epi_isl"],
       "orderByFields": ["gisaid_epi_isl"]
     },
     "filterExpression": {
@@ -16,23 +16,18 @@
   },
   "expectedQueryResult": [
     {
-      "aminoAcidInsertions": "S:214:EPE",
       "gisaid_epi_isl": "EPI_ISL_1003629"
     },
     {
-      "aminoAcidInsertions": "S:214:EPE",
       "gisaid_epi_isl": "EPI_ISL_1408408"
     },
     {
-      "aminoAcidInsertions": "S:214:EPE",
       "gisaid_epi_isl": "EPI_ISL_1597932"
     },
     {
-      "aminoAcidInsertions": "S:214:EPE",
       "gisaid_epi_isl": "EPI_ISL_1750868"
     },
     {
-      "aminoAcidInsertions": "S:214:EPE",
       "gisaid_epi_isl": "EPI_ISL_581968"
     }
   ]

diff --git a/endToEndTests/test/queries/insertionContains_exact.json b/endToEndTests/test/queries/insertionContains_exact.json
@@ -6,7 +6,6 @@
     },
     "filterExpression": {
       "type": "InsertionContains",
-      "column": "nucleotideInsertions",
       "position": 25701,
       "value": "CCC"
     }

diff --git a/endToEndTests/test/queries/insertionContains_not_exact1.json b/endToEndTests/test/queries/insertionContains_not_exact1.json
@@ -6,7 +6,6 @@
     },
     "filterExpression": {
       "type": "InsertionContains",
-      "column": "nucleotideInsertions",
       "position": 22339,
       "value": ".*GCT.*GGT.*"
     }

diff --git a/endToEndTests/test/queries/insertionContains_not_exact2.json b/endToEndTests/test/queries/insertionContains_not_exact2.json
@@ -6,7 +6,6 @@
     },
     "filterExpression": {
       "type": "InsertionContains",
-      "column": "nucleotideInsertions",
       "position": 22204,
       "value": "CAG.*AA"
     }

diff --git a/endToEndTests/test/queries/insertionContains_not_exact3.json b/endToEndTests/test/queries/insertionContains_not_exact3.json
@@ -6,7 +6,6 @@
     },
     "filterExpression": {
       "type": "InsertionContains",
-      "column": "nucleotideInsertions",
       "position": 25701,
       "value": "TCAG.*AA"
     }

diff --git a/endToEndTests/test/queries/insertionContains_not_exact4.json b/endToEndTests/test/queries/insertionContains_not_exact4.json
@@ -6,7 +6,6 @@
     },
     "filterExpression": {
       "type": "InsertionContains",
-      "column": "nucleotideInsertions",
       "position": 25701,
       "value": "CC.*"
     }

diff --git a/endToEndTests/test/queries/insertionsAction.json b/endToEndTests/test/queries/insertionsAction.json
@@ -3,7 +3,6 @@
   "query": {
     "action": {
       "type": "Insertions",
-      "column": "nucleotideInsertions",
       "orderByFields": ["insertion"]
     },
     "filterExpression": {

diff --git a/endToEndTests/test/queries/insertionsActionAndFilter.json b/endToEndTests/test/queries/insertionsActionAndFilter.json
@@ -3,11 +3,10 @@
   "query": {
     "action": {
       "type": "Insertions",
-      "column": "nucleotideInsertions"
+      "sequenceName": "main"
     },
     "filterExpression": {
       "type": "InsertionContains",
-      "column": "nucleotideInsertions",
       "position": 22339,
       "value": ".*C.*G.*"
     }

diff --git a/include/silo/common/data_version.h b/include/silo/common/data_version.h
@@ -30,7 +30,7 @@ class DataVersion {
       uint32_t value;
    };
 
-   static constexpr SerializationVersion CURRENT_SILO_SERIALIZATION_VERSION{0};
+   static constexpr SerializationVersion CURRENT_SILO_SERIALIZATION_VERSION{1};
 
   private:
    Timestamp timestamp;

diff --git a/include/silo/common/table_reader.h b/include/silo/common/table_reader.h
@@ -0,0 +1,53 @@
+#pragma once
+
+#include <filesystem>
+#include <fstream>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <duckdb.hpp>
+
+namespace silo {
+
+struct ColumnFunction {
+   std::string column_name;
+   std::function<void(size_t, const duckdb::Value&)> function;
+};
+
+class TableReader {
+  private:
+   duckdb::Connection& connection;
+   std::string table_name;
+   std::string key_column;
+   std::vector<ColumnFunction> column_functions;
+   std::string where_clause;
+   std::string order_by_clause;
+   std::unique_ptr<duckdb::MaterializedQueryResult> query_result;
+   std::unique_ptr<duckdb::DataChunk> current_chunk;
+   size_t current_row;
+
+   std::optional<std::string> nextKey();
+
+   std::string getTableQuery();
+
+   void advanceRow();
+
+  public:
+   explicit TableReader(
+      duckdb::Connection& connection,
+      std::string_view table_name,
+      std::string_view key_column,
+      std::vector<ColumnFunction> column_functions,
+      std::string_view where_clause,
+      std::string_view order_by_clause
+   );
+
+   void read();
+
+   void loadTable();
+};
+}  // namespace silo
diff --git a/include/silo/config/database_config.h b/include/silo/config/database_config.h
@@ -10,18 +10,8 @@
 
 namespace silo::config {
 
-enum class ValueType { STRING, PANGOLINEAGE, DATE, BOOL, INT, FLOAT, NUC_INSERTION, AA_INSERTION };
-enum class ColumnType {
-   STRING,
-   INDEXED_STRING,
-   INDEXED_PANGOLINEAGE,
-   DATE,
-   BOOL,
-   INT,
-   FLOAT,
-   NUC_INSERTION,
-   AA_INSERTION
-};
+enum class ValueType { STRING, PANGOLINEAGE, DATE, BOOL, INT, FLOAT };
+enum class ColumnType { STRING, INDEXED_STRING, INDEXED_PANGOLINEAGE, DATE, BOOL, INT, FLOAT };
 
 ValueType toDatabaseValueType(std::string_view type);
 

diff --git a/include/silo/config/preprocessing_config.h b/include/silo/config/preprocessing_config.h
@@ -23,12 +23,15 @@ const std::string REFERENCE_GENOME_FILENAME_OPTION = "referenceGenomeFilename";
 const std::string NUCLEOTIDE_SEQUENCE_PREFIX_OPTION = "nucleotideSequencePrefix";
 const std::string UNALIGNED_NUCLEOTIDE_SEQUENCE_PREFIX_OPTION = "unalignedNucleotideSequencePrefix";
 const std::string GENE_PREFIX_OPTION = "genePrefix";
+const std::string NUCLEOTIDE_INSERTIONS_OPTION = "nucleotideInsertionsFilename";
+const std::string AMINO_ACID_INSERTIONS_OPTION = "aminoAcidInsertionsFilename";
 
 const std::string DEFAULT_OUTPUT_DIRECTORY = "./output/";
 
 class PreprocessingConfig {
    friend class fmt::formatter<silo::config::PreprocessingConfig>;
 
+  public:
    std::filesystem::path input_directory = "./";
    std::filesystem::path output_directory = DEFAULT_OUTPUT_DIRECTORY;
    std::filesystem::path intermediate_results_directory = "./temp/";
@@ -40,9 +43,8 @@ class PreprocessingConfig {
    std::string nucleotide_sequence_prefix = "nuc_";
    std::string unaligned_nucleotide_sequence_prefix = "unaligned_";
    std::string gene_prefix = "gene_";
-
-  public:
-   explicit PreprocessingConfig();
+   std::string nuc_insertions_filename = "nuc_insertions.tsv";
+   std::string aa_insertions_filename = "aa_insertions.tsv";
 
    void validate() const;
 
@@ -67,6 +69,10 @@ class PreprocessingConfig {
 
    [[nodiscard]] std::filesystem::path getGeneFilenameNoExtension(std::string_view gene_name) const;
 
+   [[nodiscard]] std::filesystem::path getNucleotideInsertionsFilename() const;
+
+   [[nodiscard]] std::filesystem::path getAminoAcidInsertionsFilename() const;
+
    void overwrite(const silo::config::AbstractConfig& config_reader);
 };
 

diff --git a/include/silo/database.h b/include/silo/database.h
@@ -91,7 +91,6 @@ class Database {
    void initializeAASequences(
       const std::map<std::string, std::vector<AminoAcid::Symbol>>& reference_sequences
    );
-   void finalizeInsertionIndexes();
 
    template <typename SymbolType>
    static BitmapSizePerSymbol calculateBitmapSizePerSymbol(

diff --git a/include/silo/preprocessing/preprocessor.h b/include/silo/preprocessing/preprocessor.h
@@ -38,6 +38,12 @@ class Preprocessor {
    void buildPartitioningTableByColumn(const std::string& partition_by_field);
    void buildEmptyPartitioning();
 
+   void createInsertionsTableFromFile(
+      const std::map<std::string, std::string>& expected_sequences,
+      const std::filesystem::path& insertion_file,
+      const std::string& table_name
+   );
+
    void createPartitionedSequenceTablesFromNdjson(const std::filesystem::path& file_name);
 
    void createAlignedPartitionedSequenceViews(

diff --git a/include/silo/query_engine/actions/insertions.h b/include/silo/query_engine/actions/insertions.h
@@ -20,14 +20,10 @@
 namespace silo {
 class Database;
 namespace storage {
-namespace column {
-template <typename SymbolType>
-class InsertionColumnPartition;
 namespace insertion {
 template <typename SymbolType>
 class InsertionIndex;
 }  // namespace insertion
-}  // namespace column
 }  // namespace storage
 }  // namespace silo
 
@@ -41,27 +37,19 @@ class InsertionAggregation : public Action {
    static constexpr std::string_view SEQUENCE_FIELD_NAME = "sequenceName";
    static constexpr std::string_view COUNT_FIELD_NAME = "count";
 
-   std::vector<std::string> column_names;
    std::vector<std::string> sequence_names;
 
    struct PrefilteredBitmaps {
       std::vector<std::pair<
          const OperatorResult&,
-         const silo::storage::column::insertion::InsertionIndex<SymbolType>&>>
+         const silo::storage::insertion::InsertionIndex<SymbolType>&>>
          bitmaps;
       std::vector<std::pair<
          const OperatorResult&,
-         const silo::storage::column::insertion::InsertionIndex<SymbolType>&>>
+         const silo::storage::insertion::InsertionIndex<SymbolType>&>>
          full_bitmaps;
    };
 
-   void addAllColumnIndexesToPreFilteredBitmaps(
-      const silo::storage::column::InsertionColumnPartition<SymbolType>& column,
-      const OperatorResult& filter,
-      std::unordered_map<std::string, InsertionAggregation<SymbolType>::PrefilteredBitmaps>&
-         bitmaps_to_evaluate
-   ) const;
-
    void addAggregatedInsertionsToInsertionCounts(
       std::vector<QueryResultEntry>& output,
       const std::string& sequence_name,
@@ -76,10 +64,7 @@ class InsertionAggregation : public Action {
    ) const;
 
   public:
-   InsertionAggregation(
-      std::vector<std::string>&& column,
-      std::vector<std::string>&& sequence_names
-   );
+   InsertionAggregation(std::vector<std::string>&& sequence_names);
 
    void validateOrderByFields(const Database& database) const override;
 

diff --git a/include/silo/query_engine/filter_expressions/insertion_contains.h b/include/silo/query_engine/filter_expressions/insertion_contains.h
@@ -23,14 +23,12 @@ namespace silo::query_engine::filter_expressions {
 template <typename SymbolType>
 class InsertionContains : public Expression {
   private:
-   std::vector<std::string> column_names;
    std::optional<std::string> sequence_name;
    uint32_t position_idx;
    std::string value;
 
   public:
    explicit InsertionContains(
-      std::vector<std::string>&& column_names,
       std::optional<std::string> sequence_name,
       uint32_t position_idx,
       std::string value