fix: insertions being added at wrong index for large files

GenSpectrum · Jun 10, 2024 · db383e0 · db383e0
1 parent d8d171b
commit db383e0
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/include/silo/common/table_reader.h b/include/silo/common/table_reader.h
@@ -29,6 +29,7 @@ class TableReader {
    std::unique_ptr<duckdb::MaterializedQueryResult> query_result;
    std::unique_ptr<duckdb::DataChunk> current_chunk;
    size_t current_row;
+   size_t current_row_in_chunk;
 
    std::optional<std::string> nextKey();
 

diff --git a/src/silo/common/table_reader.cpp b/src/silo/common/table_reader.cpp
@@ -31,7 +31,7 @@ std::optional<std::string> silo::TableReader::nextKey() {
       return std::nullopt;
    }
 
-   return current_chunk->GetValue(0, current_row).GetValue<std::string>();
+   return current_chunk->GetValue(0, current_row_in_chunk).GetValue<std::string>();
 }
 
 void silo::TableReader::read() {
@@ -40,7 +40,7 @@ void silo::TableReader::read() {
    while (nextKey()) {
       for (size_t column_idx = 0; column_idx < column_functions.size(); column_idx++) {
          column_functions.at(column_idx)
-            .function(current_row, current_chunk->GetValue(column_idx + 1, current_row));
+            .function(current_row, current_chunk->GetValue(column_idx + 1, current_row_in_chunk));
       }
       advanceRow();
    }
@@ -81,6 +81,7 @@ void silo::TableReader::loadTable() {
    }
    current_chunk = query_result->Fetch();
    current_row = 0;
+   current_row_in_chunk = 0;
 
    while (current_chunk && current_chunk->size() == 0) {
       current_chunk = query_result->Fetch();
@@ -89,8 +90,9 @@ void silo::TableReader::loadTable() {
 
 void silo::TableReader::advanceRow() {
    current_row++;
-   if (current_row == current_chunk->size()) {
-      current_row = 0;
+   current_row_in_chunk++;
+   if (current_row_in_chunk == current_chunk->size()) {
+      current_row_in_chunk = 0;
       current_chunk = query_result->Fetch();
       while (current_chunk && current_chunk->size() == 0) {
          current_chunk = query_result->Fetch();