refactor: RAG bm25 (#939)

sigoden · Oct 20, 2024 · ac89b7a · ac89b7a
1 parent 7b20ab0
commit ac89b7a
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 10 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -64,7 +64,7 @@ sys-locale = "0.3.1"
 html_to_markdown = "0.1.0"
 rust-embed = "8.5.0"
 os_info = { version = "3.8.2", default-features = false }
-bm25 = { version = "0.3.1", features = ["language_detection", "parallelism"] }
+bm25 = { version = "0.3.1", features = ["parallelism"] }
 
 [dependencies.reqwest]
 version = "0.12.0"

diff --git a/src/rag/mod.rs b/src/rag/mod.rs
@@ -10,7 +10,7 @@ mod serde_vectors;
 mod splitter;
 
 use anyhow::{anyhow, bail, Context, Result};
-use bm25::{LanguageMode, SearchEngine, SearchEngineBuilder};
+use bm25::{Language, SearchEngine, SearchEngineBuilder};
 use hnsw_rs::prelude::*;
 use indexmap::{IndexMap, IndexSet};
 use inquire::{required, validator::Validation, Confirm, Select, Text};
@@ -724,7 +724,7 @@ impl RagData {
                 documents.push(bm25::Document::new(id, &document.page_content))
             }
         }
-        SearchEngineBuilder::<DocumentId>::with_documents(LanguageMode::Detect, documents)
+        SearchEngineBuilder::<DocumentId>::with_documents(Language::English, documents)
             .k1(1.5)
             .b(0.75)
             .build()