Merge pull request #127 from fractalego/indexing-with-large-corpus

Indexing with large corpus

requirements.txt

@@ -1,7 +1,6 @@
 flask[async]==3.0.3
 flask-cors==4.0.1
 nltk==3.8.1
-gensim==4.3.3
 sklearn==0.0
 python-Levenshtein==0.25.1
 fuzzywuzzy==0.18.0

setup.py

@@ -49,7 +49,6 @@
         "flask[async]==3.0.3",
         "flask-cors==4.0.1",
         "nltk==3.8.1",
-        "gensim==4.3.3",
         "sklearn==0.0",
         "python-Levenshtein==0.25.1",
         "fuzzywuzzy==0.18.0",

todo.txt

@@ -1,4 +1,5 @@
 * why do I need to re-initialise the retrievers after unpickling the knowledge?
+   - maybe you should save the retrievers in the knowledge object separately?
 
 /* knowledge cache does not cache the rules or facts
 

wafl/knowledge/indexing_implementation.py

@@ -45,13 +45,12 @@ async def load_knowledge(config, logger=None):
         if knowledge.hash == hash(rules_txt) and os.path.getmtime(
             cache_filename
         ) > os.path.getmtime(index_filename):
-            await knowledge.initialize_retrievers()
             return knowledge
 
     knowledge = SingleFileKnowledge(config, rules_txt, logger=logger)
     knowledge = await _add_indices_to_knowledge(knowledge, index_txt)
-    joblib.dump(knowledge, config.get_value("cache_filename"))
     await knowledge.initialize_retrievers()
+    joblib.dump(knowledge, config.get_value("cache_filename"))
     return knowledge
 
 

wafl/retriever/dense_retriever.py

@@ -2,7 +2,6 @@
 import numpy as np
 
 from typing import List, Tuple
-from gensim.models import KeyedVectors
 from wafl.connectors.factories.sentence_embedder_connector_factory import (
     SentenceEmbedderConnectorFactory,
 )
@@ -18,18 +17,24 @@ def __init__(self, model_name, config):
         self._connector = SentenceEmbedderConnectorFactory.get_connector(
             model_name, config
         )
-        self._embeddings_model = KeyedVectors(384)
+        self._matrix = np.zeros((0, 384))
+        self._indices = []
 
     async def add_text_and_index(self, text: str, index: str):
         embeddings = await self._get_embeddings_from_text(text)
-        self._embeddings_model.add_vectors([index], [embeddings])
-        self._embeddings_model.fill_norms(force=True)
+        self._matrix = np.vstack([self._matrix, embeddings])
+        self._indices.append(index)
 
     async def get_indices_and_scores_from_text(
-        self, text: str
+        self, text: str, topn: int = 5
     ) -> List[Tuple[str, float]]:
         embeddings = await self._get_embeddings_from_text(text)
-        return self._embeddings_model.similar_by_vector(embeddings, topn=5)
+        scores = np.dot(self._matrix, embeddings) / (
+            np.linalg.norm(self._matrix, axis=1) * np.linalg.norm(embeddings)
+        )
+        indices_and_scores = list(zip(self._indices, scores))
+        indices_and_scores.sort(key=lambda x: x[1], reverse=True)
+        return indices_and_scores[:topn]
 
     async def _get_embeddings_from_text(self, text: str) -> "numpy.array":
         return (await self._connector.predict(text))["embedding"]

wafl/variables.py

@@ -1,6 +1,6 @@
 def get_variables():
     return {
-        "version": "0.1.1",
+        "version": "0.1.2",
     }