Merge pull request #73 from KruxAI/qdrant-weaviate

VectorDB Support: Qdrant & Weaviate

requirements.txt

@@ -83,4 +83,7 @@ rerankers
 rerankers[flashrank]
 rerankers[gpt]
 gensim
-ragatouille
+ragatouille
+langchain-qdrant
+fastembed
+langchain-weaviate

setup.py

@@ -4,7 +4,7 @@
     long_description = fh.read()
 setup(
     name='ragbuilder',
-    version='0.0.19',
+    version='0.0.21',
     author='Ashwin Aravind, Aravind Parameswaran',
     author_email='[email protected], [email protected]',
     description='RagBuilder is a toolkit designed to help you create optimal Production-ready Retrieval-Augmented Generation (RAG) pipeline for your data',
@@ -95,7 +95,10 @@
         'rerankers[flashrank]',
         'rerankers[gpt]',
         'gensim',
-        'ragatouille'
+        'ragatouille',
+        'langchain-qdrant',
+        'fastembed',
+        'langchain-weaviate'
         # 'llama-index',
         # 'llama-index-vector-stores-chroma',
         # 'llama-index-readers-web',

src/ragbuilder/executor.py

@@ -70,6 +70,9 @@
     SelfQueryRetriever,
     TimeWeightedVectorStoreRetriever
 )
+import weaviate
+from langchain_weaviate.vectorstores import WeaviateVectorStore
+from langchain_qdrant import QdrantVectorStore, RetrievalMode, FastEmbedSparse
 from rerankers import Reranker
 from langchain_core.documents import Document
 from langchain.retrievers.document_compressors import *

src/ragbuilder/langchain_module/vectordb/vectordb.py

@@ -48,6 +48,18 @@ def getVectorDB(db_type,embedding_model):
         code_string= f"""c = Milvus.from_documents(splits,embedding,collection_name='{index_name}',connection_args={{"uri": MILVUS_CONNECTION_STRING}},)"""
         print(code_string)
         import_string = f"""from langchain_milvus import Milvus"""
+    elif db_type == "qdrantDB":
+        logger.info("qdrant DB Loaded")
+        code_string= f"""sparse_embeddings = FastEmbedSparse(model_name="Qdrant/bm25")
+c = QdrantVectorStore.from_documents(splits,embedding,location=":memory:",collection_name='{index_name}',retrieval_mode=RetrievalMode.HYBRID,sparse_embedding=sparse_embeddings)"""
+        import_string = f"""from langchain_qdrant import QdrantVectorStore, RetrievalMode, FastEmbedSparse
+from qdrant_client import QdrantClient"""
+    elif db_type == "weaviateDB":
+        logger.info("weaviate DB Loaded")
+        code_string= f"""weaviate_client = weaviate.connect_to_local()
+c = WeaviateVectorStore.from_documents(docs, embedding, client=weaviate_client)"""
+        import_string = f"""import weaviate
+from langchain_weaviate.vectorstores import WeaviateVectorStore"""
     elif db_type == "singleStoreDB":
         index_name = "testindex_ragbuilder"
         logger.info("Singlestore DB Loaded")

src/ragbuilder/templates/layouts.html

@@ -296,6 +296,14 @@ <h6 class="h6">Vector DB</h6>
                                             <input type="radio" class="form-check-input" name="vectorDB" id="singleStoreDB" unchecked>
                                             <label class="form-check-label" for="singleStoreDB">Singlestore DB</label>
                                         </div>
+                                        <div class="form-group form-check mx-3">
+                                            <input type="radio" class="form-check-input" name="vectorDB" id="qdrantDB" unchecked>
+                                            <label class="form-check-label" for="qdrantDB">Qdrant DB</label>
+                                        </div>
+                                        <div class="form-group form-check mx-3">
+                                            <input type="radio" class="form-check-input" name="vectorDB" id="weaviateDB" unchecked>
+                                            <label class="form-check-label" for="weaviateDB">Weaviate DB</label>
+                                        </div>
                                 </div>
 
                                 <div class="col-md-6 mt-5">