From 222f17bfad536e2bc79fdf9316f05e9345fb1ad3 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Mon, 11 Mar 2024 09:17:11 +0330 Subject: [PATCH 1/8] feat: Added update based on new lib changes! --- bot/retrievers/forum_summary_retriever.py | 4 ++-- bot/retrievers/retrieve_similar_nodes.py | 9 +++++---- bot/retrievers/summary_retriever_base.py | 8 ++++---- requirements.txt | 3 +-- .../integration/test_retrieve_similar_nodes.py | 2 +- tests/unit/test_discord_summary_retriever.py | 11 ++++------- .../test_level_based_platform_query_engine.py | 2 +- ...sed_platform_query_engine_prepare_context.py | 2 +- tests/unit/test_level_based_platform_util.py | 2 +- tests/unit/test_prepare_discord_query_engine.py | 2 +- .../unit/test_prepare_discourse_query_engine.py | 2 +- tests/unit/test_summary_retriever_base.py | 11 +++++------ .../level_based_platform_query_engine.py | 17 ++++++++++------- .../query_engine/level_based_platforms_util.py | 2 +- .../prepare_discord_query_engine.py | 2 +- .../prepare_discourse_query_engine.py | 2 +- 16 files changed, 40 insertions(+), 41 deletions(-) diff --git a/bot/retrievers/forum_summary_retriever.py b/bot/retrievers/forum_summary_retriever.py index 7cb2982..1df52e7 100644 --- a/bot/retrievers/forum_summary_retriever.py +++ b/bot/retrievers/forum_summary_retriever.py @@ -1,6 +1,6 @@ from bot.retrievers.summary_retriever_base import BaseSummarySearch -from llama_index.embeddings import BaseEmbedding -from llama_index.schema import NodeWithScore +from llama_index.core.embeddings import BaseEmbedding +from llama_index.core.schema import NodeWithScore from tc_hivemind_backend.embeddings.cohere import CohereEmbedding diff --git a/bot/retrievers/retrieve_similar_nodes.py b/bot/retrievers/retrieve_similar_nodes.py index c5f5f59..830d874 100644 --- a/bot/retrievers/retrieve_similar_nodes.py +++ b/bot/retrievers/retrieve_similar_nodes.py @@ -1,10 +1,11 @@ from datetime import datetime, timedelta from dateutil import parser -from llama_index.embeddings import BaseEmbedding -from llama_index.schema import NodeWithScore -from llama_index.vector_stores import PGVectorStore, VectorStoreQueryResult -from llama_index.vector_stores.postgres import DBEmbeddingRow +from llama_index.core.embeddings import BaseEmbedding +from llama_index.core.schema import NodeWithScore +from llama_index.vector_stores.postgres import PGVectorStore +from llama_index.core.vector_stores.types import VectorStoreQueryResult +from llama_index.vector_stores.postgres.base import DBEmbeddingRow from sqlalchemy import Date, and_, cast, null, or_, select, text from tc_hivemind_backend.embeddings.cohere import CohereEmbedding diff --git a/bot/retrievers/summary_retriever_base.py b/bot/retrievers/summary_retriever_base.py index d501c69..7bc5dc9 100644 --- a/bot/retrievers/summary_retriever_base.py +++ b/bot/retrievers/summary_retriever_base.py @@ -1,7 +1,7 @@ -from llama_index import VectorStoreIndex -from llama_index.embeddings import BaseEmbedding -from llama_index.indices.query.schema import QueryBundle -from llama_index.schema import NodeWithScore +from llama_index.core import VectorStoreIndex +from llama_index.core.embeddings import BaseEmbedding +from llama_index.core.schema import QueryBundle +from llama_index.core.schema import NodeWithScore from tc_hivemind_backend.embeddings.cohere import CohereEmbedding from tc_hivemind_backend.pg_vector_access import PGVectorAccess diff --git a/requirements.txt b/requirements.txt index 3a3c23c..1d1b227 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ numpy -llama-index==0.9.48 pymongo python-dotenv pgvector @@ -16,7 +15,7 @@ neo4j>=5.14.1, <6.0.0 coverage>=7.3.3, <8.0.0 pytest>=7.4.3, <8.0.0 python-dotenv==1.0.0 -tc-hivemind-backend==1.1.2 +tc-hivemind-backend==1.1.4 celery>=5.3.6, <6.0.0 guidance tc-messageBroker==1.6.6 diff --git a/tests/integration/test_retrieve_similar_nodes.py b/tests/integration/test_retrieve_similar_nodes.py index 5ee611a..0ac5348 100644 --- a/tests/integration/test_retrieve_similar_nodes.py +++ b/tests/integration/test_retrieve_similar_nodes.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock from bot.retrievers.retrieve_similar_nodes import RetrieveSimilarNodes -from llama_index.schema import TextNode +from llama_index.core.schema import TextNode class TestRetrieveSimilarNodes(TestCase): diff --git a/tests/unit/test_discord_summary_retriever.py b/tests/unit/test_discord_summary_retriever.py index 638b8b4..92c435b 100644 --- a/tests/unit/test_discord_summary_retriever.py +++ b/tests/unit/test_discord_summary_retriever.py @@ -5,7 +5,7 @@ from bot.retrievers.forum_summary_retriever import ForumBasedSummaryRetriever from dateutil import parser -from llama_index import Document, MockEmbedding, ServiceContext, VectorStoreIndex +from llama_index.core import Document, MockEmbedding, Settings, VectorStoreIndex class TestDiscordSummaryRetriever(TestCase): @@ -31,13 +31,10 @@ def test_initialize_class(self): mock_embedding_model = partial(MockEmbedding, embed_dim=1024) - service_context = ServiceContext.from_defaults( - llm=None, embed_model=mock_embedding_model() - ) + Settings.llm = None + Settings.embed_model = mock_embedding_model() ForumBasedSummaryRetriever._setup_index.return_value = ( - VectorStoreIndex.from_documents( - documents=[doc], service_context=service_context - ) + VectorStoreIndex.from_documents(documents=[doc]) ) base_summary_search = ForumBasedSummaryRetriever( diff --git a/tests/unit/test_level_based_platform_query_engine.py b/tests/unit/test_level_based_platform_query_engine.py index 15d1243..4bae75d 100644 --- a/tests/unit/test_level_based_platform_query_engine.py +++ b/tests/unit/test_level_based_platform_query_engine.py @@ -4,7 +4,7 @@ from bot.retrievers.forum_summary_retriever import ForumBasedSummaryRetriever from bot.retrievers.retrieve_similar_nodes import RetrieveSimilarNodes -from llama_index.schema import NodeWithScore, TextNode +from llama_index.core.schema import NodeWithScore, TextNode from sqlalchemy.exc import OperationalError from utils.query_engine.level_based_platform_query_engine import ( LevelBasedPlatformQueryEngine, diff --git a/tests/unit/test_level_based_platform_query_engine_prepare_context.py b/tests/unit/test_level_based_platform_query_engine_prepare_context.py index 3fa415e..d5254b6 100644 --- a/tests/unit/test_level_based_platform_query_engine_prepare_context.py +++ b/tests/unit/test_level_based_platform_query_engine_prepare_context.py @@ -3,7 +3,7 @@ from unittest.mock import patch from bot.retrievers.retrieve_similar_nodes import RetrieveSimilarNodes -from llama_index.schema import NodeWithScore, TextNode +from llama_index.core.schema import NodeWithScore, TextNode from utils.query_engine.level_based_platform_query_engine import ( LevelBasedPlatformQueryEngine, ) diff --git a/tests/unit/test_level_based_platform_util.py b/tests/unit/test_level_based_platform_util.py index cc7d721..02eced5 100644 --- a/tests/unit/test_level_based_platform_util.py +++ b/tests/unit/test_level_based_platform_util.py @@ -1,6 +1,6 @@ import unittest -from llama_index.schema import NodeWithScore, TextNode +from llama_index.core.schema import NodeWithScore, TextNode from utils.query_engine.level_based_platforms_util import LevelBasedPlatformUtils diff --git a/tests/unit/test_prepare_discord_query_engine.py b/tests/unit/test_prepare_discord_query_engine.py index 22ef04e..9959815 100644 --- a/tests/unit/test_prepare_discord_query_engine.py +++ b/tests/unit/test_prepare_discord_query_engine.py @@ -1,7 +1,7 @@ import os import unittest -from llama_index.core.base_query_engine import BaseQueryEngine +from llama_index.core.base.base_query_engine import BaseQueryEngine from utils.query_engine.prepare_discord_query_engine import prepare_discord_engine diff --git a/tests/unit/test_prepare_discourse_query_engine.py b/tests/unit/test_prepare_discourse_query_engine.py index 6f47f18..25d983e 100644 --- a/tests/unit/test_prepare_discourse_query_engine.py +++ b/tests/unit/test_prepare_discourse_query_engine.py @@ -1,7 +1,7 @@ import os import unittest -from llama_index.core.base_query_engine import BaseQueryEngine +from llama_index.core.base.base_query_engine import BaseQueryEngine from utils.query_engine.prepare_discourse_query_engine import prepare_discourse_engine diff --git a/tests/unit/test_summary_retriever_base.py b/tests/unit/test_summary_retriever_base.py index 9422571..7f53343 100644 --- a/tests/unit/test_summary_retriever_base.py +++ b/tests/unit/test_summary_retriever_base.py @@ -3,8 +3,8 @@ from unittest.mock import MagicMock from bot.retrievers.summary_retriever_base import BaseSummarySearch -from llama_index import Document, MockEmbedding, ServiceContext, VectorStoreIndex -from llama_index.schema import NodeWithScore +from llama_index.core import Document, MockEmbedding, Settings, VectorStoreIndex +from llama_index.core.schema import NodeWithScore class TestSummaryRetrieverBase(TestCase): @@ -13,11 +13,10 @@ def test_initialize_class(self): doc = Document(text="SAMPLESAMPLESAMPLE") mock_embedding_model = partial(MockEmbedding, embed_dim=1024) - service_context = ServiceContext.from_defaults( - llm=None, embed_model=mock_embedding_model() - ) + Settings.llm = None + Settings.embed_model = mock_embedding_model() BaseSummarySearch._setup_index.return_value = VectorStoreIndex.from_documents( - documents=[doc], service_context=service_context + documents=[doc] ) base_summary_search = BaseSummarySearch( diff --git a/utils/query_engine/level_based_platform_query_engine.py b/utils/query_engine/level_based_platform_query_engine.py index 5cb4e9e..4415ff9 100644 --- a/utils/query_engine/level_based_platform_query_engine.py +++ b/utils/query_engine/level_based_platform_query_engine.py @@ -3,13 +3,16 @@ from bot.retrievers.forum_summary_retriever import ForumBasedSummaryRetriever from bot.retrievers.retrieve_similar_nodes import RetrieveSimilarNodes from bot.retrievers.utils.load_hyperparams import load_hyperparams -from llama_index import VectorStoreIndex -from llama_index.llms import OpenAI -from llama_index.prompts import PromptTemplate -from llama_index.query_engine import CustomQueryEngine -from llama_index.response_synthesizers import BaseSynthesizer, get_response_synthesizer -from llama_index.retrievers import BaseRetriever -from llama_index.schema import NodeWithScore +from llama_index.core import VectorStoreIndex +from llama_index.llms.openai import OpenAI +from llama_index.core.prompts import PromptTemplate +from llama_index.core.query_engine import CustomQueryEngine +from llama_index.core.response_synthesizers import ( + BaseSynthesizer, + get_response_synthesizer, +) +from llama_index.core.retrievers import BaseRetriever +from llama_index.core.schema import NodeWithScore from tc_hivemind_backend.embeddings.cohere import CohereEmbedding from tc_hivemind_backend.pg_vector_access import PGVectorAccess from utils.query_engine.level_based_platforms_util import LevelBasedPlatformUtils diff --git a/utils/query_engine/level_based_platforms_util.py b/utils/query_engine/level_based_platforms_util.py index eb4286c..5574675 100644 --- a/utils/query_engine/level_based_platforms_util.py +++ b/utils/query_engine/level_based_platforms_util.py @@ -1,7 +1,7 @@ import logging from dateutil import parser -from llama_index.schema import NodeWithScore +from llama_index.core.schema import NodeWithScore class LevelBasedPlatformUtils: diff --git a/utils/query_engine/prepare_discord_query_engine.py b/utils/query_engine/prepare_discord_query_engine.py index 553dde3..8f6b39c 100644 --- a/utils/query_engine/prepare_discord_query_engine.py +++ b/utils/query_engine/prepare_discord_query_engine.py @@ -1,4 +1,4 @@ -from llama_index.query_engine import BaseQueryEngine +from llama_index.core.query_engine import BaseQueryEngine from .level_based_platform_query_engine import LevelBasedPlatformQueryEngine diff --git a/utils/query_engine/prepare_discourse_query_engine.py b/utils/query_engine/prepare_discourse_query_engine.py index 3c9bbd7..d724a9b 100644 --- a/utils/query_engine/prepare_discourse_query_engine.py +++ b/utils/query_engine/prepare_discourse_query_engine.py @@ -1,4 +1,4 @@ -from llama_index.query_engine import BaseQueryEngine +from llama_index.core.query_engine import BaseQueryEngine from .level_based_platform_query_engine import LevelBasedPlatformQueryEngine From 82bbeabcaac6e90222d07b0dc39037aa11c03d48 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Mon, 11 Mar 2024 09:24:06 +0330 Subject: [PATCH 2/8] fix: isort linter issues! --- bot/retrievers/retrieve_similar_nodes.py | 2 +- bot/retrievers/summary_retriever_base.py | 3 +-- utils/query_engine/level_based_platform_query_engine.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bot/retrievers/retrieve_similar_nodes.py b/bot/retrievers/retrieve_similar_nodes.py index 830d874..c3da93b 100644 --- a/bot/retrievers/retrieve_similar_nodes.py +++ b/bot/retrievers/retrieve_similar_nodes.py @@ -3,8 +3,8 @@ from dateutil import parser from llama_index.core.embeddings import BaseEmbedding from llama_index.core.schema import NodeWithScore -from llama_index.vector_stores.postgres import PGVectorStore from llama_index.core.vector_stores.types import VectorStoreQueryResult +from llama_index.vector_stores.postgres import PGVectorStore from llama_index.vector_stores.postgres.base import DBEmbeddingRow from sqlalchemy import Date, and_, cast, null, or_, select, text from tc_hivemind_backend.embeddings.cohere import CohereEmbedding diff --git a/bot/retrievers/summary_retriever_base.py b/bot/retrievers/summary_retriever_base.py index 7bc5dc9..ae28e04 100644 --- a/bot/retrievers/summary_retriever_base.py +++ b/bot/retrievers/summary_retriever_base.py @@ -1,7 +1,6 @@ from llama_index.core import VectorStoreIndex from llama_index.core.embeddings import BaseEmbedding -from llama_index.core.schema import QueryBundle -from llama_index.core.schema import NodeWithScore +from llama_index.core.schema import NodeWithScore, QueryBundle from tc_hivemind_backend.embeddings.cohere import CohereEmbedding from tc_hivemind_backend.pg_vector_access import PGVectorAccess diff --git a/utils/query_engine/level_based_platform_query_engine.py b/utils/query_engine/level_based_platform_query_engine.py index 4415ff9..dc64cf9 100644 --- a/utils/query_engine/level_based_platform_query_engine.py +++ b/utils/query_engine/level_based_platform_query_engine.py @@ -4,7 +4,6 @@ from bot.retrievers.retrieve_similar_nodes import RetrieveSimilarNodes from bot.retrievers.utils.load_hyperparams import load_hyperparams from llama_index.core import VectorStoreIndex -from llama_index.llms.openai import OpenAI from llama_index.core.prompts import PromptTemplate from llama_index.core.query_engine import CustomQueryEngine from llama_index.core.response_synthesizers import ( @@ -13,6 +12,7 @@ ) from llama_index.core.retrievers import BaseRetriever from llama_index.core.schema import NodeWithScore +from llama_index.llms.openai import OpenAI from tc_hivemind_backend.embeddings.cohere import CohereEmbedding from tc_hivemind_backend.pg_vector_access import PGVectorAccess from utils.query_engine.level_based_platforms_util import LevelBasedPlatformUtils From 038f49ea872fc1e934cc057176fbba5625d8ca80 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Tue, 12 Mar 2024 13:42:54 +0330 Subject: [PATCH 3/8] feat: increase backend lib dependency version! --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1d1b227..04aa0cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ neo4j>=5.14.1, <6.0.0 coverage>=7.3.3, <8.0.0 pytest>=7.4.3, <8.0.0 python-dotenv==1.0.0 -tc-hivemind-backend==1.1.4 +tc-hivemind-backend==1.1.5 celery>=5.3.6, <6.0.0 guidance tc-messageBroker==1.6.6 From aff751d307defd259b79765854149c35b7d4fd44 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Tue, 12 Mar 2024 14:05:14 +0330 Subject: [PATCH 4/8] feat: command update to raise error! command update to raise error if some test cases failed! --- docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index fa3c438..47a5606 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -python3 -m coverage run --omit=tests/* -m pytest . +python3 -m coverage run --omit=tests/* -m pytest . && echo "Tests Passed" || exit 1 python3 -m coverage lcov -o coverage/lcov.info From 9a260ea8ee103de9672c16c75b8263ddfb711b2d Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 13 Mar 2024 08:45:14 +0330 Subject: [PATCH 5/8] feat: completely aligning updates! - Fixing missing not updated parts. --- discord_query.py | 4 ++-- requirements.txt | 1 + subquery.py | 22 +++++++++++++--------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/discord_query.py b/discord_query.py index 4333c5c..be7037b 100644 --- a/discord_query.py +++ b/discord_query.py @@ -1,5 +1,5 @@ -from llama_index import QueryBundle -from llama_index.schema import NodeWithScore +from llama_index.core import QueryBundle +from llama_index.core.schema import NodeWithScore from tc_hivemind_backend.embeddings.cohere import CohereEmbedding from utils.query_engine.prepare_discord_query_engine import ( prepare_discord_engine_auto_filter, diff --git a/requirements.txt b/requirements.txt index 04aa0cd..932cd8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ coverage>=7.3.3, <8.0.0 pytest>=7.4.3, <8.0.0 python-dotenv==1.0.0 tc-hivemind-backend==1.1.5 +llama-index-question-gen-guidance==0.1.2 celery>=5.3.6, <6.0.0 guidance tc-messageBroker==1.6.6 diff --git a/subquery.py b/subquery.py index 6270823..c7c829c 100644 --- a/subquery.py +++ b/subquery.py @@ -1,10 +1,12 @@ from guidance.models import OpenAIChat -from llama_index import QueryBundle, ServiceContext -from llama_index.core.base_query_engine import BaseQueryEngine -from llama_index.query_engine import SubQuestionQueryEngine -from llama_index.question_gen.guidance_generator import GuidanceQuestionGenerator -from llama_index.schema import NodeWithScore -from llama_index.tools import QueryEngineTool, ToolMetadata +from llama_index.llms.openai import OpenAI +from llama_index.core import Settings +from llama_index.core import QueryBundle +from llama_index.core.base.base_query_engine import BaseQueryEngine +from llama_index.core.query_engine import SubQuestionQueryEngine +from llama_index.question_gen.guidance import GuidanceQuestionGenerator +from llama_index.core.schema import NodeWithScore +from llama_index.core.tools import QueryEngineTool, ToolMetadata from tc_hivemind_backend.embeddings.cohere import CohereEmbedding from utils.query_engine import prepare_discord_engine_auto_filter @@ -97,17 +99,19 @@ def query_multiple_source( if github: raise NotImplementedError + embed_model = CohereEmbedding() + llm = OpenAI("gpt-4") + Settings.embed_model = embed_model + Settings.llm = llm + question_gen = GuidanceQuestionGenerator.from_defaults( guidance_llm=OpenAIChat("gpt-4"), verbose=False, ) - embed_model = CohereEmbedding() - service_context = ServiceContext.from_defaults(embed_model=embed_model) s_engine = SubQuestionQueryEngine.from_defaults( question_gen=question_gen, query_engine_tools=query_engine_tools, use_async=False, - service_context=service_context, ) query_embedding = embed_model.get_text_embedding(text=query) response = s_engine.query(QueryBundle(query_str=query, embedding=query_embedding)) From a9ef3f370e3f157ed89b83ee477c21121a729a92 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 13 Mar 2024 08:51:09 +0330 Subject: [PATCH 6/8] fix: isort linter issue! --- subquery.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/subquery.py b/subquery.py index c7c829c..28e3123 100644 --- a/subquery.py +++ b/subquery.py @@ -1,12 +1,11 @@ from guidance.models import OpenAIChat -from llama_index.llms.openai import OpenAI -from llama_index.core import Settings -from llama_index.core import QueryBundle +from llama_index.core import QueryBundle, Settings from llama_index.core.base.base_query_engine import BaseQueryEngine from llama_index.core.query_engine import SubQuestionQueryEngine -from llama_index.question_gen.guidance import GuidanceQuestionGenerator from llama_index.core.schema import NodeWithScore from llama_index.core.tools import QueryEngineTool, ToolMetadata +from llama_index.llms.openai import OpenAI +from llama_index.question_gen.guidance import GuidanceQuestionGenerator from tc_hivemind_backend.embeddings.cohere import CohereEmbedding from utils.query_engine import prepare_discord_engine_auto_filter From 9a020fe11a876ab88912fd542caa4677d1b4adab Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 13 Mar 2024 09:12:46 +0330 Subject: [PATCH 7/8] feat: Added missing depdenency! --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 932cd8d..cb8c0d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ pytest>=7.4.3, <8.0.0 python-dotenv==1.0.0 tc-hivemind-backend==1.1.5 llama-index-question-gen-guidance==0.1.2 +llama-index-vector-stores-postgres==0.1.2 celery>=5.3.6, <6.0.0 guidance tc-messageBroker==1.6.6 From 651c182bec49be0f0e9a12480777ba9264d916c8 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 13 Mar 2024 17:11:31 +0330 Subject: [PATCH 8/8] feat:update response generator llm to gpt3.5-turbo --- subquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subquery.py b/subquery.py index 28e3123..58e2eec 100644 --- a/subquery.py +++ b/subquery.py @@ -99,7 +99,7 @@ def query_multiple_source( raise NotImplementedError embed_model = CohereEmbedding() - llm = OpenAI("gpt-4") + llm = OpenAI("gpt-3.5-turbo") Settings.embed_model = embed_model Settings.llm = llm