- changed the retrival method a little.

- Made modifications in Nutrition facts scraper to modify any list in form of a string
amosproj · Jun 30, 2024 · 477abff · 477abff
1 parent 0251d5d
commit 477abff
Show file tree

Hide file tree

Showing 13 changed files with 509 additions and 3 deletions.
diff --git a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/data_level0.bin b/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/data_level0.bin
diff --git a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/header.bin b/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/header.bin
diff --git a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/length.bin b/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/length.bin
diff --git a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/link_lists.bin b/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/link_lists.bin
diff --git a/chroma_db/chroma.sqlite3 b/chroma_db/chroma.sqlite3
diff --git a/src/backend/RAG/LangChain_Implementation/basic_retrieval.py b/src/backend/RAG/LangChain_Implementation/basic_retrieval.py
@@ -89,7 +89,7 @@ def _extract_from_dict(self, doc_dict, chunks_with_metadata):
 chroma_db.add_documents(documents=document_chunks)
 
 # ridiculous linting rules - thanks:
-query_text1 = 'what did the executive at the Kellogs ad firm say? '
+query_text1 = 'Can Blueberries Help with Diabetes and Repairing DNA?'
 query_text2 = 'What can we conclude from this response?'
 query_text = query_text1 + query_text2
 
@@ -104,6 +104,8 @@ def _extract_from_dict(self, doc_dict, chunks_with_metadata):
     metadata_str = ', '.join([f'{key}: {value}' for key, value in metadata.items()])
     relevant_info += f'Content: {doc_content}\nMetadata: {metadata_str}\n\n'
 
+print('Relevant information is:', relevant_info)
+
 modified_prompt = {'text': f'{query_text}\n\nHere is some relevant information:\n{relevant_info}'}
 
 prompt_text = modified_prompt['text']

diff --git a/src/backend/RAG/LangChain_Implementation/blog_data.json b/src/backend/RAG/LangChain_Implementation/blog_data.json
diff --git a/...G/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/data_level0.bin b/...G/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/data_level0.bin
diff --git a/...nd/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/header.bin b/...nd/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/header.bin
diff --git a/...nd/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/length.bin b/...nd/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/length.bin
diff --git a/...AG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/link_lists.bin b/...AG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/link_lists.bin
diff --git a/src/backend/RAG/LangChain_Implementation/chroma_db/chroma.sqlite3 b/src/backend/RAG/LangChain_Implementation/chroma_db/chroma.sqlite3
diff --git a/src/backend/Scrapers/Nutritionfacts/nutrition.py b/src/backend/Scrapers/Nutritionfacts/nutrition.py
@@ -213,7 +213,7 @@ def get_url_content(self, driver, blog_url):
             image_elements = content_element.find_all('img')
             image_urls = [img['src'] for img in image_elements if 'src' in img.attrs]
 
-            return title, date, author, content_chunks, key_take_away_chunks, image_urls, blog_url
+            return (title, date, author, content_chunks, key_take_away_chunks, image_urls, blog_url)
         except Exception:
             print(f'Error getting content from url: {blog_url}')
             error_msg = f'Error getting content from url: {blog_url}'
@@ -226,6 +226,10 @@ def get_url_content(self, driver, blog_url):
 
     def get_documents(self, data: TypeNutritionScrappingData) -> List[Document]:
         transcript = data.get('transcript', '')
+
+        if isinstance(transcript, list):
+            transcript = ' '.join(transcript)
+
         chunks = get_text_chunks(transcript)
         metadata = {
             'author': data.get('author', ''),
@@ -249,7 +253,7 @@ def _scrape(self) -> str:
             if nutrition is None:
                 raise ValueError('Data does not exist for id: ' + str(self.element_id))
 
-            title, date, author, content_chunks, key_take_away_chunks, image_urls, blog_url = (
+            (title, date, author, content_chunks, key_take_away_chunks, image_urls, blog_url) = (
                 nutrition
             )
             info: TypeNutritionScrappingData = {