Skip to content

Commit

Permalink
Rearrange to bit a bit less of a change
Browse files Browse the repository at this point in the history
  • Loading branch information
nelsonauner committed Jan 31, 2024
1 parent 0d7e14c commit acc4779
Showing 1 changed file with 19 additions and 20 deletions.
39 changes: 19 additions & 20 deletions libs/community/langchain_community/vectorstores/pinecone.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,34 +144,33 @@ def add_texts(
for metadata, text in zip(metadatas, texts):
metadata[self._text_key] = text

if async_req:
# For loops to avoid memory issues when using HTTP-based embeddings
# First loop runs embeddings, benefits when using OpenAI embeddings
# The second loops runs the pinecone upsert asynchronously.
for i in range(0, len(texts), embedding_chunk_size):
chunk_texts = texts[i : i + embedding_chunk_size]
chunk_ids = ids[i : i + embedding_chunk_size]
chunk_metadatas = metadatas[i : i + embedding_chunk_size]
embeddings = self._embed_documents(chunk_texts)
# For loops to avoid memory issues when using HTTP-based embeddings
# First loop runs embeddings, benefits when using OpenAI embeddings
for i in range(0, len(texts), embedding_chunk_size):
chunk_texts = texts[i : i + embedding_chunk_size]
chunk_ids = ids[i : i + embedding_chunk_size]
chunk_metadatas = metadatas[i : i + embedding_chunk_size]
embeddings = self._embed_documents(chunk_texts)
vector_tuples = zip(chunk_ids, embeddings, chunk_metadatas)
if async_req:
# Runs the pinecone upsert asynchronously.
async_res = [
self._index.upsert(
vectors=batch,
vectors=batch_vector_tuples,
namespace=namespace,
async_req=async_req,
**kwargs,
)
for batch in batch_iterate(
batch_size, zip(chunk_ids, embeddings, chunk_metadatas)
)
for batch_vector_tuples in batch_iterate(batch_size, vector_tuples)
]
[res.get() for res in async_res]
else:
self._index.upsert(
vectors=list(zip(ids, self._embed_documents(texts), metadatas)),
namespace=namespace,
async_req=async_req,
**kwargs,
)
else:
self._index.upsert(
vectors=vector_tuples,
namespace=namespace,
async_req=async_req,
**kwargs,
)

return ids

Expand Down

0 comments on commit acc4779

Please sign in to comment.