-
-
Notifications
You must be signed in to change notification settings - Fork 93
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Added ability to use all SemanticKernel connectors databases (#421
) * refactoring semantic kernel connectors databases * refactor: Renamed Connectors to SemanticKernel. * style: Format .csproj. --------- Co-authored-by: HavenDV <[email protected]>
- Loading branch information
Showing
23 changed files
with
347 additions
and
335 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,65 +1,10 @@ | ||
| using Microsoft.SemanticKernel.Connectors.DuckDB; | ||
| using Microsoft.SemanticKernel.Memory; | ||
| using LangChain.Databases.SemanticKernel; | ||
| using Microsoft.SemanticKernel.Connectors.DuckDB; | ||
|
|
||
| namespace LangChain.Databases.DuckDb | ||
| { | ||
| public class DuckDbVectorCollection( | ||
| DuckDBMemoryStore store, | ||
| string name = VectorCollection.DefaultName, | ||
| string? id = null) : VectorCollection(name, id), IVectorCollection | ||
| { | ||
| public async Task<IReadOnlyCollection<string>> AddAsync(IReadOnlyCollection<Vector> items, CancellationToken cancellationToken = default) | ||
| { | ||
| items = items ?? throw new ArgumentNullException(nameof(items)); | ||
|
|
||
| List<string> list = []; | ||
| foreach (var item in items) | ||
| { | ||
| string? metadata = null; | ||
| //TODO: review way to map metadata | ||
| if (item.Metadata != null) | ||
| metadata = string.Join("#", item.Metadata.Select(kv => kv.Key + "&" + kv.Value)); | ||
| var record = MemoryRecord.LocalRecord(item.Id, item.Text, null, item.Embedding, metadata); | ||
| var insert = await store.UpsertAsync(Name, record, cancellationToken).ConfigureAwait(false); | ||
| list.Add(insert); | ||
| } | ||
| return list; | ||
|
|
||
| } | ||
|
|
||
| public async Task<bool> DeleteAsync(IEnumerable<string> ids, CancellationToken cancellationToken = default) | ||
| { | ||
| await store.RemoveBatchAsync(Name, ids, cancellationToken).ConfigureAwait(false); | ||
| return true; | ||
| } | ||
|
|
||
| public async Task<Vector?> GetAsync(string id, CancellationToken cancellationToken = default) | ||
| { | ||
| var record = await store.GetAsync(Name, id, cancellationToken: cancellationToken).ConfigureAwait(false); | ||
|
|
||
| Dictionary<string, object>? metadata = null; | ||
| if(record?.Metadata?.AdditionalMetadata!=null) | ||
| metadata = record.Metadata.AdditionalMetadata | ||
| .Split('#') | ||
| .Select(part => part.Split('&')) | ||
| .ToDictionary(split => split[0], split => (object)split[1]); | ||
|
|
||
| return record != null ? new Vector { Id = id, Text = record.Metadata.Text, Metadata = metadata } : null; | ||
| } | ||
|
|
||
| public async Task<bool> IsEmptyAsync(CancellationToken cancellationToken = default) | ||
| { | ||
| var collections = store.GetCollectionsAsync(cancellationToken); | ||
| return !(await collections.CountAsync(cancellationToken).ConfigureAwait(false) > 0); | ||
| } | ||
|
|
||
| public async Task<VectorSearchResponse> SearchAsync(VectorSearchRequest request, VectorSearchSettings? settings = null, CancellationToken cancellationToken = default) | ||
| { | ||
| request = request ?? throw new ArgumentNullException(nameof(request)); | ||
| settings ??= new VectorSearchSettings(); | ||
| var results = await store.GetNearestMatchesAsync(Name, request.Embeddings.First(), limit: settings.NumberOfResults, cancellationToken: cancellationToken) | ||
| .ToListAsync(cancellationToken).ConfigureAwait(false); | ||
| return new VectorSearchResponse { Items = results.Select(x => new Vector { Text = x.Item1.Metadata.Text }).ToList() }; | ||
| } | ||
| } | ||
| string? id = null) : SemanticKernelMemoryStoreCollection(store, name, id); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,43 +1,7 @@ | ||
| using Microsoft.SemanticKernel.Connectors.DuckDB; | ||
| using LangChain.Databases.SemanticKernel; | ||
| using Microsoft.SemanticKernel.Connectors.DuckDB; | ||
|
|
||
| namespace LangChain.Databases.DuckDb | ||
| { | ||
| public class DuckDbVectorDatabase(DuckDBMemoryStore store) : IVectorDatabase | ||
| { | ||
| public async Task CreateCollectionAsync(string collectionName, int dimensions, CancellationToken cancellationToken = default) | ||
| { | ||
| await store.CreateCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| public async Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) | ||
| { | ||
| await store.DeleteCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| public async Task<IVectorCollection> GetCollectionAsync(string collectionName, CancellationToken cancellationToken = default) | ||
| { | ||
| var collections = await ListCollectionsAsync(cancellationToken).ConfigureAwait(false); | ||
| var collection = collections.FirstOrDefault(x => x == collectionName); | ||
| return collection != null ? new DuckDbVectorCollection(store, collection) | ||
| : throw new InvalidOperationException("Collection not found"); | ||
| } | ||
|
|
||
| public async Task<IVectorCollection> GetOrCreateCollectionAsync(string collectionName, int dimensions, CancellationToken cancellationToken = default) | ||
| { | ||
| if(!await IsCollectionExistsAsync(collectionName, cancellationToken).ConfigureAwait(false)) | ||
| await store.CreateCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| return new DuckDbVectorCollection(store, collectionName); | ||
| } | ||
|
|
||
| public async Task<bool> IsCollectionExistsAsync(string collectionName, CancellationToken cancellationToken = default) | ||
| { | ||
| return await store.DoesCollectionExistAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| public async Task<IReadOnlyList<string>> ListCollectionsAsync(CancellationToken cancellationToken = default) | ||
| { | ||
| var collections = store.GetCollectionsAsync(cancellationToken); | ||
| return await collections.ToListAsync(cancellationToken).ConfigureAwait(false); | ||
| } | ||
| } | ||
| public class DuckDbVectorDatabase(DuckDBMemoryStore store) : SemanticKernelMemoryDatabase(store); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,64 +1,9 @@ | ||
| using Microsoft.SemanticKernel.Connectors.Milvus; | ||
| using Microsoft.SemanticKernel.Memory; | ||
| using LangChain.Databases.SemanticKernel; | ||
| using Microsoft.SemanticKernel.Connectors.Milvus; | ||
|
|
||
| namespace LangChain.Databases.Milvus; | ||
|
|
||
| public sealed class MilvusVectorCollection( | ||
| MilvusMemoryStore store, | ||
| string name = VectorCollection.DefaultName, | ||
| string? id = null) | ||
| : VectorCollection(name, id), IVectorCollection | ||
| { | ||
| public async Task<IReadOnlyCollection<string>> AddAsync(IReadOnlyCollection<Vector> items, CancellationToken cancellationToken = default) | ||
| { | ||
| items = items ?? throw new ArgumentNullException(nameof(items)); | ||
|
|
||
| List<string> list = []; | ||
| foreach (var item in items) | ||
| { | ||
| string? metadata = null; | ||
| //TODO: review way to map metadata | ||
| if (item.Metadata != null) | ||
| metadata = string.Join("#", item.Metadata.Select(kv => kv.Key + "&" + kv.Value)); | ||
| var record = MemoryRecord.LocalRecord(item.Id, item.Text, null, item.Embedding, metadata); | ||
| var insert = await store.UpsertAsync(Name, record, cancellationToken).ConfigureAwait(false); | ||
| list.Add(insert); | ||
| } | ||
| return list; | ||
| } | ||
|
|
||
| public async Task<bool> DeleteAsync(IEnumerable<string> ids, CancellationToken cancellationToken = default) | ||
| { | ||
| await store.RemoveBatchAsync(Name, ids, cancellationToken).ConfigureAwait(false); | ||
| return true; | ||
| } | ||
|
|
||
| public async Task<Vector?> GetAsync(string id, CancellationToken cancellationToken = default) | ||
| { | ||
| var record = await store.GetAsync(Name, id, cancellationToken: cancellationToken).ConfigureAwait(false); | ||
|
|
||
| Dictionary<string, object>? metadata = null; | ||
| if (record?.Metadata?.AdditionalMetadata != null) | ||
| metadata = record.Metadata.AdditionalMetadata | ||
| .Split('#') | ||
| .Select(part => part.Split('&')) | ||
| .ToDictionary(split => split[0], split => (object)split[1]); | ||
|
|
||
| return record != null ? new Vector { Id = id, Text = record.Metadata.Text, Metadata = metadata } : null; | ||
| } | ||
|
|
||
| public async Task<bool> IsEmptyAsync(CancellationToken cancellationToken = default) | ||
| { | ||
| var collections = store.GetCollectionsAsync(cancellationToken); | ||
| return !(await collections.CountAsync(cancellationToken).ConfigureAwait(false) > 0); | ||
| } | ||
|
|
||
| public async Task<VectorSearchResponse> SearchAsync(VectorSearchRequest request, VectorSearchSettings? settings = null, CancellationToken cancellationToken = default) | ||
| { | ||
| request = request ?? throw new ArgumentNullException(nameof(request)); | ||
| settings ??= new VectorSearchSettings(); | ||
| var results = await store.GetNearestMatchesAsync(Name, request.Embeddings.First(), limit: settings.NumberOfResults, cancellationToken: cancellationToken) | ||
| .ToListAsync(cancellationToken).ConfigureAwait(false); | ||
| return new VectorSearchResponse { Items = results.Select(x => new Vector { Text = x.Item1.Metadata.Text }).ToList() }; | ||
| } | ||
| } | ||
| public class MilvusVectorCollection( | ||
| MilvusMemoryStore store, | ||
| string name = VectorCollection.DefaultName, | ||
| string? id = null) : SemanticKernelMemoryStoreCollection(store, name, id); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,44 +1,6 @@ | ||
| using Microsoft.SemanticKernel.Connectors.Milvus; | ||
| using LangChain.Databases.SemanticKernel; | ||
| using Microsoft.SemanticKernel.Connectors.Milvus; | ||
|
|
||
| namespace LangChain.Databases.Milvus; | ||
|
|
||
| public class MilvusVectorDatabase( | ||
| MilvusMemoryStore store) | ||
| : IVectorDatabase | ||
| { | ||
| public async Task CreateCollectionAsync(string collectionName, int dimensions, CancellationToken cancellationToken = default) | ||
| { | ||
| await store.CreateCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| public async Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) | ||
| { | ||
| await store.DeleteCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| public async Task<IVectorCollection> GetCollectionAsync(string collectionName, CancellationToken cancellationToken = default) | ||
| { | ||
| var collections = await ListCollectionsAsync(cancellationToken).ConfigureAwait(false); | ||
| var collection = collections.FirstOrDefault(x => x == collectionName); | ||
| return collection != null ? new MilvusVectorCollection(store, collection) | ||
| : throw new InvalidOperationException("Collection not found"); | ||
| } | ||
|
|
||
| public async Task<IVectorCollection> GetOrCreateCollectionAsync(string collectionName, int dimensions, CancellationToken cancellationToken = default) | ||
| { | ||
| if (!await IsCollectionExistsAsync(collectionName, cancellationToken).ConfigureAwait(false)) | ||
| await store.CreateCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| return new MilvusVectorCollection(store, collectionName); | ||
| } | ||
|
|
||
| public async Task<bool> IsCollectionExistsAsync(string collectionName, CancellationToken cancellationToken = default) | ||
| { | ||
| return await store.DoesCollectionExistAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| public async Task<IReadOnlyList<string>> ListCollectionsAsync(CancellationToken cancellationToken = default) | ||
| { | ||
| var collections = store.GetCollectionsAsync(cancellationToken); | ||
| return await collections.ToListAsync(cancellationToken).ConfigureAwait(false); | ||
| } | ||
| } | ||
| public class MilvusVectorDatabase(MilvusMemoryStore store) : SemanticKernelMemoryDatabase(store); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.