-
-
Notifications
You must be signed in to change notification settings - Fork 93
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Co-authored-by: Konstantin S. <[email protected]>
- Loading branch information
Showing
8 changed files
with
134 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| using Microsoft.SemanticKernel.Connectors.DuckDB; | ||
| using Microsoft.SemanticKernel.Memory; | ||
|
|
||
| namespace LangChain.Databases.DuckDb | ||
| { | ||
| public class DuckDbVectorCollection( | ||
| DuckDBMemoryStore store, | ||
| string name = VectorCollection.DefaultName, | ||
| string? id = null) : VectorCollection(name, id), IVectorCollection | ||
| { | ||
| public async Task<IReadOnlyCollection<string>> AddAsync(IReadOnlyCollection<Vector> items, CancellationToken cancellationToken = default) | ||
| { | ||
| items = items ?? throw new ArgumentNullException(nameof(items)); | ||
|
|
||
| List<string> list = []; | ||
| foreach (var item in items) | ||
| { | ||
| string? metadata = null; | ||
| //TODO: review way to map metadata | ||
| if (item.Metadata != null) | ||
| metadata = string.Join("#", item.Metadata.Select(kv => kv.Key + "&" + kv.Value)); | ||
| var record = MemoryRecord.LocalRecord(item.Id, item.Text, null, item.Embedding, metadata); | ||
| var insert = await store.UpsertAsync(Name, record, cancellationToken).ConfigureAwait(false); | ||
| list.Add(insert); | ||
| } | ||
| return list; | ||
|
|
||
| } | ||
|
|
||
| public async Task<bool> DeleteAsync(IEnumerable<string> ids, CancellationToken cancellationToken = default) | ||
| { | ||
| await store.RemoveBatchAsync(Name, ids, cancellationToken).ConfigureAwait(false); | ||
| return true; | ||
| } | ||
|
|
||
| public async Task<Vector?> GetAsync(string id, CancellationToken cancellationToken = default) | ||
| { | ||
| var record = await store.GetAsync(Name, id, cancellationToken: cancellationToken).ConfigureAwait(false); | ||
|
|
||
| Dictionary<string, object>? metadata = null; | ||
| if(record?.Metadata?.AdditionalMetadata!=null) | ||
| metadata = record.Metadata.AdditionalMetadata | ||
| .Split('#') | ||
| .Select(part => part.Split('&')) | ||
| .ToDictionary(split => split[0], split => (object)split[1]); | ||
|
|
||
| return record != null ? new Vector { Id = id, Text = record.Metadata.Text, Metadata = metadata } : null; | ||
| } | ||
|
|
||
| public async Task<bool> IsEmptyAsync(CancellationToken cancellationToken = default) | ||
| { | ||
| var collections = store.GetCollectionsAsync(cancellationToken); | ||
| return !(await collections.CountAsync(cancellationToken).ConfigureAwait(false) > 0); | ||
| } | ||
|
|
||
| public async Task<VectorSearchResponse> SearchAsync(VectorSearchRequest request, VectorSearchSettings? settings = null, CancellationToken cancellationToken = default) | ||
| { | ||
| request = request ?? throw new ArgumentNullException(nameof(request)); | ||
| settings ??= new VectorSearchSettings(); | ||
| var results = await store.GetNearestMatchesAsync(Name, request.Embeddings.First(), limit: settings.NumberOfResults, cancellationToken: cancellationToken) | ||
| .ToListAsync(cancellationToken).ConfigureAwait(false); | ||
| return new VectorSearchResponse { Items = results.Select(x => new Vector { Text = x.Item1.Metadata.Text }).ToList() }; | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| using Microsoft.SemanticKernel.Connectors.DuckDB; | ||
|
|
||
| namespace LangChain.Databases.DuckDb | ||
| { | ||
| public class DuckDbVectorDatabase(DuckDBMemoryStore store) : IVectorDatabase | ||
| { | ||
| public async Task CreateCollectionAsync(string collectionName, int dimensions, CancellationToken cancellationToken = default) | ||
| { | ||
| await store.CreateCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| public async Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) | ||
| { | ||
| await store.DeleteCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| public async Task<IVectorCollection> GetCollectionAsync(string collectionName, CancellationToken cancellationToken = default) | ||
| { | ||
| var collections = await ListCollectionsAsync(cancellationToken).ConfigureAwait(false); | ||
| var collection = collections.FirstOrDefault(x => x == collectionName); | ||
| return collection != null ? new DuckDbVectorCollection(store, collection) | ||
| : throw new InvalidOperationException("Collection not found"); | ||
| } | ||
|
|
||
| public async Task<IVectorCollection> GetOrCreateCollectionAsync(string collectionName, int dimensions, CancellationToken cancellationToken = default) | ||
| { | ||
| if(!await IsCollectionExistsAsync(collectionName, cancellationToken).ConfigureAwait(false)) | ||
| await store.CreateCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| return new DuckDbVectorCollection(store, collectionName); | ||
| } | ||
|
|
||
| public async Task<bool> IsCollectionExistsAsync(string collectionName, CancellationToken cancellationToken = default) | ||
| { | ||
| return await store.DoesCollectionExistAsync(collectionName, cancellationToken).ConfigureAwait(false); | ||
| } | ||
|
|
||
| public async Task<IReadOnlyList<string>> ListCollectionsAsync(CancellationToken cancellationToken = default) | ||
| { | ||
| var collections = store.GetCollectionsAsync(cancellationToken); | ||
| return await collections.ToListAsync(cancellationToken).ConfigureAwait(false); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,4 +10,5 @@ public enum SupportedDatabase | |
| Postgres, | ||
| Redis, | ||
| Mongo, | ||
| DuckDb | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters