From ce2d4981fa2118ff923290904da699bb364a1fb3 Mon Sep 17 00:00:00 2001 From: Yasir Ekinci Date: Mon, 2 Oct 2023 15:52:35 +0200 Subject: [PATCH 01/11] stash --- pkg/plugin/vector/store/qdrant.go | 2 +- pkg/plugin/vector/store/store.go | 2 +- pkg/plugin/vector/store/vectorapi.go | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/plugin/vector/store/qdrant.go b/pkg/plugin/vector/store/qdrant.go index c7ba1b1b..0a5ec584 100644 --- a/pkg/plugin/vector/store/qdrant.go +++ b/pkg/plugin/vector/store/qdrant.go @@ -82,7 +82,7 @@ func (q *qdrantStore) CollectionExists(ctx context.Context, collection string) ( return true, nil } -func (q *qdrantStore) Search(ctx context.Context, collection string, vector []float32, topK uint64) ([]SearchResult, error) { +func (q *qdrantStore) Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]any) ([]SearchResult, error) { if q.md != nil { ctx = metadata.NewOutgoingContext(ctx, *q.md) } diff --git a/pkg/plugin/vector/store/store.go b/pkg/plugin/vector/store/store.go index 91bcee38..11dd6339 100644 --- a/pkg/plugin/vector/store/store.go +++ b/pkg/plugin/vector/store/store.go @@ -20,7 +20,7 @@ type SearchResult struct { type ReadVectorStore interface { CollectionExists(ctx context.Context, collection string) (bool, error) - Search(ctx context.Context, collection string, vector []float32, topK uint64) ([]SearchResult, error) + Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]any) ([]SearchResult, error) } type WriteVectorStore interface { diff --git a/pkg/plugin/vector/store/vectorapi.go b/pkg/plugin/vector/store/vectorapi.go index 230d4e71..a57413d8 100644 --- a/pkg/plugin/vector/store/vectorapi.go +++ b/pkg/plugin/vector/store/vectorapi.go @@ -31,14 +31,17 @@ func (g *grafanaVectorAPI) CollectionExists(ctx context.Context, collection stri return true, nil } -func (g *grafanaVectorAPI) Search(ctx context.Context, collection string, vector []float32, topK uint64) ([]SearchResult, error) { +func (g *grafanaVectorAPI) Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]any) ([]SearchResult, error) { type queryPointsRequest struct { Query []float32 `json:"query"` TopK uint64 `json:"top_k"` + // optional filter json field + Filter map[string]any `json:"filter"` } reqBody := queryPointsRequest{ - Query: vector, - TopK: topK, + Query: vector, + TopK: topK, + Filter: filter, } reqJSON, err := json.Marshal(reqBody) if err != nil { From 565ad64397f478c48b68692db9af6c8f8c8725fc Mon Sep 17 00:00:00 2001 From: Yasir Ekinci Date: Wed, 4 Oct 2023 14:48:40 +0200 Subject: [PATCH 02/11] qdrant filter test --- pkg/plugin/vector/store/qdrant.go | 39 +++++++++++++++++++++++++++++++ pkg/plugin/vector/store/store.go | 23 ++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/pkg/plugin/vector/store/qdrant.go b/pkg/plugin/vector/store/qdrant.go index 0a5ec584..ff0470fa 100644 --- a/pkg/plugin/vector/store/qdrant.go +++ b/pkg/plugin/vector/store/qdrant.go @@ -82,14 +82,53 @@ func (q *qdrantStore) CollectionExists(ctx context.Context, collection string) ( return true, nil } +func (q *qdrantStore) mapFilters(ctx context.Context, filter map[string]any) (*qdrant.Filter, error) { + // map input filter to qdrant filter + qdrantFilterMap := qdrant.Filter{} + if filter == nil { + return &qdrantFilterMap, nil + } + for k, v := range filter { + switch v := v.(type) { + case map[string]any: + for op, val := range v { + switch op { + case "$eq": + condition := qdrant.Condition{ + ConditionOneOf: &qdrant.Condition_Field{ + Field: &qdrant.FieldCondition{ + Key: k, + Match: &qdrant.Match{ + MatchValue: &qdrant.Match_Keyword{ + Keyword: val.(string), + }, + }, + }, + }, + } + qdrantFilterMap.Must = append(qdrantFilterMap.Must, &condition) + } + } + } + } + + return &qdrantFilterMap, nil +} func (q *qdrantStore) Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]any) ([]SearchResult, error) { if q.md != nil { ctx = metadata.NewOutgoingContext(ctx, *q.md) } + + qdrantFilter, err := q.mapFilters(ctx, filter) + if err != nil { + return nil, err + } + result, err := q.pointsClient.Search(ctx, &qdrant.SearchPoints{ CollectionName: collection, Vector: vector, Limit: topK, + Filter: qdrantFilter, // Include all payloads in the search result WithVectors: &qdrant.WithVectorsSelector{SelectorOptions: &qdrant.WithVectorsSelector_Enable{Enable: false}}, WithPayload: &qdrant.WithPayloadSelector{SelectorOptions: &qdrant.WithPayloadSelector_Enable{Enable: true}}, diff --git a/pkg/plugin/vector/store/store.go b/pkg/plugin/vector/store/store.go index 11dd6339..1428f017 100644 --- a/pkg/plugin/vector/store/store.go +++ b/pkg/plugin/vector/store/store.go @@ -18,6 +18,29 @@ type SearchResult struct { Score float64 `json:"score"` } +// exmaple filters +// only counter metrics +// {"": {"$eq": ""} } +// e.g. +// {"metric_type": {"$eq": "counter"} } +// {"year": {"$eq": 2007} } +// all counter or histogram metrics +// +// { +// "$or": [ +// { +// "metric_type": { +// "$eq": "counter" +// } +// }, +// { +// "metric_type": { +// "$eq": "histogram" +// } +// } +// ] +// } + type ReadVectorStore interface { CollectionExists(ctx context.Context, collection string) (bool, error) Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]any) ([]SearchResult, error) From dadd0a4ee32b2fc879a94986d5dee2e7e1d5022f Mon Sep 17 00:00:00 2001 From: Yasir Ekinci Date: Wed, 4 Oct 2023 14:56:04 +0200 Subject: [PATCH 03/11] match with val type --- pkg/plugin/vector/store/qdrant.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/plugin/vector/store/qdrant.go b/pkg/plugin/vector/store/qdrant.go index ff0470fa..f92942f1 100644 --- a/pkg/plugin/vector/store/qdrant.go +++ b/pkg/plugin/vector/store/qdrant.go @@ -3,6 +3,7 @@ package store import ( "context" "crypto/tls" + "fmt" "github.com/grafana/grafana-plugin-sdk-go/backend/log" qdrant "github.com/qdrant/go-client/qdrant" @@ -94,15 +95,30 @@ func (q *qdrantStore) mapFilters(ctx context.Context, filter map[string]any) (*q for op, val := range v { switch op { case "$eq": + // map value to qdrant match + match := qdrant.Match{} + switch val := val.(type) { + case string: + match.MatchValue = &qdrant.Match_Keyword{ + Keyword: val, + } + case int: + match.MatchValue = &qdrant.Match_Integer{ + Integer: int64(val), + } + case bool: + match.MatchValue = &qdrant.Match_Boolean{ + Boolean: val, + } + default: + return nil, fmt.Errorf("unsupported filter type: %T", val) + } + condition := qdrant.Condition{ ConditionOneOf: &qdrant.Condition_Field{ Field: &qdrant.FieldCondition{ - Key: k, - Match: &qdrant.Match{ - MatchValue: &qdrant.Match_Keyword{ - Keyword: val.(string), - }, - }, + Key: k, + Match: &match, }, }, } From 2bc90582bbe83dffab03a8a75d4fe448e6da6783 Mon Sep 17 00:00:00 2001 From: ioanarm Date: Wed, 4 Oct 2023 17:31:23 +0300 Subject: [PATCH 04/11] add or support and small iteration --- pkg/plugin/vector/store/qdrant.go | 85 +++++++++++++++++++------------ 1 file changed, 53 insertions(+), 32 deletions(-) diff --git a/pkg/plugin/vector/store/qdrant.go b/pkg/plugin/vector/store/qdrant.go index f92942f1..454aea28 100644 --- a/pkg/plugin/vector/store/qdrant.go +++ b/pkg/plugin/vector/store/qdrant.go @@ -83,53 +83,74 @@ func (q *qdrantStore) CollectionExists(ctx context.Context, collection string) ( return true, nil } -func (q *qdrantStore) mapFilters(ctx context.Context, filter map[string]any) (*qdrant.Filter, error) { - // map input filter to qdrant filter - qdrantFilterMap := qdrant.Filter{} +func (q *qdrantStore) mapFilters(ctx context.Context, filter map[string]interface{}) (*qdrant.Filter, error) { + qdrantFilterMap := &qdrant.Filter{} + if filter == nil { - return &qdrantFilterMap, nil + return qdrantFilterMap, nil } + for k, v := range filter { switch v := v.(type) { - case map[string]any: + case map[string]interface{}: for op, val := range v { + match, err := createQdrantMatch(val) + if err != nil { + return nil, err + } + + condition := &qdrant.Condition{ + ConditionOneOf: &qdrant.Condition_Field{ + Field: &qdrant.FieldCondition{ + Key: k, + Match: match, + }, + }, + } + switch op { case "$eq": - // map value to qdrant match - match := qdrant.Match{} - switch val := val.(type) { - case string: - match.MatchValue = &qdrant.Match_Keyword{ - Keyword: val, - } - case int: - match.MatchValue = &qdrant.Match_Integer{ - Integer: int64(val), - } - case bool: - match.MatchValue = &qdrant.Match_Boolean{ - Boolean: val, - } - default: - return nil, fmt.Errorf("unsupported filter type: %T", val) + qdrantFilterMap.Must = append(qdrantFilterMap.Must, condition) + case "$ne": + qdrantFilterMap.MustNot = append(qdrantFilterMap.MustNot, condition) + } + } + case []interface{}: + switch k { + case "$or": + for _, u := range v { + filterMap, err := q.mapFilters(ctx, u.(map[string]interface{})) + if err != nil { + return nil, err } - - condition := qdrant.Condition{ - ConditionOneOf: &qdrant.Condition_Field{ - Field: &qdrant.FieldCondition{ - Key: k, - Match: &match, - }, + qdrantFilterMap.Should = append(qdrantFilterMap.Should, &qdrant.Condition{ + ConditionOneOf: &qdrant.Condition_Filter{ + Filter: filterMap, }, - } - qdrantFilterMap.Must = append(qdrantFilterMap.Must, &condition) + }) } } + default: + return nil, fmt.Errorf("unsupported filter struct: %T", v) } } - return &qdrantFilterMap, nil + return qdrantFilterMap, nil +} + +func createQdrantMatch(val interface{}) (*qdrant.Match, error) { + match := &qdrant.Match{} + switch val := val.(type) { + case string: + match.MatchValue = &qdrant.Match_Keyword{ + Keyword: val, + } + default: + return nil, fmt.Errorf("unsupported filter type: %T", val) + } + return match, nil } + func (q *qdrantStore) Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]any) ([]SearchResult, error) { if q.md != nil { ctx = metadata.NewOutgoingContext(ctx, *q.md) From a26cf1568fa0eea3c4ab3e26dc6e51e35e084367 Mon Sep 17 00:00:00 2001 From: Yasir Ekinci Date: Thu, 5 Oct 2023 09:31:51 +0200 Subject: [PATCH 05/11] update filter type --- pkg/plugin/vector/store/qdrant.go | 2 +- pkg/plugin/vector/store/store.go | 2 +- pkg/plugin/vector/store/vectorapi.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/plugin/vector/store/qdrant.go b/pkg/plugin/vector/store/qdrant.go index 454aea28..f9958cbe 100644 --- a/pkg/plugin/vector/store/qdrant.go +++ b/pkg/plugin/vector/store/qdrant.go @@ -151,7 +151,7 @@ func createQdrantMatch(val interface{}) (*qdrant.Match, error) { return match, nil } -func (q *qdrantStore) Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]any) ([]SearchResult, error) { +func (q *qdrantStore) Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]interface{}) ([]SearchResult, error) { if q.md != nil { ctx = metadata.NewOutgoingContext(ctx, *q.md) } diff --git a/pkg/plugin/vector/store/store.go b/pkg/plugin/vector/store/store.go index 1428f017..75826b7c 100644 --- a/pkg/plugin/vector/store/store.go +++ b/pkg/plugin/vector/store/store.go @@ -43,7 +43,7 @@ type SearchResult struct { type ReadVectorStore interface { CollectionExists(ctx context.Context, collection string) (bool, error) - Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]any) ([]SearchResult, error) + Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]interface{}) ([]SearchResult, error) } type WriteVectorStore interface { diff --git a/pkg/plugin/vector/store/vectorapi.go b/pkg/plugin/vector/store/vectorapi.go index a57413d8..84cdddc7 100644 --- a/pkg/plugin/vector/store/vectorapi.go +++ b/pkg/plugin/vector/store/vectorapi.go @@ -31,12 +31,12 @@ func (g *grafanaVectorAPI) CollectionExists(ctx context.Context, collection stri return true, nil } -func (g *grafanaVectorAPI) Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]any) ([]SearchResult, error) { +func (g *grafanaVectorAPI) Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]interface{}) ([]SearchResult, error) { type queryPointsRequest struct { Query []float32 `json:"query"` TopK uint64 `json:"top_k"` // optional filter json field - Filter map[string]any `json:"filter"` + Filter map[string]interface{} `json:"filter"` } reqBody := queryPointsRequest{ Query: vector, From 1230041e3c6d284c2915fbbe6fd4000e9e81ad10 Mon Sep 17 00:00:00 2001 From: Yasir Ekinci Date: Thu, 5 Oct 2023 09:56:15 +0200 Subject: [PATCH 06/11] cleanup --- pkg/plugin/vector/store/store.go | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/pkg/plugin/vector/store/store.go b/pkg/plugin/vector/store/store.go index 75826b7c..e7f31a6c 100644 --- a/pkg/plugin/vector/store/store.go +++ b/pkg/plugin/vector/store/store.go @@ -18,29 +18,6 @@ type SearchResult struct { Score float64 `json:"score"` } -// exmaple filters -// only counter metrics -// {"": {"$eq": ""} } -// e.g. -// {"metric_type": {"$eq": "counter"} } -// {"year": {"$eq": 2007} } -// all counter or histogram metrics -// -// { -// "$or": [ -// { -// "metric_type": { -// "$eq": "counter" -// } -// }, -// { -// "metric_type": { -// "$eq": "histogram" -// } -// } -// ] -// } - type ReadVectorStore interface { CollectionExists(ctx context.Context, collection string) (bool, error) Search(ctx context.Context, collection string, vector []float32, topK uint64, filter map[string]interface{}) ([]SearchResult, error) From f79ef230d49f9b1562758112292cedd9331ebfad Mon Sep 17 00:00:00 2001 From: Yasir Ekinci Date: Mon, 16 Oct 2023 14:02:25 +0200 Subject: [PATCH 07/11] Add missing Filter params --- pkg/plugin/health.go | 2 +- pkg/plugin/resources.go | 3 ++- pkg/plugin/vector/service.go | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/plugin/health.go b/pkg/plugin/health.go index 7ec2f716..64733bd9 100644 --- a/pkg/plugin/health.go +++ b/pkg/plugin/health.go @@ -122,7 +122,7 @@ func (a *App) testVectorService(ctx context.Context) error { if a.vectorService == nil { return fmt.Errorf("vector service not configured") } - _, err := a.vectorService.Search(ctx, vectorCollections[0], "test", 1) + _, err := a.vectorService.Search(ctx, vectorCollections[0], "test", 1, nil) return err } diff --git a/pkg/plugin/resources.go b/pkg/plugin/resources.go index 1d795d89..8647e74b 100644 --- a/pkg/plugin/resources.go +++ b/pkg/plugin/resources.go @@ -171,6 +171,7 @@ type vectorSearchRequest struct { Query string `json:"query"` Collection string `json:"collection"` TopK uint64 `json:"topK"` + Filter map[string]interface{} } type vectorSearchResponse struct { @@ -194,7 +195,7 @@ func (app *App) handleVectorSearch(w http.ResponseWriter, req *http.Request) { if body.TopK == 0 { body.TopK = 10 } - results, err := app.vectorService.Search(req.Context(), body.Collection, body.Query, body.TopK) + results, err := app.vectorService.Search(req.Context(), body.Collection, body.Query, body.TopK, body.Filter) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/pkg/plugin/vector/service.go b/pkg/plugin/vector/service.go index b1a6b4bb..6120c7e8 100644 --- a/pkg/plugin/vector/service.go +++ b/pkg/plugin/vector/service.go @@ -12,7 +12,7 @@ import ( ) type Service interface { - Search(ctx context.Context, collection string, query string, topK uint64) ([]store.SearchResult, error) + Search(ctx context.Context, collection string, query string, topK uint64, filter map[string]interface{}) ([]store.SearchResult, error) Cancel() } @@ -57,7 +57,7 @@ func NewService(s VectorSettings, secrets map[string]string) (Service, error) { }, nil } -func (v *vectorService) Search(ctx context.Context, collection string, query string, topK uint64) ([]store.SearchResult, error) { +func (v *vectorService) Search(ctx context.Context, collection string, query string, topK uint64, filter map[string]interface{}) ([]store.SearchResult, error) { if query == "" { return nil, fmt.Errorf("query cannot be empty") } @@ -78,7 +78,7 @@ func (v *vectorService) Search(ctx context.Context, collection string, query str log.DefaultLogger.Info("Searching", "collection", collection, "query", query) // Search the vector store for similar vectors. - results, err := v.store.Search(ctx, collection, e, topK) + results, err := v.store.Search(ctx, collection, e, topK, filter) if err != nil { return nil, fmt.Errorf("vector store search: %w", err) } From 27979bde9f1d24ec049fa478de25f09b5520da11 Mon Sep 17 00:00:00 2001 From: Yasir Ekinci Date: Mon, 16 Oct 2023 14:07:16 +0200 Subject: [PATCH 08/11] mockVectorService: fix wrong type for Search --- pkg/plugin/health_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/health_test.go b/pkg/plugin/health_test.go index 3cc4b54b..e24b4c9f 100644 --- a/pkg/plugin/health_test.go +++ b/pkg/plugin/health_test.go @@ -23,7 +23,7 @@ func (m *mockHealthCheckClient) Do(req *http.Request) (*http.Response, error) { type mockVectorService struct{} -func (m *mockVectorService) Search(ctx context.Context, collection string, query string, topK uint64) ([]store.SearchResult, error) { +func (m *mockVectorService) Search(ctx context.Context, collection string, query string, topK uint64, filter map[string]interface{}) ([]store.SearchResult, error) { return []store.SearchResult{{Payload: map[string]any{"a": "b"}, Score: 1.0}}, nil } From 6824369a3588150d3d12138640ea7e58ce622fe9 Mon Sep 17 00:00:00 2001 From: Yasir Ekinci Date: Mon, 16 Oct 2023 19:42:09 +0200 Subject: [PATCH 09/11] vectorSearchRequest: add json tag --- pkg/plugin/resources.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/plugin/resources.go b/pkg/plugin/resources.go index 8647e74b..7915f999 100644 --- a/pkg/plugin/resources.go +++ b/pkg/plugin/resources.go @@ -168,10 +168,10 @@ func newAzureOpenAIProxy(settings Settings) http.Handler { } type vectorSearchRequest struct { - Query string `json:"query"` - Collection string `json:"collection"` - TopK uint64 `json:"topK"` - Filter map[string]interface{} + Query string `json:"query"` + Collection string `json:"collection"` + TopK uint64 `json:"topK"` + Filter map[string]interface{} `json:"filter"` } type vectorSearchResponse struct { From a332284a67114a04aac26cb46fbd88f1a840c406 Mon Sep 17 00:00:00 2001 From: Yasir Ekinci Date: Mon, 16 Oct 2023 19:44:12 +0200 Subject: [PATCH 10/11] mapFilters: bail out on unsupported operator --- pkg/plugin/vector/store/qdrant.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/plugin/vector/store/qdrant.go b/pkg/plugin/vector/store/qdrant.go index f9958cbe..9988ce5f 100644 --- a/pkg/plugin/vector/store/qdrant.go +++ b/pkg/plugin/vector/store/qdrant.go @@ -113,6 +113,8 @@ func (q *qdrantStore) mapFilters(ctx context.Context, filter map[string]interfac qdrantFilterMap.Must = append(qdrantFilterMap.Must, condition) case "$ne": qdrantFilterMap.MustNot = append(qdrantFilterMap.MustNot, condition) + default: + return nil, fmt.Errorf("unsupported operator: %s", op) } } case []interface{}: @@ -129,6 +131,8 @@ func (q *qdrantStore) mapFilters(ctx context.Context, filter map[string]interfac }, }) } + default: + return nil, fmt.Errorf("unsupported operator: %s", k) } default: return nil, fmt.Errorf("unsupported filter struct: %T", v) From 8b6736eb457e4797ecc00c6a804e9f5a03c70d40 Mon Sep 17 00:00:00 2001 From: Yasir Ekinci Date: Mon, 16 Oct 2023 19:48:09 +0200 Subject: [PATCH 11/11] qdrant: support $and filter --- pkg/plugin/vector/store/qdrant.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/plugin/vector/store/qdrant.go b/pkg/plugin/vector/store/qdrant.go index 9988ce5f..573bba0e 100644 --- a/pkg/plugin/vector/store/qdrant.go +++ b/pkg/plugin/vector/store/qdrant.go @@ -131,6 +131,18 @@ func (q *qdrantStore) mapFilters(ctx context.Context, filter map[string]interfac }, }) } + case "$and": + for _, u := range v { + filterMap, err := q.mapFilters(ctx, u.(map[string]interface{})) + if err != nil { + return nil, err + } + qdrantFilterMap.Must = append(qdrantFilterMap.Must, &qdrant.Condition{ + ConditionOneOf: &qdrant.Condition_Filter{ + Filter: filterMap, + }, + }) + } default: return nil, fmt.Errorf("unsupported operator: %s", k) }