From b0007998fa0bde238da88840f59f784cb73f5023 Mon Sep 17 00:00:00 2001 From: Ben Sully Date: Wed, 30 Aug 2023 15:51:33 +0100 Subject: [PATCH] Add dashboard syncing in background goroutine --- cspell.config.json | 9 + docker-compose.yaml | 5 + go.mod | 6 + go.sum | 16 ++ grafana.ini | 4 + pkg/plugin/app.go | 13 +- pkg/plugin/vector/service.go | 74 +++++++- pkg/plugin/vector/store/qdrant.go | 154 +++++++++++++++- pkg/plugin/vector/store/store.go | 22 ++- pkg/plugin/vector/sync.go | 173 ++++++++++++++++++ provisioning/dashboards/default.yaml | 7 + provisioning/dashboards/mimir-usage.json | 134 ++++++++++++++ .../datasources/robust-perception.yaml | 10 + provisioning/plugins/grafana-llm-app.yaml | 2 +- src/plugin.json | 20 +- 15 files changed, 631 insertions(+), 18 deletions(-) create mode 100644 grafana.ini create mode 100644 pkg/plugin/vector/sync.go create mode 100644 provisioning/dashboards/default.yaml create mode 100644 provisioning/dashboards/mimir-usage.json create mode 100644 provisioning/datasources/robust-perception.yaml diff --git a/cspell.config.json b/cspell.config.json index f596389d..78e2fb6e 100644 --- a/cspell.config.json +++ b/cspell.config.json @@ -30,15 +30,24 @@ "embedder", "errcheck", "eventsource", + "gapi", "grafana", "httpadapter", + "httpclient", "instancemgmt", + "jdoc", "llms", "nolint", + "oauthtokenretriever", "openai", "proxying", "qdrant", + "rgba", + "structs", + "templating", "testid", + "timepicker", + "timeseries", "unmarshalling", "Upsert", "vectorapi" diff --git a/docker-compose.yaml b/docker-compose.yaml index ddef8de9..ade4bfee 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,12 +8,17 @@ services: args: grafana_version: ${GRAFANA_VERSION:-10.1.0} environment: + GF_FEATURE_TOGGLES_ENABLE: 'externalServiceAuth' + GF_AUTH_EXTENDED_JWT_ENABLED: true + GF_AUTH_EXTENDED_JWT_EXPECT_ISSUER: http://localhost:3000/ + GF_AUTH_EXTENDED_JWT_EXPECT_AUDIENCE: http://localhost:3000/ OPENAI_API_KEY: $OPENAI_API_KEY ports: - 3000:3000/tcp volumes: - ./dist:/var/lib/grafana/plugins/grafana-llm-app - ./provisioning:/etc/grafana/provisioning + - ./grafana.ini:/etc/grafana/grafana.ini qdrant: image: qdrant/qdrant diff --git a/go.mod b/go.mod index 52ae50dc..215fc9ad 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/grafana/grafana-llm-app go 1.19 require ( + github.com/grafana/grafana-api-golang-client v0.23.0 github.com/grafana/grafana-plugin-sdk-go v0.174.0 github.com/launchdarkly/eventsource v1.7.1 github.com/qdrant/go-client v1.5.0 @@ -22,6 +23,7 @@ require ( github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027 // indirect github.com/fatih/color v1.15.0 // indirect github.com/getkin/kin-openapi v0.112.0 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -35,6 +37,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-plugin v1.4.9 // indirect github.com/hashicorp/yamux v0.1.1 // indirect @@ -76,10 +79,13 @@ require ( go.opentelemetry.io/otel/sdk v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect + golang.org/x/crypto v0.11.0 // indirect golang.org/x/net v0.12.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf // indirect google.golang.org/protobuf v1.31.0 // indirect diff --git a/go.sum b/go.sum index e7b97ad4..4792714b 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,8 @@ github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3 github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -81,12 +83,15 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b h1:/vQ+oYKu+JoyaMPDsv5FzwuL2wwWBgBbtj/YLCi4LuA= +github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= @@ -123,6 +128,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/grafana/grafana-api-golang-client v0.23.0 h1:Uta0dSkxWYf1D83/E7MRLCG69387FiUc+k9U/35nMhY= +github.com/grafana/grafana-api-golang-client v0.23.0/go.mod h1:24W29gPe9yl0/3A9X624TPkAOR8DpHno490cPwnkv8E= github.com/grafana/grafana-plugin-sdk-go v0.174.0 h1:b0bHa5DUO71f2NtSCg6AAdPXQ0Z1axxfJFRMS1hvui0= github.com/grafana/grafana-plugin-sdk-go v0.174.0/go.mod h1:9crAQqSzxvPe0VKC/T23cd+2I9TZb43yoOcUL/qZ5FU= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= @@ -132,6 +139,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 h1:dygLcbEBA+t/P7ck6a8AkXv6juQ4cK0RHBoh32jxhHM= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2/go.mod h1:Ap9RLCIJVtgQg1/BBgVEfypOAySvvlcpcVQkSzJCH4Y= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-plugin v1.4.9 h1:ESiK220/qE0aGxWdzKIvRH69iLiuN/PjoLTm69RoWtU= @@ -292,8 +301,11 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -327,6 +339,7 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -338,6 +351,7 @@ golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -370,6 +384,7 @@ golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -404,6 +419,7 @@ gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= diff --git a/grafana.ini b/grafana.ini new file mode 100644 index 00000000..fbdb61ef --- /dev/null +++ b/grafana.ini @@ -0,0 +1,4 @@ +[auth.extended_jwt] +enabled = true +expect_issuer = http://localhost:3000/ +expect_audience = http://localhost:3000/ diff --git a/pkg/plugin/app.go b/pkg/plugin/app.go index 21e7566a..8e99a0c1 100644 --- a/pkg/plugin/app.go +++ b/pkg/plugin/app.go @@ -2,6 +2,7 @@ package plugin import ( "context" + "fmt" "net/http" "github.com/grafana/grafana-llm-app/pkg/plugin/vector" @@ -46,13 +47,21 @@ func NewApp(appSettings backend.AppInstanceSettings) (instancemgmt.Instance, err var err error log.DefaultLogger.Debug("Creating vector service") - app.vectorService, err = vector.NewService(settings.EmbeddingSettings, settings.VectorStoreSettings) + httpOpts, err := appSettings.HTTPClientOptions() + if err != nil { + log.DefaultLogger.Error("Invalid HTTP settings", "err", err) + return nil, fmt.Errorf("invalid http settings: %w", err) + } + app.vectorService, err = vector.NewService( + settings.EmbeddingSettings, + settings.VectorStoreSettings, + httpOpts, + ) if err != nil { log.DefaultLogger.Error("Error creating vector service", "err", err) return nil, err } - log.DefaultLogger.Debug("App instance created") return &app, nil } diff --git a/pkg/plugin/vector/service.go b/pkg/plugin/vector/service.go index af8c4c80..42b5e7f0 100644 --- a/pkg/plugin/vector/service.go +++ b/pkg/plugin/vector/service.go @@ -5,25 +5,45 @@ package vector import ( "context" "fmt" + "net/http" + "os" + "strings" + gapi "github.com/grafana/grafana-api-golang-client" "github.com/grafana/grafana-llm-app/pkg/plugin/vector/embed" "github.com/grafana/grafana-llm-app/pkg/plugin/vector/store" + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana-plugin-sdk-go/backend/log" + "github.com/grafana/grafana-plugin-sdk-go/experimental/oauthtokenretriever" ) type Service interface { Search(ctx context.Context, collection string, query string, limit uint64) ([]store.SearchResult, error) + StartSync() Cancel() } type vectorService struct { embedder embed.Embedder - store store.ReadVectorStore + store store.VectorStore collections map[string]store.Collection - cancel context.CancelFunc + // cancel is a function to cancel the context used by the vector service + // and/or the underlying vector store. + cancel context.CancelFunc + + // httpClient is the http client used to make requests to the Grafana API. + httpClient *http.Client + // grafanaAppURL is the URL of the Grafana app. It is obtained from the + // `GF_APP_URL` environment variable. + grafanaAppURL string + // tokenRetriever is used to obtain OAuth2 tokens for the vector sync process. + tokenRetriever oauthtokenretriever.TokenRetriever + // ctx is the context used by the vector service. It is used to obtain + // OAuth2 tokens for the vector sync process. + ctx context.Context } -func NewService(embedSettings embed.Settings, storeSettings store.Settings) (Service, error) { +func NewService(embedSettings embed.Settings, storeSettings store.Settings, httpOpts httpclient.Options) (Service, error) { log.DefaultLogger.Debug("Creating embedder") em, err := embed.NewEmbedder(embedSettings) if err != nil { @@ -34,7 +54,7 @@ func NewService(embedSettings embed.Settings, storeSettings store.Settings) (Ser return nil, nil } log.DefaultLogger.Info("Creating vector store") - st, cancel, err := store.NewReadVectorStore(storeSettings) + st, cancel, err := store.NewVectorStore(storeSettings) if err != nil { return nil, fmt.Errorf("new vector store: %w", err) } @@ -46,12 +66,52 @@ func NewService(embedSettings embed.Settings, storeSettings store.Settings) (Ser for _, c := range storeSettings.Collections { collections[c.Name] = c } - return &vectorService{ + v := &vectorService{ embedder: em, store: st, collections: collections, cancel: cancel, - }, nil + } + + v.ctx = context.Background() + v.tokenRetriever, err = oauthtokenretriever.New() + if err != nil { + log.DefaultLogger.Warn("Error creating token retriever, vector sync will not run", "error", err) + return v, nil + } + + // The Grafana URL is required to obtain tokens later on + v.grafanaAppURL = strings.TrimRight(os.Getenv("GF_APP_URL"), "/") + if v.grafanaAppURL == "" { + // For debugging purposes only + v.grafanaAppURL = "http://localhost:3000" + } + + v.httpClient, err = httpclient.New(httpOpts) + if err != nil { + return nil, fmt.Errorf("httpclient new: %w", err) + } + + v.StartSync() + + return v, nil +} + +func (v *vectorService) grafanaClient(ctx context.Context) (*gapi.Client, error) { + token, err := v.tokenRetriever.Self(ctx) + if err != nil { + return nil, fmt.Errorf("get OAuth token for Grafana: %w", err) + } + g, err := gapi.New(v.grafanaAppURL, gapi.Config{ + APIKey: token, + Client: v.httpClient, + // OrgID must be '1' for now. + OrgID: 1, + }) + if err != nil { + return nil, fmt.Errorf("create Grafana client: %w", err) + } + return g, nil } func (v *vectorService) Search(ctx context.Context, collection string, query string, limit uint64) ([]store.SearchResult, error) { @@ -93,7 +153,7 @@ func (v *vectorService) Search(ctx context.Context, collection string, query str return results, nil } -func (v vectorService) Cancel() { +func (v *vectorService) Cancel() { if v.cancel != nil { v.cancel() } diff --git a/pkg/plugin/vector/store/qdrant.go b/pkg/plugin/vector/store/qdrant.go index 5e5d7df2..cc7b4d40 100644 --- a/pkg/plugin/vector/store/qdrant.go +++ b/pkg/plugin/vector/store/qdrant.go @@ -3,13 +3,16 @@ package store import ( "context" "crypto/tls" + "reflect" "github.com/grafana/grafana-plugin-sdk-go/backend/log" qdrant "github.com/qdrant/go-client/qdrant" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" ) type qdrantSettings struct { @@ -28,7 +31,7 @@ type qdrantStore struct { pointsClient qdrant.PointsClient } -func newQdrantStore(s qdrantSettings) (ReadVectorStore, func(), error) { +func newQdrantStore(s qdrantSettings) (VectorStore, func(), error) { var md *metadata.MD dialOptions := []grpc.DialOption{} if s.Secure { @@ -106,6 +109,126 @@ func (q *qdrantStore) Search(ctx context.Context, collection string, vector []fl return results, nil } +func (q *qdrantStore) CollectionExists(ctx context.Context, collection string) (bool, error) { + _, err := q.collectionsClient.Get(ctx, &qdrant.GetCollectionInfoRequest{ + CollectionName: collection, + }, grpc.WaitForReady(true)) + if err != nil { + st, ok := status.FromError(err) + if !ok { + return false, err + // Error was not a status error + } + if st.Code() == codes.NotFound { + return false, nil + } + return false, err + } + return true, nil +} + +func (q *qdrantStore) CreateCollection(ctx context.Context, collection string, size uint64) error { + _, err := q.collectionsClient.Create(ctx, &qdrant.CreateCollection{ + CollectionName: collection, + VectorsConfig: &qdrant.VectorsConfig{ + Config: &qdrant.VectorsConfig_Params{ + Params: &qdrant.VectorParams{ + Size: size, + // TODO: make this customizable + Distance: qdrant.Distance_Cosine, + }, + }, + }, + }) + return err +} + +func (q *qdrantStore) PointExists(ctx context.Context, collection string, id uint64) (bool, error) { + point, err := q.pointsClient.Get(ctx, &qdrant.GetPoints{ + CollectionName: collection, + Ids: []*qdrant.PointId{ + {PointIdOptions: &qdrant.PointId_Num{Num: id}}, + }, + }, grpc.WaitForReady(true)) + if err != nil { + st, ok := status.FromError(err) + if !ok { + return false, err + // Error was not a status error + } + if st.Code() == codes.NotFound { + return false, nil + } + return false, err + } + if point.Result == nil { + return false, nil + } + return true, nil +} + +func toQdrantValue(v reflect.Value) *qdrant.Value { + out := &qdrant.Value{} + switch v.Kind() { + + // Atoms + case reflect.Invalid: + out.Kind = &qdrant.Value_NullValue{NullValue: qdrant.NullValue_NULL_VALUE} + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + out.Kind = &qdrant.Value_IntegerValue{IntegerValue: int64(v.Uint())} + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + out.Kind = &qdrant.Value_IntegerValue{IntegerValue: v.Int()} + case reflect.Float32, reflect.Float64: + out.Kind = &qdrant.Value_DoubleValue{DoubleValue: v.Float()} + case reflect.Bool: + out.Kind = &qdrant.Value_BoolValue{BoolValue: v.Bool()} + case reflect.String: + out.Kind = &qdrant.Value_StringValue{StringValue: v.String()} + + // Slices and arrays + case reflect.Slice, reflect.Array: + values := make([]*qdrant.Value, 0, v.Len()) + for i := 0; i < v.Len(); i++ { + values = append(values, toQdrantValue(v.Index(i))) + } + out.Kind = &qdrant.Value_ListValue{ListValue: &qdrant.ListValue{Values: values}} + + // Maps and structs + case reflect.Map: + keys := v.MapKeys() + fields := make(map[string]*qdrant.Value, len(keys)) + for _, key := range keys { + if key.Kind() == reflect.String { + fields[key.String()] = toQdrantValue(v.MapIndex(key)) + } else { + log.DefaultLogger.Warn("unsupported map key type", "type", key.Kind()) + } + } + out.Kind = &qdrant.Value_StructValue{StructValue: &qdrant.Struct{Fields: fields}} + case reflect.Struct: + fields := make(map[string]*qdrant.Value, v.NumField()) + for i := 0; i < v.NumField(); i++ { + fields[v.Type().Field(i).Name] = toQdrantValue(v.Field(i)) + } + out.Kind = &qdrant.Value_StructValue{StructValue: &qdrant.Struct{Fields: fields}} + + // Pointers and interfaces + case reflect.Ptr: + if v.IsNil() { + out.Kind = &qdrant.Value_NullValue{NullValue: qdrant.NullValue_NULL_VALUE} + } else { + out = toQdrantValue(v.Elem()) + } + case reflect.Interface: + if v.IsNil() { + out.Kind = &qdrant.Value_NullValue{NullValue: qdrant.NullValue_NULL_VALUE} + } else { + out = toQdrantValue(v.Elem()) + } + } + return out +} + func fromQdrantValue(in *qdrant.Value) any { switch v := in.Kind.(type) { case *qdrant.Value_NullValue: @@ -133,3 +256,32 @@ func fromQdrantValue(in *qdrant.Value) any { } return nil } + +func (q *qdrantStore) UpsertColumnar(ctx context.Context, collection string, ids []uint64, embeddings [][]float32, payloads []Payload) error { + waitUpsert := false + upsertPoints := make([]*qdrant.PointStruct, 0, len(ids)) + for i, id := range ids { + payload := make(map[string]*qdrant.Value, len(payloads[i])) + for k, v := range payloads[i] { + if newV := toQdrantValue(reflect.ValueOf(v)); newV != nil { + payload[k] = newV + } else { + log.DefaultLogger.Warn("unsupported payload value type", "key", k, "value", v) + } + } + point := &qdrant.PointStruct{ + Id: &qdrant.PointId{ + PointIdOptions: &qdrant.PointId_Num{Num: id}, + }, + Vectors: &qdrant.Vectors{VectorsOptions: &qdrant.Vectors_Vector{Vector: &qdrant.Vector{Data: embeddings[i]}}}, + Payload: payload, + } + upsertPoints = append(upsertPoints, point) + } + _, err := q.pointsClient.Upsert(ctx, &qdrant.UpsertPoints{ + CollectionName: collection, + Points: upsertPoints, + Wait: &waitUpsert, + }, grpc.WaitForReady(true)) + return err +} diff --git a/pkg/plugin/vector/store/store.go b/pkg/plugin/vector/store/store.go index 1c768ee2..3578a7f3 100644 --- a/pkg/plugin/vector/store/store.go +++ b/pkg/plugin/vector/store/store.go @@ -2,6 +2,7 @@ package store import ( "context" + "fmt" "github.com/grafana/grafana-plugin-sdk-go/backend/log" ) @@ -19,9 +20,11 @@ type Collection struct { Model string `json:"model"` } +type Payload map[string]any + type SearchResult struct { - Payload map[string]any `json:"payload"` - Score float64 `json:"score"` + Payload Payload `json:"payload"` + Score float64 `json:"score"` } type ReadVectorStore interface { @@ -34,7 +37,7 @@ type WriteVectorStore interface { CollectionExists(ctx context.Context, collection string) (bool, error) CreateCollection(ctx context.Context, collection string, size uint64) error PointExists(ctx context.Context, collection string, id uint64) (bool, error) - UpsertColumnar(ctx context.Context, collection string, ids []uint64, embeddings [][]float32, payloadJSONs []string) error + UpsertColumnar(ctx context.Context, collection string, ids []uint64, embeddings [][]float32, payloads []Payload) error } type VectorStore interface { @@ -64,7 +67,14 @@ func NewReadVectorStore(s Settings) (ReadVectorStore, context.CancelFunc, error) return nil, nil, nil } -func NewVectorStore(s Settings) (VectorStore, error) { - // TODO: Implement write vector store. - return nil, nil +func NewVectorStore(s Settings) (VectorStore, context.CancelFunc, error) { + switch VectorStoreType(s.Type) { + case VectorStoreTypeGrafanaVectorAPI: + log.DefaultLogger.Debug("Grafana Vector API can not yet be used for vector sync") + return nil, nil, fmt.Errorf("unimplemented") + case VectorStoreTypeQdrant: + log.DefaultLogger.Debug("Creating Qdrant store") + return newQdrantStore(s.Qdrant) + } + return nil, nil, nil } diff --git a/pkg/plugin/vector/sync.go b/pkg/plugin/vector/sync.go new file mode 100644 index 00000000..c3c2f751 --- /dev/null +++ b/pkg/plugin/vector/sync.go @@ -0,0 +1,173 @@ +package vector + +import ( + "context" + "encoding/json" + "fmt" + "hash/fnv" + "math" + "runtime/debug" + "time" + + "github.com/grafana/grafana-llm-app/pkg/plugin/vector/store" + "github.com/grafana/grafana-plugin-sdk-go/backend/log" +) + +const ( + GrafanaDashboardsCollection = "grafana.core.dashboards" +) + +// startVectorSync starts a ticker which periodically syncs Grafana metadata to the vector store. +func (v *vectorService) StartSync() { + go func() { + log.DefaultLogger.Info("Running initial vector sync") + if err := v.syncVectorStore(v.ctx); err != nil { + log.DefaultLogger.Error("Error syncing vector store", "error", err) + } + log.DefaultLogger.Info("Starting vector sync ticker") + // TODO: make sync interval configurable + ticker := time.NewTicker(15 * time.Minute) + defer ticker.Stop() + for { + select { + case <-v.ctx.Done(): + return + case <-ticker.C: + if err := v.syncVectorStore(v.ctx); err != nil { + log.DefaultLogger.Error("Error syncing vector store", "error", err) + } + } + } + }() +} + +// syncVectorStore syncs Grafana metadata to the vector store. +func (v *vectorService) syncVectorStore(ctx context.Context) (err error) { + defer func() { + if pan := recover(); pan != nil { + err = fmt.Errorf("sync process panicked: %s %s", pan, debug.Stack()) + } + }() + if err = v.syncDashboardsToVectorStore(ctx); err != nil { + log.DefaultLogger.Error("Error syncing dashboards to vector store", "error", err) + return err + } + return nil +} + +// syncDashboardsToVectorStore syncs Grafana dashboards to the vector store. +// TODO: refactor this later to be generic over the type of metadata in some way. +func (v *vectorService) syncDashboardsToVectorStore(ctx context.Context) error { + log.DefaultLogger.Info("Syncing dashboards to vector store") + + c, ok := v.collections[GrafanaDashboardsCollection] + if !ok { + return fmt.Errorf("collection config not found for %s", GrafanaDashboardsCollection) + } + + if exists, err := v.store.CollectionExists(ctx, GrafanaDashboardsCollection); err == nil && !exists { + log.DefaultLogger.Info( + "Creating dashboard collection", + "collection", GrafanaDashboardsCollection, + "dimensions", c.Dimension) + err = v.store.CreateCollection(ctx, GrafanaDashboardsCollection, uint64(c.Dimension)) + if err != nil { + return fmt.Errorf("create dashboard collection: %w", err) + } + } + + log.DefaultLogger.Info("Creating Grafana client") + client, err := v.grafanaClient(ctx) + if err != nil { + return fmt.Errorf("create Grafana client: %w", err) + } + dashboards, err := client.Dashboards() + if err != nil { + log.DefaultLogger.Error("Error fetching dashboards", "err", err) + return fmt.Errorf("get dashboards: %w", err) + } + log.DefaultLogger.Info("Fetched dashboards", "count", len(dashboards)) + // chunkSize is the number of dashboards to embed and add to the vector store at once. + chunkSize := 100 + ids := make([]uint64, 0, chunkSize) + embeddings := make([][]float32, 0, chunkSize) + payloads := make([]store.Payload, 0, chunkSize) + + for i := 0; i < len(dashboards); i += chunkSize { + ids = ids[:0] + embeddings = embeddings[:0] + payloads = payloads[:0] + + chunk := dashboards[i:int(math.Min(float64(i+chunkSize), float64(len(dashboards))))] + + for j, folderDashboard := range chunk { + log.DefaultLogger.Debug("Fetching dashboard", "uid", folderDashboard.UID) + dashboard, err := client.DashboardByUID(folderDashboard.UID) + if err != nil { + log.DefaultLogger.Warn("Unable to fetch dashboard", "uid", folderDashboard.UID, "err", err) + } + model := store.Payload{ + "title": folderDashboard.Title, + "description": dashboard.Model["description"], + } + // All these type assertions kinda suck, but we don't have the raw JSON model. + // I guess we could marshal and unmarshal into a custom type? + if panels, ok := dashboard.Model["panels"]; ok { + if panels, ok := panels.([]any); ok { + modelPanels := make([]map[string]any, len(panels)) + for i, panel := range panels { + if p, ok := panel.(map[string]any); ok { + modelPanels[i] = map[string]any{ + "title": p["title"], + "description": p["description"], + } + } + } + log.DefaultLogger.Info("panels", "panels", modelPanels) + model["panels"] = modelPanels + log.DefaultLogger.Info("model", "model", model) + } + } + + jdoc, err := json.Marshal(model) + if err != nil { + log.DefaultLogger.Warn("Unable to marshal dashboard", "uid", folderDashboard.UID, "err", err) + continue + } + + // Check if dashboard exists in vector store + hash := fnv.New64a() + hash.Write([]byte(jdoc)) + id := hash.Sum64() + if exists, err := v.store.PointExists(ctx, GrafanaDashboardsCollection, id); err != nil { + log.DefaultLogger.Warn("Error checking whether vector exists", "collection", GrafanaDashboardsCollection, "id", id, "err", err) + continue + } else if exists { + log.DefaultLogger.Debug("Vector already exists, skipping", "collection", GrafanaDashboardsCollection, "id", id, "err", err) + continue + } + + // If we're here, we have a new dashboard to embed and add. + log.DefaultLogger.Debug("Getting embeddings for dashboard", "collection", GrafanaDashboardsCollection, "index", i+j, "count", len(dashboards)) + // TODO: process the dashboard JSON. + e, err := v.embedder.Embed(ctx, c.Model, string(jdoc)) + if err != nil { + log.DefaultLogger.Warn("Error getting embeddings", "collection", GrafanaDashboardsCollection, "err", err) + continue + } + ids = append(ids, id) + embeddings = append(embeddings, e) + payloads = append(payloads, model) + } + if len(ids) == 0 { + log.DefaultLogger.Debug("No new embeddings to add") + return nil + } + log.DefaultLogger.Debug("Adding embeddings to vector DB", "collection", GrafanaDashboardsCollection, "count", len(embeddings)) + err := v.store.UpsertColumnar(ctx, GrafanaDashboardsCollection, ids, embeddings, payloads) + if err != nil { + return fmt.Errorf("upsert columnar: %w", err) + } + } + return nil +} diff --git a/provisioning/dashboards/default.yaml b/provisioning/dashboards/default.yaml new file mode 100644 index 00000000..168ff28d --- /dev/null +++ b/provisioning/dashboards/default.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 + +providers: + - name: Default # A uniquely identifiable name for the provider + type: file + options: + path: /etc/grafana/provisioning/dashboards diff --git a/provisioning/dashboards/mimir-usage.json b/provisioning/dashboards/mimir-usage.json new file mode 100644 index 00000000..6ac6ccad --- /dev/null +++ b/provisioning/dashboards/mimir-usage.json @@ -0,0 +1,134 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana", + "uid": "grafana" + }, + "description": "CPU usage of Mimir", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "CPU usage", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Mimir resource usage", + "uid": "b3558bf0-ab3f-4bf2-848c-f2e2f6ce2c56", + "version": 1, + "weekStart": "" +} diff --git a/provisioning/datasources/robust-perception.yaml b/provisioning/datasources/robust-perception.yaml new file mode 100644 index 00000000..fed5afb8 --- /dev/null +++ b/provisioning/datasources/robust-perception.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Robust Perception + id: 1 + uid: robustperception + type: prometheus + access: proxy + url: http://demo.robustperception.io:9090 + isDefault: true diff --git a/provisioning/plugins/grafana-llm-app.yaml b/provisioning/plugins/grafana-llm-app.yaml index c275cb7c..f9c5d336 100644 --- a/provisioning/plugins/grafana-llm-app.yaml +++ b/provisioning/plugins/grafana-llm-app.yaml @@ -13,7 +13,7 @@ apps: qdrant: address: qdrant:6334 collections: - - name: grafana:core:dashboards + - name: grafana.core.dashboards model: text-embedding-ada-002 dimension: 1536 diff --git a/src/plugin.json b/src/plugin.json index 39996b1e..74b09546 100644 --- a/src/plugin.json +++ b/src/plugin.json @@ -29,8 +29,26 @@ "updated": "%TODAY%" }, "includes": [], + "externalServiceRegistration": { + "self": { + "permissions": [ + { + "action": "dashboards:read", + "scope": "dashboards:*" + }, + { + "action": "dashboards:read", + "scope": "folders:*" + }, + { + "action": "folders:read", + "scope": "folders:*" + } + ] + } + }, "dependencies": { - "grafanaDependency": ">=9.5.2", + "grafanaDependency": ">=10.1.0", "plugins": [] } }