From 7925b2a8780cd63b4b8e9e929a52c7cad7243071 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Wed, 20 Sep 2023 16:41:29 -0400 Subject: [PATCH 01/28] added the azure openai proxy resource endpoint --- pkg/plugin/resources.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pkg/plugin/resources.go b/pkg/plugin/resources.go index 2f81e408..dbff8b5e 100644 --- a/pkg/plugin/resources.go +++ b/pkg/plugin/resources.go @@ -1,7 +1,10 @@ package plugin import ( + "bytes" "encoding/json" + "fmt" + "io/ioutil" "net/http" "net/http/httputil" "net/url" @@ -63,6 +66,31 @@ func newOpenAIProxy() http.Handler { } } +func newAzureOpenAIProxy() http.Handler { + director := func(req *http.Request) { + config := httpadapter.PluginConfigFromContext(req.Context()) + settings := loadSettings(*config.AppInstanceSettings) + + bodyBytes, _ := ioutil.ReadAll(req.Body) + var requestBody map[string]interface{} + json.Unmarshal(bodyBytes, &requestBody) + + req.URL.Scheme = "https" + req.URL.Host = fmt.Sprintf("%s.openai.azure.com", requestBody["resource"]) + req.URL.Path = fmt.Sprintf("/openai/deployments/%s/chat/completions", requestBody["deployment"]) + req.Header.Add("api-key", settings.OpenAI.apiKey) + + // Remove extra fields + delete(requestBody, "resource") + delete(requestBody, "deployment") + + newBodyBytes, _ := json.Marshal(requestBody) + req.Body = ioutil.NopCloser(bytes.NewBuffer(newBodyBytes)) + req.ContentLength = int64(len(newBodyBytes)) + } + return &httputil.ReverseProxy{Director: director} +} + type vectorSearchRequest struct { Query string `json:"query"` Collection string `json:"collection"` @@ -110,5 +138,6 @@ func (a *App) registerRoutes(mux *http.ServeMux) { mux.HandleFunc("/ping", a.handlePing) mux.HandleFunc("/echo", a.handleEcho) mux.Handle("/openai/", newOpenAIProxy()) + mux.Handle("/azure/", newAzureOpenAIProxy()) mux.HandleFunc("/vector/search", a.handleVectorSearch) } From f720acb67396d0a22c2b054dfdd0a0f79fb70a16 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Wed, 20 Sep 2023 16:55:09 -0400 Subject: [PATCH 02/28] add api-version url paramter --- pkg/plugin/resources.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/plugin/resources.go b/pkg/plugin/resources.go index dbff8b5e..d7268215 100644 --- a/pkg/plugin/resources.go +++ b/pkg/plugin/resources.go @@ -78,6 +78,7 @@ func newAzureOpenAIProxy() http.Handler { req.URL.Scheme = "https" req.URL.Host = fmt.Sprintf("%s.openai.azure.com", requestBody["resource"]) req.URL.Path = fmt.Sprintf("/openai/deployments/%s/chat/completions", requestBody["deployment"]) + req.URL.RawQuery = "api-version=2023-03-15-preview" req.Header.Add("api-key", settings.OpenAI.apiKey) // Remove extra fields From 573de4b0e32d63e01b5d6f2c5d1e15909073e607 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Wed, 20 Sep 2023 17:16:23 -0400 Subject: [PATCH 03/28] added Azure openai setting --- pkg/plugin/resources.go | 5 ++--- pkg/plugin/settings.go | 10 +++++++++- provisioning/plugins/grafana-llm-app.yaml | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pkg/plugin/resources.go b/pkg/plugin/resources.go index d7268215..e64ed67f 100644 --- a/pkg/plugin/resources.go +++ b/pkg/plugin/resources.go @@ -76,13 +76,12 @@ func newAzureOpenAIProxy() http.Handler { json.Unmarshal(bodyBytes, &requestBody) req.URL.Scheme = "https" - req.URL.Host = fmt.Sprintf("%s.openai.azure.com", requestBody["resource"]) + req.URL.Host = fmt.Sprintf("%s.openai.azure.com", settings.AzureOpenAI.ResourceName) req.URL.Path = fmt.Sprintf("/openai/deployments/%s/chat/completions", requestBody["deployment"]) req.URL.RawQuery = "api-version=2023-03-15-preview" - req.Header.Add("api-key", settings.OpenAI.apiKey) + req.Header.Add("api-key", settings.AzureOpenAI.apiKey) // Remove extra fields - delete(requestBody, "resource") delete(requestBody, "deployment") newBodyBytes, _ := json.Marshal(requestBody) diff --git a/pkg/plugin/settings.go b/pkg/plugin/settings.go index 9e4e975c..04a5b520 100644 --- a/pkg/plugin/settings.go +++ b/pkg/plugin/settings.go @@ -8,6 +8,7 @@ import ( ) const openAIKey = "openAIKey" +const azureOpenAIKey = "azureOpenAIKey" type OpenAISettings struct { URL string `json:"url"` @@ -15,8 +16,14 @@ type OpenAISettings struct { apiKey string } +type AzureOpenAISettings struct { + ResourceName string `json:"resource"` + apiKey string +} + type Settings struct { - OpenAI OpenAISettings `json:"openAI"` + OpenAI OpenAISettings `json:"openAI"` + AzureOpenAI AzureOpenAISettings `json:"azureOpenAI"` Vector vector.VectorSettings `json:"vector"` } @@ -30,5 +37,6 @@ func loadSettings(appSettings backend.AppInstanceSettings) Settings { _ = json.Unmarshal(appSettings.JSONData, &settings) settings.OpenAI.apiKey = appSettings.DecryptedSecureJSONData[openAIKey] + settings.AzureOpenAI.apiKey = appSettings.DecryptedSecureJSONData[azureOpenAIKey] return settings } diff --git a/provisioning/plugins/grafana-llm-app.yaml b/provisioning/plugins/grafana-llm-app.yaml index 5e656bfb..89140b1e 100644 --- a/provisioning/plugins/grafana-llm-app.yaml +++ b/provisioning/plugins/grafana-llm-app.yaml @@ -12,6 +12,8 @@ apps: type: qdrant qdrant: address: qdrant:6334 + azureopenai: + url: https://api.openai.com secureJsonData: openAIKey: $OPENAI_API_KEY From 8c3cfd8a5a618eec360660ecc197e26588c5f846 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Wed, 20 Sep 2023 17:16:58 -0400 Subject: [PATCH 04/28] Added azure api key --- provisioning/plugins/grafana-llm-app.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/provisioning/plugins/grafana-llm-app.yaml b/provisioning/plugins/grafana-llm-app.yaml index 89140b1e..b29fd15b 100644 --- a/provisioning/plugins/grafana-llm-app.yaml +++ b/provisioning/plugins/grafana-llm-app.yaml @@ -17,3 +17,4 @@ apps: secureJsonData: openAIKey: $OPENAI_API_KEY + azureOpenAIKey: $AZURE_OPENAI_API_KEY From 5128a3ac6657a96fe2895eafc0768cab0c0a32bc Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Wed, 20 Sep 2023 17:17:52 -0400 Subject: [PATCH 05/28] updated resource field --- provisioning/plugins/grafana-llm-app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioning/plugins/grafana-llm-app.yaml b/provisioning/plugins/grafana-llm-app.yaml index b29fd15b..4bff404f 100644 --- a/provisioning/plugins/grafana-llm-app.yaml +++ b/provisioning/plugins/grafana-llm-app.yaml @@ -13,7 +13,7 @@ apps: qdrant: address: qdrant:6334 azureopenai: - url: https://api.openai.com + resource: grafana-machine-learning-dev secureJsonData: openAIKey: $OPENAI_API_KEY From 514f79685dbe1c8f6d26fc9373d8f14195ca8a73 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Wed, 20 Sep 2023 17:28:59 -0400 Subject: [PATCH 06/28] added streaming azure path --- pkg/plugin/stream.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/plugin/stream.go b/pkg/plugin/stream.go index baedf9c7..c51e096d 100644 --- a/pkg/plugin/stream.go +++ b/pkg/plugin/stream.go @@ -14,6 +14,7 @@ import ( ) const openAIChatCompletionsPath = "openai/v1/chat/completions" +const azureOpenAIChatCompletionsPath = "azure/completions" type chatCompletionsMessage struct { Role string `json:"role"` @@ -32,7 +33,7 @@ func (a *App) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamR resp := &backend.SubscribeStreamResponse{ Status: backend.SubscribeStreamStatusNotFound, } - if req.Path == openAIChatCompletionsPath { + if req.Path == openAIChatCompletionsPath || req.Path == azureOpenAIChatCompletionsPath { resp.Status = backend.SubscribeStreamStatusOK } return resp, nil @@ -135,7 +136,7 @@ func (a *App) runOpenAIChatCompletionsStream(ctx context.Context, req *backend.R func (a *App) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { log.DefaultLogger.Debug(fmt.Sprintf("RunStream: %s", req.Path), "data", string(req.Data)) - if req.Path == openAIChatCompletionsPath { + if req.Path == openAIChatCompletionsPath || req.Path == azureOpenAIChatCompletionsPath { return a.runOpenAIChatCompletionsStream(ctx, req, sender) } return fmt.Errorf("unknown stream path: %s", req.Path) From ac73e60bfb3f5b3eb588441a42fcdf03f6ebe102 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Thu, 21 Sep 2023 16:20:43 -0400 Subject: [PATCH 07/28] merged azure and openai proxy --- pkg/plugin/resources.go | 41 ++++++++++++++++++++++++++--------------- pkg/plugin/settings.go | 16 +++++----------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/pkg/plugin/resources.go b/pkg/plugin/resources.go index e64ed67f..2b2f9709 100644 --- a/pkg/plugin/resources.go +++ b/pkg/plugin/resources.go @@ -66,27 +66,39 @@ func newOpenAIProxy() http.Handler { } } -func newAzureOpenAIProxy() http.Handler { +func newOpenAIProxy2() http.Handler { director := func(req *http.Request) { config := httpadapter.PluginConfigFromContext(req.Context()) settings := loadSettings(*config.AppInstanceSettings) - bodyBytes, _ := ioutil.ReadAll(req.Body) - var requestBody map[string]interface{} - json.Unmarshal(bodyBytes, &requestBody) + u, _ := url.Parse(settings.OpenAI.URL) + req.URL.Scheme = u.Scheme + req.URL.Host = u.Host - req.URL.Scheme = "https" - req.URL.Host = fmt.Sprintf("%s.openai.azure.com", settings.AzureOpenAI.ResourceName) - req.URL.Path = fmt.Sprintf("/openai/deployments/%s/chat/completions", requestBody["deployment"]) - req.URL.RawQuery = "api-version=2023-03-15-preview" - req.Header.Add("api-key", settings.AzureOpenAI.apiKey) + if settings.OpenAI.UseAzure { + // Map model to deployment + bodyBytes, _ := ioutil.ReadAll(req.Body) + var requestBody map[string]interface{} + json.Unmarshal(bodyBytes, &requestBody) - // Remove extra fields - delete(requestBody, "deployment") + deployment := settings.OpenAI.AzureMapping[requestBody["model"].(string)] - newBodyBytes, _ := json.Marshal(requestBody) - req.Body = ioutil.NopCloser(bytes.NewBuffer(newBodyBytes)) - req.ContentLength = int64(len(newBodyBytes)) + req.URL.Path = fmt.Sprintf("/openai/deployments/%s/%s", deployment, strings.TrimPrefix(req.URL.Path, "/azure")) + req.Header.Add("api-key", settings.OpenAI.apiKey) + req.URL.RawQuery = "api-version=2023-03-15-preview" + + // Remove extra fields + delete(requestBody, "model") + + newBodyBytes, _ := json.Marshal(requestBody) + req.Body = ioutil.NopCloser(bytes.NewBuffer(newBodyBytes)) + req.ContentLength = int64(len(newBodyBytes)) + } else { + req.URL.Path = strings.TrimPrefix(req.URL.Path, "/openai") + req.Header.Add("Authorization", "Bearer "+settings.OpenAI.apiKey) + req.Header.Add("OpenAI-Organization", settings.OpenAI.OrganizationID) + + } } return &httputil.ReverseProxy{Director: director} } @@ -138,6 +150,5 @@ func (a *App) registerRoutes(mux *http.ServeMux) { mux.HandleFunc("/ping", a.handlePing) mux.HandleFunc("/echo", a.handleEcho) mux.Handle("/openai/", newOpenAIProxy()) - mux.Handle("/azure/", newAzureOpenAIProxy()) mux.HandleFunc("/vector/search", a.handleVectorSearch) } diff --git a/pkg/plugin/settings.go b/pkg/plugin/settings.go index 04a5b520..ba488971 100644 --- a/pkg/plugin/settings.go +++ b/pkg/plugin/settings.go @@ -8,22 +8,17 @@ import ( ) const openAIKey = "openAIKey" -const azureOpenAIKey = "azureOpenAIKey" type OpenAISettings struct { - URL string `json:"url"` - OrganizationID string `json:"organizationId"` + URL string `json:"url"` + OrganizationID string `json:"organizationId"` + UseAzure bool `json:"azureOpenAI"` + AzureMapping map[string]string `json:"azureOpenAIModelMapping"` apiKey string } -type AzureOpenAISettings struct { - ResourceName string `json:"resource"` - apiKey string -} - type Settings struct { - OpenAI OpenAISettings `json:"openAI"` - AzureOpenAI AzureOpenAISettings `json:"azureOpenAI"` + OpenAI OpenAISettings `json:"openAI"` Vector vector.VectorSettings `json:"vector"` } @@ -37,6 +32,5 @@ func loadSettings(appSettings backend.AppInstanceSettings) Settings { _ = json.Unmarshal(appSettings.JSONData, &settings) settings.OpenAI.apiKey = appSettings.DecryptedSecureJSONData[openAIKey] - settings.AzureOpenAI.apiKey = appSettings.DecryptedSecureJSONData[azureOpenAIKey] return settings } From 12a4d082e3dc449331466e117eb26b3fbc221060 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Thu, 21 Sep 2023 22:26:28 -0400 Subject: [PATCH 08/28] removed azure openai streaming logic --- pkg/plugin/stream.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/plugin/stream.go b/pkg/plugin/stream.go index c51e096d..baedf9c7 100644 --- a/pkg/plugin/stream.go +++ b/pkg/plugin/stream.go @@ -14,7 +14,6 @@ import ( ) const openAIChatCompletionsPath = "openai/v1/chat/completions" -const azureOpenAIChatCompletionsPath = "azure/completions" type chatCompletionsMessage struct { Role string `json:"role"` @@ -33,7 +32,7 @@ func (a *App) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamR resp := &backend.SubscribeStreamResponse{ Status: backend.SubscribeStreamStatusNotFound, } - if req.Path == openAIChatCompletionsPath || req.Path == azureOpenAIChatCompletionsPath { + if req.Path == openAIChatCompletionsPath { resp.Status = backend.SubscribeStreamStatusOK } return resp, nil @@ -136,7 +135,7 @@ func (a *App) runOpenAIChatCompletionsStream(ctx context.Context, req *backend.R func (a *App) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { log.DefaultLogger.Debug(fmt.Sprintf("RunStream: %s", req.Path), "data", string(req.Data)) - if req.Path == openAIChatCompletionsPath || req.Path == azureOpenAIChatCompletionsPath { + if req.Path == openAIChatCompletionsPath { return a.runOpenAIChatCompletionsStream(ctx, req, sender) } return fmt.Errorf("unknown stream path: %s", req.Path) From 9dfbc7297a4e2c1fb05bb84639136cbabcbdec55 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Fri, 22 Sep 2023 09:36:52 -0400 Subject: [PATCH 09/28] added streaming proxy settings --- pkg/plugin/stream.go | 69 +++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/pkg/plugin/stream.go b/pkg/plugin/stream.go index baedf9c7..6252fdc1 100644 --- a/pkg/plugin/stream.go +++ b/pkg/plugin/stream.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "strings" "github.com/grafana/grafana-plugin-sdk-go/backend" @@ -20,11 +21,11 @@ type chatCompletionsMessage struct { Content string `json:"content"` } -type chatCompletionsRequest struct { - Model string `json:"model"` - Messages []chatCompletionsMessage `json:"messages"` - Stream bool `json:"stream"` -} +// type chatCompletionsRequest struct { +// Model string `json:"model"` +// Messages []chatCompletionsMessage `json:"messages"` +// Stream bool `json:"stream"` +// } func (a *App) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) { log.DefaultLogger.Debug(fmt.Sprintf("SubscribeStream: %s", req.Path)) @@ -39,29 +40,51 @@ func (a *App) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamR } func (a *App) runOpenAIChatCompletionsStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { - // Deserialize request data. - incomingBody := chatCompletionsRequest{Stream: true} - err := json.Unmarshal(req.Data, &incomingBody) - if err != nil { - return err - } - // Load app settings. settings := loadSettings(*req.PluginContext.AppInstanceSettings) + var requestBody map[string]interface{} + json.Unmarshal(req.Data, &requestBody) + + // set stream to true + requestBody["stream"] = true + + u, _ := url.Parse(settings.OpenAI.URL) + + var outgoingBody []byte + var err error + + if settings.OpenAI.UseAzure { + // Map model to deployment + + settings.OpenAI.AzureMapping = map[string]string{ + "gpt-3.5-turbo": "gpt-35-turbo", + } + + deployment := settings.OpenAI.AzureMapping[requestBody["model"].(string)] + + apiPath := strings.TrimPrefix(req.Path, "openai/v1/") + + u.Path = fmt.Sprintf("/openai/deployments/%s/%s", deployment, apiPath) + u.RawQuery = "api-version=2023-03-15-preview" + + // Remove extra fields + delete(requestBody, "model") + + } else { + u.Path = strings.TrimPrefix(req.Path, "openai") - // Create and send OpenAI request. - outgoingBody, err := json.Marshal(incomingBody) - if err != nil { - return err } - path := strings.TrimPrefix(req.Path, "openai") - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, settings.OpenAI.URL+path, bytes.NewReader(outgoingBody)) - if err != nil { - return fmt.Errorf("proxy: stream: error creating request: %w", err) + + outgoingBody, err = json.Marshal(requestBody) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), bytes.NewReader(outgoingBody)) + + if settings.OpenAI.UseAzure { + httpReq.Header.Set("api-key", settings.OpenAI.apiKey) + } else { + httpReq.Header.Set("Authorization", "Bearer "+settings.OpenAI.apiKey) + httpReq.Header.Set("OpenAI-Organization", settings.OpenAI.OrganizationID) } - httpReq.Header.Set("Authorization", fmt.Sprintf("Bearer %s", settings.OpenAI.apiKey)) - httpReq.Header.Set("OpenAI-Organization", settings.OpenAI.OrganizationID) - httpReq.Header.Set("Content-Type", "application/json") + lastEventID := "" // no last event id eventStream, err := eventsource.SubscribeWithRequest(lastEventID, httpReq) if err != nil { From 2ef99e6a08fce9c6c606fda9986ffca22efb00c0 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Fri, 22 Sep 2023 09:37:18 -0400 Subject: [PATCH 10/28] update setting to match frontend --- pkg/plugin/settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/settings.go b/pkg/plugin/settings.go index ba488971..f98ec289 100644 --- a/pkg/plugin/settings.go +++ b/pkg/plugin/settings.go @@ -12,7 +12,7 @@ const openAIKey = "openAIKey" type OpenAISettings struct { URL string `json:"url"` OrganizationID string `json:"organizationId"` - UseAzure bool `json:"azureOpenAI"` + UseAzure bool `json:"useAzure"` AzureMapping map[string]string `json:"azureOpenAIModelMapping"` apiKey string } From bad2b628f905e743371c2aa3c670a9c0cc9e0834 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Fri, 22 Sep 2023 09:38:45 -0400 Subject: [PATCH 11/28] fixed resource proxy routing --- pkg/plugin/resources.go | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/pkg/plugin/resources.go b/pkg/plugin/resources.go index 2b2f9709..500f9cc4 100644 --- a/pkg/plugin/resources.go +++ b/pkg/plugin/resources.go @@ -51,22 +51,6 @@ func (a *App) handleEcho(w http.ResponseWriter, req *http.Request) { } func newOpenAIProxy() http.Handler { - return &httputil.ReverseProxy{ - Rewrite: func(r *httputil.ProxyRequest) { - config := httpadapter.PluginConfigFromContext(r.In.Context()) - settings := loadSettings(*config.AppInstanceSettings) - u, _ := url.Parse(settings.OpenAI.URL) - r.SetURL(u) - r.Out.Header.Set("Authorization", "Bearer "+settings.OpenAI.apiKey) - organizationID := settings.OpenAI.OrganizationID - r.Out.Header.Set("OpenAI-Organization", organizationID) - r.Out.URL.Path = strings.TrimPrefix(r.In.URL.Path, "/openai") - log.DefaultLogger.Info("proxying to url", "url", r.Out.URL.String()) - }, - } -} - -func newOpenAIProxy2() http.Handler { director := func(req *http.Request) { config := httpadapter.PluginConfigFromContext(req.Context()) settings := loadSettings(*config.AppInstanceSettings) @@ -81,9 +65,13 @@ func newOpenAIProxy2() http.Handler { var requestBody map[string]interface{} json.Unmarshal(bodyBytes, &requestBody) + settings.OpenAI.AzureMapping = map[string]string{ + "gpt-3.5-turbo": "gpt-35-turbo", + } + deployment := settings.OpenAI.AzureMapping[requestBody["model"].(string)] - req.URL.Path = fmt.Sprintf("/openai/deployments/%s/%s", deployment, strings.TrimPrefix(req.URL.Path, "/azure")) + req.URL.Path = fmt.Sprintf("/openai/deployments/%s/%s", deployment, strings.TrimPrefix(req.URL.Path, "/openai/v1")) req.Header.Add("api-key", settings.OpenAI.apiKey) req.URL.RawQuery = "api-version=2023-03-15-preview" @@ -97,8 +85,8 @@ func newOpenAIProxy2() http.Handler { req.URL.Path = strings.TrimPrefix(req.URL.Path, "/openai") req.Header.Add("Authorization", "Bearer "+settings.OpenAI.apiKey) req.Header.Add("OpenAI-Organization", settings.OpenAI.OrganizationID) - } + log.DefaultLogger.Info(fmt.Sprintf("UUUUUUURL: %+v", req.URL)) } return &httputil.ReverseProxy{Director: director} } From 4862d70caf0e13f6154e174af5d05624a5d19dc8 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Fri, 22 Sep 2023 09:39:03 -0400 Subject: [PATCH 12/28] added useAzureOpenAI logic --- src/components/AppConfig/AppConfig.tsx | 36 ++++++++++++++++++++------ src/components/testIds.ts | 1 + 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/components/AppConfig/AppConfig.tsx b/src/components/AppConfig/AppConfig.tsx index 40ec9413..fba1279e 100644 --- a/src/components/AppConfig/AppConfig.tsx +++ b/src/components/AppConfig/AppConfig.tsx @@ -3,12 +3,13 @@ import { lastValueFrom } from 'rxjs'; import { css } from '@emotion/css'; import { AppPluginMeta, GrafanaTheme2, PluginConfigPageProps, PluginMeta } from '@grafana/data'; import { getBackendSrv } from '@grafana/runtime'; -import { Button, Field, FieldSet, Input, SecretInput, useStyles2 } from '@grafana/ui'; +import { Button, Field, FieldSet, Input, SecretInput, useStyles2, Switch } from '@grafana/ui'; import { testIds } from '../testIds'; type OpenAISettings = { url?: string; organizationId?: string; + useAzure?: boolean; } export type AppPluginSettings = { @@ -24,6 +25,10 @@ type State = { isOpenAIKeySet: boolean; // A secret key for our custom API. openAIKey: string; + // A flag to tell us if we should use Azure OpenAI. + useAzureOpenAI: boolean; + // A flag to tell us if state was updated + updated: boolean; }; export interface AppConfigProps extends PluginConfigPageProps> { } @@ -36,33 +41,48 @@ export const AppConfig = ({ plugin }: AppConfigProps) => { openAIOrganizationID: jsonData?.openAI?.organizationId || '', openAIKey: '', isOpenAIKeySet: Boolean(secureJsonFields?.openAIKey), + useAzureOpenAI: jsonData?.openAI?.useAzure || false, + updated: false, }); const onResetApiKey = () => setState({ ...state, + openAIUrl: '', openAIKey: '', openAIOrganizationID: '', isOpenAIKeySet: false, + useAzureOpenAI: state.useAzureOpenAI, + updated: true, }); const onChange = (event: ChangeEvent) => { setState({ ...state, - [event.target.name]: event.target.value.trim(), + [event.target.name]: (event.target.type === 'checkbox' ? event.target.checked : event.target.value.trim()), + updated: true, }); }; return (
+ + + .openai.azure.com` : `https://api.openai.com`} onChange={onChange} /> @@ -73,8 +93,9 @@ export const AppConfig = ({ plugin }: AppConfigProps) => { name="openAIOrganizationID" data-testid={testIds.appConfig.openAIOrganizationID} value={state.openAIOrganizationID} - placeholder={'org-...'} + placeholder={state.useAzureOpenAI ? '' : 'org-...'} onChange={onChange} + disabled={state.useAzureOpenAI} /> @@ -85,7 +106,7 @@ export const AppConfig = ({ plugin }: AppConfigProps) => { name="openAIKey" value={state.openAIKey} isConfigured={state.isOpenAIKeySet} - placeholder={'sk-...'} + placeholder={state.useAzureOpenAI ? '' : 'sk-...'} onChange={onChange} onReset={onResetApiKey} /> @@ -103,6 +124,7 @@ export const AppConfig = ({ plugin }: AppConfigProps) => { openAI: { url: state.openAIUrl, organizationId: state.openAIOrganizationID, + useAzure: state.useAzureOpenAI, }, }, // This cannot be queried later by the frontend. @@ -114,9 +136,7 @@ export const AppConfig = ({ plugin }: AppConfigProps) => { }, }) } - disabled={Boolean( - !state.openAIUrl || !state.openAIOrganizationID || (!state.isOpenAIKeySet && !state.openAIKey) - )} + disabled={!state.updated} > Save API settings diff --git a/src/components/testIds.ts b/src/components/testIds.ts index 32c995cd..d0808c5c 100644 --- a/src/components/testIds.ts +++ b/src/components/testIds.ts @@ -1,6 +1,7 @@ export const testIds = { appConfig: { container: 'data-testid ac-container', + useAzureOpenAI: 'data-testid ac-use-azure-openai', openAIKey: 'data-testid ac-openai-api-key', openAIOrganizationID: 'data-testid ac-openai-api-organization-id', openAIUrl: 'data-testid ac-openai-api-url', From a0d5195474168124c529af0f5251c2a7a0d3655b Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Fri, 22 Sep 2023 12:04:11 -0400 Subject: [PATCH 13/28] updated deployment logic --- pkg/plugin/resources.go | 13 ++++++++++--- pkg/plugin/stream.go | 14 ++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/pkg/plugin/resources.go b/pkg/plugin/resources.go index 500f9cc4..da81d53d 100644 --- a/pkg/plugin/resources.go +++ b/pkg/plugin/resources.go @@ -65,11 +65,18 @@ func newOpenAIProxy() http.Handler { var requestBody map[string]interface{} json.Unmarshal(bodyBytes, &requestBody) - settings.OpenAI.AzureMapping = map[string]string{ - "gpt-3.5-turbo": "gpt-35-turbo", + var deployment string = "" + for _, v := range settings.OpenAI.AzureMapping { + if val, ok := requestBody["model"].(string); ok && val == v[0] { + deployment = v[1] + break + } } - deployment := settings.OpenAI.AzureMapping[requestBody["model"].(string)] + if deployment == "" { + log.DefaultLogger.Error(fmt.Sprintf("No deployment found for model: %s", requestBody["model"])) + deployment = "DEPLOYMENT_IS_MISSING" + } req.URL.Path = fmt.Sprintf("/openai/deployments/%s/%s", deployment, strings.TrimPrefix(req.URL.Path, "/openai/v1")) req.Header.Add("api-key", settings.OpenAI.apiKey) diff --git a/pkg/plugin/stream.go b/pkg/plugin/stream.go index 6252fdc1..da7a72f5 100644 --- a/pkg/plugin/stream.go +++ b/pkg/plugin/stream.go @@ -56,11 +56,18 @@ func (a *App) runOpenAIChatCompletionsStream(ctx context.Context, req *backend.R if settings.OpenAI.UseAzure { // Map model to deployment - settings.OpenAI.AzureMapping = map[string]string{ - "gpt-3.5-turbo": "gpt-35-turbo", + var deployment string = "" + for _, v := range settings.OpenAI.AzureMapping { + if val, ok := requestBody["model"].(string); ok && val == v[0] { + deployment = v[1] + break + } } - deployment := settings.OpenAI.AzureMapping[requestBody["model"].(string)] + if deployment == "" { + log.DefaultLogger.Error(fmt.Sprintf("No deployment found for model: %s", requestBody["model"])) + deployment = "DEPLOYMENT_IS_MISSING" + } apiPath := strings.TrimPrefix(req.Path, "openai/v1/") @@ -72,7 +79,6 @@ func (a *App) runOpenAIChatCompletionsStream(ctx context.Context, req *backend.R } else { u.Path = strings.TrimPrefix(req.Path, "openai") - } outgoingBody, err = json.Marshal(requestBody) From f6970d85fe56f27a1f455e370aa8df278bb580c6 Mon Sep 17 00:00:00 2001 From: Edward Qian Date: Fri, 22 Sep 2023 12:04:29 -0400 Subject: [PATCH 14/28] added azureModelMapping Logic --- pkg/plugin/settings.go | 8 +- src/components/AppConfig/AppConfig.tsx | 109 ++++++++++++++++++++++++- 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/pkg/plugin/settings.go b/pkg/plugin/settings.go index f98ec289..f3ee452b 100644 --- a/pkg/plugin/settings.go +++ b/pkg/plugin/settings.go @@ -10,10 +10,10 @@ import ( const openAIKey = "openAIKey" type OpenAISettings struct { - URL string `json:"url"` - OrganizationID string `json:"organizationId"` - UseAzure bool `json:"useAzure"` - AzureMapping map[string]string `json:"azureOpenAIModelMapping"` + URL string `json:"url"` + OrganizationID string `json:"organizationId"` + UseAzure bool `json:"useAzure"` + AzureMapping [][]string `json:"azureModelMapping"` apiKey string } diff --git a/src/components/AppConfig/AppConfig.tsx b/src/components/AppConfig/AppConfig.tsx index fba1279e..376f7d03 100644 --- a/src/components/AppConfig/AppConfig.tsx +++ b/src/components/AppConfig/AppConfig.tsx @@ -3,15 +3,18 @@ import { lastValueFrom } from 'rxjs'; import { css } from '@emotion/css'; import { AppPluginMeta, GrafanaTheme2, PluginConfigPageProps, PluginMeta } from '@grafana/data'; import { getBackendSrv } from '@grafana/runtime'; -import { Button, Field, FieldSet, Input, SecretInput, useStyles2, Switch } from '@grafana/ui'; +import { Button, Field, FieldSet, Input, SecretInput, useStyles2, Switch, InlineField, IconButton, Select, InlineFieldRow } from '@grafana/ui'; import { testIds } from '../testIds'; type OpenAISettings = { url?: string; organizationId?: string; useAzure?: boolean; + azureModelMapping?: ModelDeploymentMap; } +type ModelDeploymentMap = Array<[string, string]>; + export type AppPluginSettings = { openAI?: OpenAISettings; }; @@ -27,12 +30,95 @@ type State = { openAIKey: string; // A flag to tell us if we should use Azure OpenAI. useAzureOpenAI: boolean; + // A mapping of Azure models to OpenAI models. + azureModelMapping: ModelDeploymentMap; // A flag to tell us if state was updated updated: boolean; }; export interface AppConfigProps extends PluginConfigPageProps> { } +function ModelMappingConfig({ modelMapping, modelNames, onChange }: { + modelMapping: Array<[string, string]>; + modelNames: string[]; + onChange: (modelMapping: Array<[string, string]>) => void; +}) { + return ( + <> + { + e.preventDefault(); + onChange([...modelMapping, ['', '']]); + }} + /> + {modelMapping.map(([model, deployment], i) => ( + { + onChange([ + ...modelMapping.slice(0, i), + [model, deployment], + ...modelMapping.slice(i + 1), + ]); + }} + onRemove={() => onChange([ + ...modelMapping.slice(0, i), + ...modelMapping.slice(i + 1), + ])} + /> + ) + )} + + ); +} + +function ModelMappingField({ model, deployment, modelNames, onChange, onRemove }: { + model: string; + deployment: string; + modelNames: string[], + onChange: (model: string, deployment: string) => void; + onRemove: () => void; +}): JSX.Element { + return ( + + + event.currentTarget.value !== undefined && onChange(model, event.currentTarget.value)} + /> + + { + e.preventDefault(); + onRemove() + }} + /> + + ); +} + + export const AppConfig = ({ plugin }: AppConfigProps) => { const s = useStyles2(getStyles); const { enabled, pinned, jsonData, secureJsonFields } = plugin.meta; @@ -42,6 +128,7 @@ export const AppConfig = ({ plugin }: AppConfigProps) => { openAIKey: '', isOpenAIKeySet: Boolean(secureJsonFields?.openAIKey), useAzureOpenAI: jsonData?.openAI?.useAzure || false, + azureModelMapping: jsonData?.openAI?.azureModelMapping || [], updated: false, }); @@ -53,6 +140,7 @@ export const AppConfig = ({ plugin }: AppConfigProps) => { openAIOrganizationID: '', isOpenAIKeySet: false, useAzureOpenAI: state.useAzureOpenAI, + azureModelMapping: [], updated: true, }); @@ -89,7 +177,7 @@ export const AppConfig = ({ plugin }: AppConfigProps) => { { /> + {state.useAzureOpenAI && ( + + setState({ + ...state, + azureModelMapping, + updated: true, + }) + } + /> + + )} +