From 459917e33d802a58f5018c19efd6d14d33099047 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Mon, 17 Jun 2024 19:33:01 -0400 Subject: [PATCH 1/4] start breaking up the giant refactor once again. This contains simple fixes to warnings and errors, removes a broken / outdated test, runs go mod tidy, and as the actual change, centralizes base64 image handling Signed-off-by: Dave Lee --- core/http/endpoints/openai/assistant.go | 2 +- core/http/endpoints/openai/assistant_test.go | 12 +++--- core/http/endpoints/openai/request.go | 44 ++------------------ go.mod | 2 +- go.sum | 10 ----- pkg/utils/base64.go | 11 +++-- pkg/utils/base64_test.go | 9 +++- tests/integration/reflect_test.go | 23 ---------- 8 files changed, 27 insertions(+), 86 deletions(-) delete mode 100644 tests/integration/reflect_test.go diff --git a/core/http/endpoints/openai/assistant.go b/core/http/endpoints/openai/assistant.go index c1efd8bd4e67..ce7b29923233 100644 --- a/core/http/endpoints/openai/assistant.go +++ b/core/http/endpoints/openai/assistant.go @@ -339,7 +339,7 @@ func CreateAssistantFileEndpoint(cl *config.BackendConfigLoader, ml *model.Model } } - return c.Status(fiber.StatusNotFound).SendString(fmt.Sprintf("Unable to find ")) + return c.Status(fiber.StatusNotFound).SendString(fmt.Sprintf("Unable to find %q", assistantID)) } } diff --git a/core/http/endpoints/openai/assistant_test.go b/core/http/endpoints/openai/assistant_test.go index e7c09033f3cd..a4cca8334fc5 100644 --- a/core/http/endpoints/openai/assistant_test.go +++ b/core/http/endpoints/openai/assistant_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "net/http/httptest" "os" @@ -183,7 +182,7 @@ func TestAssistantEndpoints(t *testing.T) { assert.NoError(t, err) assert.Equal(t, tt.expectedStatus, response.StatusCode) if tt.expectedStatus != fiber.StatusOK { - all, _ := ioutil.ReadAll(response.Body) + all, _ := io.ReadAll(response.Body) assert.Equal(t, tt.expectedStringResult, string(all)) } else { var result []Assistant @@ -279,6 +278,7 @@ func TestAssistantEndpoints(t *testing.T) { assert.NoError(t, err) var getAssistant Assistant err = json.NewDecoder(modifyResponse.Body).Decode(&getAssistant) + assert.NoError(t, err) t.Cleanup(cleanupAllAssistants(t, app, []string{getAssistant.ID})) @@ -391,7 +391,10 @@ func createAssistantFile(app *fiber.App, afr AssistantFileRequest, assistantId s } var assistantFile AssistantFile - all, err := ioutil.ReadAll(resp.Body) + all, err := io.ReadAll(resp.Body) + if err != nil { + return AssistantFile{}, resp, err + } err = json.NewDecoder(strings.NewReader(string(all))).Decode(&assistantFile) if err != nil { return AssistantFile{}, resp, err @@ -422,8 +425,7 @@ func createAssistant(app *fiber.App, ar AssistantRequest) (Assistant, *http.Resp var resultAssistant Assistant err = json.NewDecoder(strings.NewReader(string(bodyString))).Decode(&resultAssistant) - - return resultAssistant, resp, nil + return resultAssistant, resp, err } func cleanupAllAssistants(t *testing.T, app *fiber.App, ids []string) func() { diff --git a/core/http/endpoints/openai/request.go b/core/http/endpoints/openai/request.go index 728645d79d70..697ebfe6f5a7 100644 --- a/core/http/endpoints/openai/request.go +++ b/core/http/endpoints/openai/request.go @@ -2,18 +2,15 @@ package openai import ( "context" - "encoding/base64" "encoding/json" "fmt" - "io" - "net/http" - "strings" "github.com/go-skynet/LocalAI/core/config" fiberContext "github.com/go-skynet/LocalAI/core/http/ctx" "github.com/go-skynet/LocalAI/core/schema" "github.com/go-skynet/LocalAI/pkg/functions" - model "github.com/go-skynet/LocalAI/pkg/model" + "github.com/go-skynet/LocalAI/pkg/model" + "github.com/go-skynet/LocalAI/pkg/utils" "github.com/gofiber/fiber/v2" "github.com/rs/zerolog/log" ) @@ -39,41 +36,6 @@ func readRequest(c *fiber.Ctx, ml *model.ModelLoader, o *config.ApplicationConfi return modelFile, input, err } -// this function check if the string is an URL, if it's an URL downloads the image in memory -// encodes it in base64 and returns the base64 string -func getBase64Image(s string) (string, error) { - if strings.HasPrefix(s, "http") { - // download the image - resp, err := http.Get(s) - if err != nil { - return "", err - } - defer resp.Body.Close() - - // read the image data into memory - data, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - // encode the image data in base64 - encoded := base64.StdEncoding.EncodeToString(data) - - // return the base64 string - return encoded, nil - } - - // if the string instead is prefixed with "data:image/...;base64,", drop it - dropPrefix := []string{"data:image/jpeg;base64,", "data:image/png;base64,"} - for _, prefix := range dropPrefix { - if strings.HasPrefix(s, prefix) { - return strings.ReplaceAll(s, prefix, ""), nil - } - } - - return "", fmt.Errorf("not valid string") -} - func updateRequestConfig(config *config.BackendConfig, input *schema.OpenAIRequest) { if input.Echo { config.Echo = input.Echo @@ -187,7 +149,7 @@ func updateRequestConfig(config *config.BackendConfig, input *schema.OpenAIReque input.Messages[i].StringContent = pp.Text } else if pp.Type == "image_url" { // Detect if pp.ImageURL is an URL, if it is download the image and encode it in base64: - base64, err := getBase64Image(pp.ImageURL.URL) + base64, err := utils.GetImageURLAsBase64(pp.ImageURL.URL) if err == nil { input.Messages[i].StringImages = append(input.Messages[i].StringImages, base64) // TODO: make sure that we only return base64 stuff // set a placeholder for each image diff --git a/go.mod b/go.mod index 2a1d39d3aec4..205125a9f8fa 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/shirou/gopsutil/v3 v3.23.9 github.com/stretchr/testify v1.9.0 github.com/swaggo/swag v1.16.3 + github.com/thxcode/gguf-parser-go v0.0.6 github.com/tmc/langchaingo v0.0.0-20231019140956-c636b3da7701 github.com/valyala/fasthttp v1.51.0 go.opentelemetry.io/otel v1.19.0 @@ -137,7 +138,6 @@ require ( github.com/smallnest/ringbuffer v0.0.0-20240423223918-bab516b2000b // indirect github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/thxcode/gguf-parser-go v0.0.6 // indirect github.com/tinylib/msgp v1.1.8 // indirect github.com/vishvananda/netlink v1.1.0 // indirect github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect diff --git a/go.sum b/go.sum index 4413aea009df..df35e9ff7d94 100644 --- a/go.sum +++ b/go.sum @@ -653,8 +653,6 @@ github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0J github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/thxcode/gguf-parser-go v0.0.5 h1:fYjrrQ6DFMTOCxP5iWolFIgAS9uB6Lj0MLsabYd+WUA= -github.com/thxcode/gguf-parser-go v0.0.5/go.mod h1:xHPU1OI4c0KHVTGYjTZIkLRJhBZUb9wDTFYFvkRXo9M= github.com/thxcode/gguf-parser-go v0.0.6 h1:2lbnqA9r/4kyfOUZxy3VWRP60IkfNb31l57GmzOzYKE= github.com/thxcode/gguf-parser-go v0.0.6/go.mod h1:xHPU1OI4c0KHVTGYjTZIkLRJhBZUb9wDTFYFvkRXo9M= github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= @@ -759,8 +757,6 @@ golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4 golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -776,8 +772,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -820,8 +814,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -906,8 +898,6 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/utils/base64.go b/pkg/utils/base64.go index 977156e92adb..3fbb405b38af 100644 --- a/pkg/utils/base64.go +++ b/pkg/utils/base64.go @@ -42,9 +42,12 @@ func GetImageURLAsBase64(s string) (string, error) { return encoded, nil } - // if the string instead is prefixed with "data:image/jpeg;base64,", drop it - if strings.HasPrefix(s, "data:image/jpeg;base64,") { - return strings.ReplaceAll(s, "data:image/jpeg;base64,", ""), nil + // if the string instead is prefixed with "data:image/...;base64,", drop it + dropPrefix := []string{"data:image/jpeg;base64,", "data:image/png;base64,"} + for _, prefix := range dropPrefix { + if strings.HasPrefix(s, prefix) { + return strings.ReplaceAll(s, prefix, ""), nil + } } return "", fmt.Errorf("not valid string") -} \ No newline at end of file +} diff --git a/pkg/utils/base64_test.go b/pkg/utils/base64_test.go index 28a09d171ffe..f81f0e403552 100644 --- a/pkg/utils/base64_test.go +++ b/pkg/utils/base64_test.go @@ -7,13 +7,20 @@ import ( ) var _ = Describe("utils/base64 tests", func() { - It("GetImageURLAsBase64 can strip data url prefixes", func() { + It("GetImageURLAsBase64 can strip jpeg data url prefixes", func() { // This one doesn't actually _care_ that it's base64, so feed "bad" data in this test in order to catch a change in that behavior for informational purposes. input := "data:image/jpeg;base64,FOO" b64, err := GetImageURLAsBase64(input) Expect(err).To(BeNil()) Expect(b64).To(Equal("FOO")) }) + It("GetImageURLAsBase64 can strip png data url prefixes", func() { + // This one doesn't actually _care_ that it's base64, so feed "bad" data in this test in order to catch a change in that behavior for informational purposes. + input := "data:image/png;base64,BAR" + b64, err := GetImageURLAsBase64(input) + Expect(err).To(BeNil()) + Expect(b64).To(Equal("BAR")) + }) It("GetImageURLAsBase64 returns an error for bogus data", func() { input := "FOO" b64, err := GetImageURLAsBase64(input) diff --git a/tests/integration/reflect_test.go b/tests/integration/reflect_test.go deleted file mode 100644 index 5fd6011437de..000000000000 --- a/tests/integration/reflect_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package integration_test - -import ( - "reflect" - - "github.com/go-skynet/LocalAI/core/config" - model "github.com/go-skynet/LocalAI/pkg/model" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Integration Tests involving reflection in liue of code generation", func() { - Context("config.TemplateConfig and model.TemplateType must stay in sync", func() { - - ttc := reflect.TypeOf(config.TemplateConfig{}) - - It("TemplateConfig and TemplateType should have the same number of valid values", func() { - const lastValidTemplateType = model.IntegrationTestTemplate - 1 - Expect(lastValidTemplateType).To(Equal(ttc.NumField())) - }) - - }) -}) From 3214066e592d004606aec55a57319888a78ae058 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Mon, 17 Jun 2024 19:41:53 -0400 Subject: [PATCH 2/4] add rwkv to logline to make it distinct while debugging Signed-off-by: Dave Lee --- backend/go/llm/rwkv/rwkv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/llm/rwkv/rwkv.go b/backend/go/llm/rwkv/rwkv.go index d315fcc2f218..c812fe9f4f93 100644 --- a/backend/go/llm/rwkv/rwkv.go +++ b/backend/go/llm/rwkv/rwkv.go @@ -31,7 +31,7 @@ func (llm *LLM) Load(opts *pb.ModelOptions) error { model := rwkv.LoadFiles(opts.ModelFile, tokenizerPath, uint32(opts.GetThreads())) if model == nil { - return fmt.Errorf("could not load model") + return fmt.Errorf("rwkv could not load model") } llm.rwkv = model return nil From 37e15aa72ef3914847f4d9875c0ac77d024db1e5 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Mon, 17 Jun 2024 19:58:48 -0400 Subject: [PATCH 3/4] propagate errors, handle them etc, all to make secscan happy. Signed-off-by: Dave Lee --- core/http/render.go | 5 ++--- core/startup/startup.go | 5 ++++- pkg/library/dynaload.go | 17 +++++++++++------ pkg/model/loader_test.go | 21 ++++++++++----------- pkg/model/process.go | 5 ++++- 5 files changed, 31 insertions(+), 22 deletions(-) diff --git a/core/http/render.go b/core/http/render.go index 1becf0b21311..d250b515adc7 100644 --- a/core/http/render.go +++ b/core/http/render.go @@ -21,14 +21,13 @@ func notFoundHandler(c *fiber.Ctx) error { // Check if the request accepts JSON if string(c.Context().Request.Header.ContentType()) == "application/json" || len(c.Accepts("html")) == 0 { // The client expects a JSON response - c.Status(fiber.StatusNotFound).JSON(schema.ErrorResponse{ + return c.Status(fiber.StatusNotFound).JSON(schema.ErrorResponse{ Error: &schema.APIError{Message: "Resource not found", Code: fiber.StatusNotFound}, }) } else { // The client expects an HTML response - c.Status(fiber.StatusNotFound).Render("views/404", fiber.Map{}) + return c.Status(fiber.StatusNotFound).Render("views/404", fiber.Map{}) } - return nil } func renderEngine() *fiberhtml.Engine { diff --git a/core/startup/startup.go b/core/startup/startup.go index 0a135a62ec72..f7409e074287 100644 --- a/core/startup/startup.go +++ b/core/startup/startup.go @@ -112,7 +112,10 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode if options.LibPath != "" { // If there is a lib directory, set LD_LIBRARY_PATH to include it - library.LoadExternal(options.LibPath) + err := library.LoadExternal(options.LibPath) + if err != nil { + log.Error().Err(err).Str("LibPath", options.LibPath).Msg("Error while loading external libraries") + } } // turn off any process that was started by GRPC if the context is canceled diff --git a/pkg/library/dynaload.go b/pkg/library/dynaload.go index 57b002591d8c..8e14216de85b 100644 --- a/pkg/library/dynaload.go +++ b/pkg/library/dynaload.go @@ -1,27 +1,30 @@ package library import ( + "errors" "fmt" "os" "path/filepath" "runtime" ) -func LoadExtractedLibs(dir string) { +func LoadExtractedLibs(dir string) error { // Skip this if LOCALAI_SKIP_LIBRARY_PATH is set if os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" { - return + return nil } + var err error = nil for _, libDir := range []string{filepath.Join(dir, "backend-assets", "lib"), filepath.Join(dir, "lib")} { - LoadExternal(libDir) + err = errors.Join(err, LoadExternal(libDir)) } + return err } -func LoadExternal(dir string) { +func LoadExternal(dir string) error { // Skip this if LOCALAI_SKIP_LIBRARY_PATH is set if os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" { - return + return nil } lpathVar := "LD_LIBRARY_PATH" @@ -29,6 +32,7 @@ func LoadExternal(dir string) { lpathVar = "DYLD_FALLBACK_LIBRARY_PATH" // should it be DYLD_LIBRARY_PATH ? } + var err error = nil if _, err := os.Stat(dir); err == nil { ldLibraryPath := os.Getenv(lpathVar) if ldLibraryPath == "" { @@ -36,6 +40,7 @@ func LoadExternal(dir string) { } else { ldLibraryPath = fmt.Sprintf("%s:%s", ldLibraryPath, dir) } - os.Setenv(lpathVar, ldLibraryPath) + err = errors.Join(err, os.Setenv(lpathVar, ldLibraryPath)) } + return err } diff --git a/pkg/model/loader_test.go b/pkg/model/loader_test.go index c0768051584f..be6bab17f9f5 100644 --- a/pkg/model/loader_test.go +++ b/pkg/model/loader_test.go @@ -1,7 +1,6 @@ package model_test import ( - "github.com/go-skynet/LocalAI/pkg/model" . "github.com/go-skynet/LocalAI/pkg/model" . "github.com/onsi/ginkgo/v2" @@ -44,7 +43,7 @@ var llama3TestMatch map[string]map[string]interface{} = map[string]map[string]in "user": { "template": llama3, "expected": "<|start_header_id|>user<|end_header_id|>\n\nA long time ago in a galaxy far, far away...<|eot_id|>", - "data": model.ChatMessageTemplateData{ + "data": ChatMessageTemplateData{ SystemPrompt: "", Role: "user", RoleName: "user", @@ -59,7 +58,7 @@ var llama3TestMatch map[string]map[string]interface{} = map[string]map[string]in "assistant": { "template": llama3, "expected": "<|start_header_id|>assistant<|end_header_id|>\n\nA long time ago in a galaxy far, far away...<|eot_id|>", - "data": model.ChatMessageTemplateData{ + "data": ChatMessageTemplateData{ SystemPrompt: "", Role: "assistant", RoleName: "assistant", @@ -74,7 +73,7 @@ var llama3TestMatch map[string]map[string]interface{} = map[string]map[string]in "function_call": { "template": llama3, "expected": "<|start_header_id|>assistant<|end_header_id|>\n\nFunction call:\n{\"function\":\"test\"}<|eot_id|>", - "data": model.ChatMessageTemplateData{ + "data": ChatMessageTemplateData{ SystemPrompt: "", Role: "assistant", RoleName: "assistant", @@ -89,7 +88,7 @@ var llama3TestMatch map[string]map[string]interface{} = map[string]map[string]in "function_response": { "template": llama3, "expected": "<|start_header_id|>tool<|end_header_id|>\n\nFunction response:\nResponse from tool<|eot_id|>", - "data": model.ChatMessageTemplateData{ + "data": ChatMessageTemplateData{ SystemPrompt: "", Role: "tool", RoleName: "tool", @@ -107,7 +106,7 @@ var chatMLTestMatch map[string]map[string]interface{} = map[string]map[string]in "user": { "template": chatML, "expected": "<|im_start|>user\nA long time ago in a galaxy far, far away...<|im_end|>", - "data": model.ChatMessageTemplateData{ + "data": ChatMessageTemplateData{ SystemPrompt: "", Role: "user", RoleName: "user", @@ -122,7 +121,7 @@ var chatMLTestMatch map[string]map[string]interface{} = map[string]map[string]in "assistant": { "template": chatML, "expected": "<|im_start|>assistant\nA long time ago in a galaxy far, far away...<|im_end|>", - "data": model.ChatMessageTemplateData{ + "data": ChatMessageTemplateData{ SystemPrompt: "", Role: "assistant", RoleName: "assistant", @@ -137,7 +136,7 @@ var chatMLTestMatch map[string]map[string]interface{} = map[string]map[string]in "function_call": { "template": chatML, "expected": "<|im_start|>assistant\n\n{\"function\":\"test\"}\n<|im_end|>", - "data": model.ChatMessageTemplateData{ + "data": ChatMessageTemplateData{ SystemPrompt: "", Role: "assistant", RoleName: "assistant", @@ -152,7 +151,7 @@ var chatMLTestMatch map[string]map[string]interface{} = map[string]map[string]in "function_response": { "template": chatML, "expected": "<|im_start|>tool\n\nResponse from tool\n<|im_end|>", - "data": model.ChatMessageTemplateData{ + "data": ChatMessageTemplateData{ SystemPrompt: "", Role: "tool", RoleName: "tool", @@ -175,7 +174,7 @@ var _ = Describe("Templates", func() { for key := range chatMLTestMatch { foo := chatMLTestMatch[key] It("renders correctly `"+key+"`", func() { - templated, err := modelLoader.EvaluateTemplateForChatMessage(foo["template"].(string), foo["data"].(model.ChatMessageTemplateData)) + templated, err := modelLoader.EvaluateTemplateForChatMessage(foo["template"].(string), foo["data"].(ChatMessageTemplateData)) Expect(err).ToNot(HaveOccurred()) Expect(templated).To(Equal(foo["expected"]), templated) }) @@ -189,7 +188,7 @@ var _ = Describe("Templates", func() { for key := range llama3TestMatch { foo := llama3TestMatch[key] It("renders correctly `"+key+"`", func() { - templated, err := modelLoader.EvaluateTemplateForChatMessage(foo["template"].(string), foo["data"].(model.ChatMessageTemplateData)) + templated, err := modelLoader.EvaluateTemplateForChatMessage(foo["template"].(string), foo["data"].(ChatMessageTemplateData)) Expect(err).ToNot(HaveOccurred()) Expect(templated).To(Equal(foo["expected"]), templated) }) diff --git a/pkg/model/process.go b/pkg/model/process.go index b7180f5d8948..5009653b69e9 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -103,7 +103,10 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) <-c - grpcControlProcess.Stop() + err := grpcControlProcess.Stop() + if err != nil { + log.Error().Err(err).Msg("error while shutting down grpc process") + } }() go func() { From 91746a1cf95a951119c961edfb5d648ad1ec5cc9 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sun, 23 Jun 2024 00:30:20 -0400 Subject: [PATCH 4/4] use the variable Signed-off-by: Dave Lee --- pkg/library/dynaload.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/library/dynaload.go b/pkg/library/dynaload.go index 3847e71e2a85..e2a87490eec6 100644 --- a/pkg/library/dynaload.go +++ b/pkg/library/dynaload.go @@ -20,7 +20,7 @@ var skipLibraryPath = os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" // LoadExtractedLibs loads the extracted libraries from the asset dir func LoadExtractedLibs(dir string) error { // Skip this if LOCALAI_SKIP_LIBRARY_PATH is set - if os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" { + if skipLibraryPath { return nil } @@ -63,7 +63,7 @@ func LoadLDSO(assetDir string, args []string, grpcProcess string) ([]string, str // LoadExternal sets the LD_LIBRARY_PATH to include the given directory func LoadExternal(dir string) error { // Skip this if LOCALAI_SKIP_LIBRARY_PATH is set - if os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" { + if skipLibraryPath { return nil }