From ad0e30bca5b84229e4d84c2222e49e83258a5ea9 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 13 Nov 2023 22:40:16 +0100 Subject: [PATCH] refactor: move backends into the backends directory (#1279) * refactor: move backends into the backends directory Signed-off-by: Ettore Di Giacinto * refactor: move main close to implementation for every backend Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: Ettore Di Giacinto --- .github/workflows/test.yml | 2 +- Dockerfile | 28 +++----- Makefile | 66 +++++++++---------- {pkg/grpc/proto => backend}/backend.proto | 0 backend/cpp/llama/CMakeLists.txt | 2 +- .../go/image}/main.go | 3 +- .../go}/image/stablediffusion.go | 2 +- {pkg/backend => backend/go}/llm/bert/bert.go | 2 +- .../grpc/rwkv => backend/go/llm/bert}/main.go | 4 +- {cmd/grpc => backend/go/llm}/dolly/main.go | 2 +- .../go/llm}/falcon-ggml/main.go | 2 +- {cmd/grpc => backend/go/llm}/gpt2/main.go | 2 +- .../go}/llm/gpt4all/gpt4all.go | 2 +- {cmd/grpc => backend/go/llm}/gpt4all/main.go | 4 +- {cmd/grpc => backend/go/llm}/gptj/main.go | 2 +- {cmd/grpc => backend/go/llm}/gptneox/main.go | 2 +- .../go}/llm/langchain/langchain.go | 2 +- backend/go/llm/langchain/main.go | 21 ++++++ .../go}/llm/llama-stable/llama.go | 2 +- .../go/llm}/llama-stable/main.go | 4 +- .../backend => backend/go}/llm/llama/llama.go | 2 +- .../piper => backend/go/llm/llama}/main.go | 6 +- {cmd/grpc => backend/go/llm}/mpt/main.go | 2 +- {cmd/grpc => backend/go/llm}/replit/main.go | 2 +- backend/go/llm/rwkv/main.go | 21 ++++++ {pkg/backend => backend/go}/llm/rwkv/rwkv.go | 2 +- .../grpc => backend/go/llm}/starcoder/main.go | 2 +- .../go}/llm/transformers/dolly.go | 0 .../go}/llm/transformers/falcon.go | 0 .../go}/llm/transformers/gpt2.go | 0 .../go}/llm/transformers/gptj.go | 0 .../go}/llm/transformers/gptneox.go | 0 .../go}/llm/transformers/mpt.go | 0 .../go}/llm/transformers/predict.go | 0 .../go}/llm/transformers/replit.go | 0 .../go}/llm/transformers/starcoder.go | 0 backend/go/transcribe/main.go | 21 ++++++ .../go}/transcribe/transcript.go | 2 +- .../go}/transcribe/whisper.go | 2 +- backend/go/tts/main.go | 21 ++++++ {pkg/backend => backend/go}/tts/piper.go | 2 +- {extra/grpc => backend/python}/README.md | 0 .../grpc => backend/python}/autogptq/Makefile | 0 .../python}/autogptq/README.md | 0 .../python}/autogptq/autogptq.py | 0 .../python}/autogptq/autogptq.yml | 0 .../python}/autogptq/backend_pb2.py | 0 .../python}/autogptq/backend_pb2_grpc.py | 0 .../grpc => backend/python}/autogptq/run.sh | 0 {extra/grpc => backend/python}/bark/Makefile | 0 {extra/grpc => backend/python}/bark/README.md | 0 .../python}/bark/backend_pb2.py | 0 .../python}/bark/backend_pb2_grpc.py | 0 {extra/grpc => backend/python}/bark/run.sh | 0 .../python}/bark/test_ttsbark.py | 0 .../grpc => backend/python}/bark/ttsbark.py | 0 .../grpc => backend/python}/bark/ttsbark.yml | 0 .../python}/diffusers/Makefile | 0 .../python}/diffusers/README.md | 0 .../python}/diffusers/backend_diffusers.py | 0 .../python}/diffusers/backend_pb2.py | 0 .../python}/diffusers/backend_pb2_grpc.py | 0 .../python}/diffusers/diffusers.yml | 0 .../grpc => backend/python}/diffusers/run.sh | 0 .../grpc => backend/python}/exllama/Makefile | 0 .../grpc => backend/python}/exllama/README.md | 0 .../python}/exllama/backend_pb2.py | 0 .../python}/exllama/backend_pb2_grpc.py | 0 .../python}/exllama/exllama.py | 0 .../python}/exllama/exllama.yml | 0 {extra/grpc => backend/python}/exllama/run.sh | 0 .../python}/huggingface/Makefile | 0 .../python}/huggingface/README.md | 0 .../python}/huggingface/backend_pb2.py | 0 .../python}/huggingface/backend_pb2_grpc.py | 0 .../python}/huggingface/huggingface.py | 0 .../python}/huggingface/huggingface.yml | 0 .../python}/huggingface/run.sh | 0 .../python}/huggingface/test.sh | 0 .../python}/huggingface/test_huggingface.py | 0 .../grpc => backend/python}/vall-e-x/Makefile | 0 .../python}/vall-e-x/README.md | 0 .../python}/vall-e-x/backend_pb2.py | 0 .../python}/vall-e-x/backend_pb2_grpc.py | 0 .../grpc => backend/python}/vall-e-x/run.sh | 0 .../python}/vall-e-x/ttsvalle.py | 0 .../python}/vall-e-x/ttsvalle.yml | 0 {extra/grpc => backend/python}/vllm/Makefile | 0 {extra/grpc => backend/python}/vllm/README.md | 0 .../python}/vllm/backend_pb2.py | 0 .../python}/vllm/backend_pb2_grpc.py | 0 .../python}/vllm/backend_vllm.py | 0 {extra/grpc => backend/python}/vllm/run.sh | 0 .../python}/vllm/test_backend_vllm.py | 0 {extra/grpc => backend/python}/vllm/vllm.yml | 0 cmd/grpc/langchain-huggingface/main.go | 23 ------- cmd/grpc/llama/main.go | 25 ------- cmd/grpc/stablediffusion/main.go | 23 ------- cmd/grpc/whisper/main.go | 23 ------- extra/requirements.txt | 7 -- pkg/grpc/proto/backend.pb.go | 2 +- pkg/grpc/proto/backend_grpc.pb.go | 4 +- 102 files changed, 156 insertions(+), 190 deletions(-) rename {pkg/grpc/proto => backend}/backend.proto (100%) rename {cmd/grpc/bert-embeddings => backend/go/image}/main.go (71%) rename {pkg/backend => backend/go}/image/stablediffusion.go (98%) rename {pkg/backend => backend/go}/llm/bert/bert.go (98%) rename {cmd/grpc/rwkv => backend/go/llm/bert}/main.go (71%) rename {cmd/grpc => backend/go/llm}/dolly/main.go (83%) rename {cmd/grpc => backend/go/llm}/falcon-ggml/main.go (83%) rename {cmd/grpc => backend/go/llm}/gpt2/main.go (83%) rename {pkg/backend => backend/go}/llm/gpt4all/gpt4all.go (99%) rename {cmd/grpc => backend/go/llm}/gpt4all/main.go (69%) rename {cmd/grpc => backend/go/llm}/gptj/main.go (83%) rename {cmd/grpc => backend/go/llm}/gptneox/main.go (83%) rename {pkg/backend => backend/go}/llm/langchain/langchain.go (98%) create mode 100644 backend/go/llm/langchain/main.go rename {pkg/backend => backend/go}/llm/llama-stable/llama.go (99%) rename {cmd/grpc => backend/go/llm}/llama-stable/main.go (61%) rename {pkg/backend => backend/go}/llm/llama/llama.go (99%) rename {cmd/grpc/piper => backend/go/llm/llama}/main.go (72%) rename {cmd/grpc => backend/go/llm}/mpt/main.go (83%) rename {cmd/grpc => backend/go/llm}/replit/main.go (83%) create mode 100644 backend/go/llm/rwkv/main.go rename {pkg/backend => backend/go}/llm/rwkv/rwkv.go (99%) rename {cmd/grpc => backend/go/llm}/starcoder/main.go (83%) rename {pkg/backend => backend/go}/llm/transformers/dolly.go (100%) rename {pkg/backend => backend/go}/llm/transformers/falcon.go (100%) rename {pkg/backend => backend/go}/llm/transformers/gpt2.go (100%) rename {pkg/backend => backend/go}/llm/transformers/gptj.go (100%) rename {pkg/backend => backend/go}/llm/transformers/gptneox.go (100%) rename {pkg/backend => backend/go}/llm/transformers/mpt.go (100%) rename {pkg/backend => backend/go}/llm/transformers/predict.go (100%) rename {pkg/backend => backend/go}/llm/transformers/replit.go (100%) rename {pkg/backend => backend/go}/llm/transformers/starcoder.go (100%) create mode 100644 backend/go/transcribe/main.go rename {pkg/backend => backend/go}/transcribe/transcript.go (99%) rename {pkg/backend => backend/go}/transcribe/whisper.go (97%) create mode 100644 backend/go/tts/main.go rename {pkg/backend => backend/go}/tts/piper.go (98%) rename {extra/grpc => backend/python}/README.md (100%) rename {extra/grpc => backend/python}/autogptq/Makefile (100%) rename {extra/grpc => backend/python}/autogptq/README.md (100%) rename {extra/grpc => backend/python}/autogptq/autogptq.py (100%) rename {extra/grpc => backend/python}/autogptq/autogptq.yml (100%) rename {extra/grpc => backend/python}/autogptq/backend_pb2.py (100%) rename {extra/grpc => backend/python}/autogptq/backend_pb2_grpc.py (100%) rename {extra/grpc => backend/python}/autogptq/run.sh (100%) rename {extra/grpc => backend/python}/bark/Makefile (100%) rename {extra/grpc => backend/python}/bark/README.md (100%) rename {extra/grpc => backend/python}/bark/backend_pb2.py (100%) rename {extra/grpc => backend/python}/bark/backend_pb2_grpc.py (100%) rename {extra/grpc => backend/python}/bark/run.sh (100%) rename {extra/grpc => backend/python}/bark/test_ttsbark.py (100%) rename {extra/grpc => backend/python}/bark/ttsbark.py (100%) rename {extra/grpc => backend/python}/bark/ttsbark.yml (100%) rename {extra/grpc => backend/python}/diffusers/Makefile (100%) rename {extra/grpc => backend/python}/diffusers/README.md (100%) rename {extra/grpc => backend/python}/diffusers/backend_diffusers.py (100%) rename {extra/grpc => backend/python}/diffusers/backend_pb2.py (100%) rename {extra/grpc => backend/python}/diffusers/backend_pb2_grpc.py (100%) rename {extra/grpc => backend/python}/diffusers/diffusers.yml (100%) rename {extra/grpc => backend/python}/diffusers/run.sh (100%) rename {extra/grpc => backend/python}/exllama/Makefile (100%) rename {extra/grpc => backend/python}/exllama/README.md (100%) rename {extra/grpc => backend/python}/exllama/backend_pb2.py (100%) rename {extra/grpc => backend/python}/exllama/backend_pb2_grpc.py (100%) rename {extra/grpc => backend/python}/exllama/exllama.py (100%) rename {extra/grpc => backend/python}/exllama/exllama.yml (100%) rename {extra/grpc => backend/python}/exllama/run.sh (100%) rename {extra/grpc => backend/python}/huggingface/Makefile (100%) rename {extra/grpc => backend/python}/huggingface/README.md (100%) rename {extra/grpc => backend/python}/huggingface/backend_pb2.py (100%) rename {extra/grpc => backend/python}/huggingface/backend_pb2_grpc.py (100%) rename {extra/grpc => backend/python}/huggingface/huggingface.py (100%) rename {extra/grpc => backend/python}/huggingface/huggingface.yml (100%) rename {extra/grpc => backend/python}/huggingface/run.sh (100%) rename {extra/grpc => backend/python}/huggingface/test.sh (100%) rename {extra/grpc => backend/python}/huggingface/test_huggingface.py (100%) rename {extra/grpc => backend/python}/vall-e-x/Makefile (100%) rename {extra/grpc => backend/python}/vall-e-x/README.md (100%) rename {extra/grpc => backend/python}/vall-e-x/backend_pb2.py (100%) rename {extra/grpc => backend/python}/vall-e-x/backend_pb2_grpc.py (100%) rename {extra/grpc => backend/python}/vall-e-x/run.sh (100%) rename {extra/grpc => backend/python}/vall-e-x/ttsvalle.py (100%) rename {extra/grpc => backend/python}/vall-e-x/ttsvalle.yml (100%) rename {extra/grpc => backend/python}/vllm/Makefile (100%) rename {extra/grpc => backend/python}/vllm/README.md (100%) rename {extra/grpc => backend/python}/vllm/backend_pb2.py (100%) rename {extra/grpc => backend/python}/vllm/backend_pb2_grpc.py (100%) rename {extra/grpc => backend/python}/vllm/backend_vllm.py (100%) rename {extra/grpc => backend/python}/vllm/run.sh (100%) rename {extra/grpc => backend/python}/vllm/test_backend_vllm.py (100%) rename {extra/grpc => backend/python}/vllm/vllm.yml (100%) delete mode 100644 cmd/grpc/langchain-huggingface/main.go delete mode 100644 cmd/grpc/llama/main.go delete mode 100644 cmd/grpc/stablediffusion/main.go delete mode 100644 cmd/grpc/whisper/main.go delete mode 100644 extra/requirements.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ece8a57be5e..3f86c1c1df71 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2 sudo rm -rfv /usr/bin/conda || true - PATH=$PATH:/opt/conda/bin make -C extra/grpc/huggingface + PATH=$PATH:/opt/conda/bin make -C backend/python/huggingface # Pre-build piper before we start tests in order to have shared libraries in place make go-piper && \ diff --git a/Dockerfile b/Dockerfile index 2091e2e3ccb1..be6d747b00bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ARG TARGETARCH ARG TARGETVARIANT ENV BUILD_TYPE=${BUILD_TYPE} -ENV EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/build/extra/grpc/huggingface/run.sh,autogptq:/build/extra/grpc/autogptq/run.sh,bark:/build/extra/grpc/bark/run.sh,diffusers:/build/extra/grpc/diffusers/run.sh,exllama:/build/extra/grpc/exllama/run.sh,vall-e-x:/build/extra/grpc/vall-e-x/run.sh,vllm:/build/extra/grpc/vllm/run.sh" +ENV EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/build/backend/python/huggingface/run.sh,autogptq:/build/backend/python/autogptq/run.sh,bark:/build/backend/python/bark/run.sh,diffusers:/build/backend/python/diffusers/run.sh,exllama:/build/backend/python/exllama/run.sh,vall-e-x:/build/backend/python/vall-e-x/run.sh,vllm:/build/backend/python/vllm/run.sh" ENV GALLERIES='[{"name":"model-gallery", "url":"github:go-skynet/model-gallery/index.yaml"}, {"url": "github:go-skynet/model-gallery/huggingface.yaml","name":"huggingface"}]' ARG GO_TAGS="stablediffusion tts" @@ -64,20 +64,10 @@ RUN curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmo apt-get update && \ apt-get install -y conda -COPY extra/requirements.txt /build/extra/requirements.txt ENV PATH="/root/.cargo/bin:${PATH}" RUN pip install --upgrade pip RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -#RUN if [ "${TARGETARCH}" = "amd64" ]; then \ -# pip install git+https://github.com/suno-ai/bark.git diffusers invisible_watermark transformers accelerate safetensors;\ -# fi -#RUN if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "amd64" ]; then \ -# pip install torch vllm && pip install auto-gptq https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu${CUDA_MAJOR_VERSION}${CUDA_MINOR_VERSION}-cp39-cp39-linux_x86_64.whl;\ - # fi -#RUN pip install -r /build/extra/requirements.txt && rm -rf /build/extra/requirements.txt -# Vall-e-X -RUN git clone https://github.com/Plachtaa/VALL-E-X.git /usr/lib/vall-e-x && cd /usr/lib/vall-e-x && pip install -r requirements.txt # \ # ; fi @@ -162,25 +152,25 @@ COPY --from=builder /build/backend-assets/grpc/stablediffusion ./backend-assets/ ## Duplicated from Makefile to avoid having a big layer that's hard to push RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \ - PATH=$PATH:/opt/conda/bin make -C extra/grpc/autogptq \ + PATH=$PATH:/opt/conda/bin make -C backend/python/autogptq \ ; fi RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \ - PATH=$PATH:/opt/conda/bin make -C extra/grpc/bark \ + PATH=$PATH:/opt/conda/bin make -C backend/python/bark \ ; fi RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \ - PATH=$PATH:/opt/conda/bin make -C extra/grpc/diffusers \ + PATH=$PATH:/opt/conda/bin make -C backend/python/diffusers \ ; fi RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \ - PATH=$PATH:/opt/conda/bin make -C extra/grpc/vllm \ + PATH=$PATH:/opt/conda/bin make -C backend/python/vllm \ ; fi RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \ - PATH=$PATH:/opt/conda/bin make -C extra/grpc/huggingface \ + PATH=$PATH:/opt/conda/bin make -C backend/python/huggingface \ ; fi RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \ - PATH=$PATH:/opt/conda/bin make -C extra/grpc/vall-e-x \ + PATH=$PATH:/opt/conda/bin make -C backend/python/vall-e-x \ ; fi RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \ - PATH=$PATH:/opt/conda/bin make -C extra/grpc/exllama \ + PATH=$PATH:/opt/conda/bin make -C backend/python/exllama \ ; fi # Copy VALLE-X as it's not a real "lib" @@ -190,7 +180,7 @@ RUN if [ -d /usr/lib/vall-e-x ]; then \ # we also copy exllama libs over to resolve exllama import error RUN if [ -d /usr/local/lib/python3.9/dist-packages/exllama ]; then \ - cp -rfv /usr/local/lib/python3.9/dist-packages/exllama extra/grpc/exllama/;\ + cp -rfv /usr/local/lib/python3.9/dist-packages/exllama backend/python/exllama/;\ fi # Define the health check command diff --git a/Makefile b/Makefile index 1b03de09bee4..b36e2d0b308b 100644 --- a/Makefile +++ b/Makefile @@ -306,7 +306,7 @@ test: prepare test-models/testmodel grpcs @echo 'Running tests' export GO_TAGS="tts stablediffusion" $(MAKE) prepare-test - HUGGINGFACE_GRPC=$(abspath ./)/extra/grpc/huggingface/run.sh TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \ + HUGGINGFACE_GRPC=$(abspath ./)/backend/python/huggingface/run.sh TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \ $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="!gpt4all && !llama && !llama-gguf" --flake-attempts 5 --fail-fast -v -r ./api ./pkg $(MAKE) test-gpt4all $(MAKE) test-llama @@ -374,27 +374,27 @@ protogen: protogen-go protogen-python protogen-go: protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative \ - pkg/grpc/proto/backend.proto + backend/backend.proto protogen-python: - python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/huggingface/ --grpc_python_out=extra/grpc/huggingface/ pkg/grpc/proto/backend.proto - python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/autogptq/ --grpc_python_out=extra/grpc/autogptq/ pkg/grpc/proto/backend.proto - python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/exllama/ --grpc_python_out=extra/grpc/exllama/ pkg/grpc/proto/backend.proto - python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/bark/ --grpc_python_out=extra/grpc/bark/ pkg/grpc/proto/backend.proto - python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/diffusers/ --grpc_python_out=extra/grpc/diffusers/ pkg/grpc/proto/backend.proto - python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/vall-e-x/ --grpc_python_out=extra/grpc/vall-e-x/ pkg/grpc/proto/backend.proto - python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/vllm/ --grpc_python_out=extra/grpc/vllm/ pkg/grpc/proto/backend.proto + python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/huggingface/ --grpc_python_out=backend/python/huggingface/ backend/backend.proto + python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/autogptq/ --grpc_python_out=backend/python/autogptq/ backend/backend.proto + python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/exllama/ --grpc_python_out=backend/python/exllama/ backend/backend.proto + python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/bark/ --grpc_python_out=backend/python/bark/ backend/backend.proto + python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/diffusers/ --grpc_python_out=backend/python/diffusers/ backend/backend.proto + python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/vall-e-x/ --grpc_python_out=backend/python/vall-e-x/ backend/backend.proto + python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/vllm/ --grpc_python_out=backend/python/vllm/ backend/backend.proto ## GRPC # Note: it is duplicated in the Dockerfile prepare-extra-conda-environments: - $(MAKE) -C extra/grpc/autogptq - $(MAKE) -C extra/grpc/bark - $(MAKE) -C extra/grpc/diffusers - $(MAKE) -C extra/grpc/vllm - $(MAKE) -C extra/grpc/huggingface - $(MAKE) -C extra/grpc/vall-e-x - $(MAKE) -C extra/grpc/exllama + $(MAKE) -C backend/python/autogptq + $(MAKE) -C backend/python/bark + $(MAKE) -C backend/python/diffusers + $(MAKE) -C backend/python/vllm + $(MAKE) -C backend/python/huggingface + $(MAKE) -C backend/python/vall-e-x + $(MAKE) -C backend/python/exllama backend-assets/grpc: @@ -403,7 +403,7 @@ backend-assets/grpc: backend-assets/grpc/llama: backend-assets/grpc go-llama/libbinding.a $(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(shell pwd)/go-llama CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-llama LIBRARY_PATH=$(shell pwd)/go-llama \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama ./cmd/grpc/llama/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama ./backend/go/llm/llama/ # TODO: every binary should have its own folder instead, so can have different metal implementations ifeq ($(BUILD_TYPE),metal) cp go-llama/build/bin/ggml-metal.metal backend-assets/grpc/ @@ -443,68 +443,68 @@ endif backend-assets/grpc/llama-stable: backend-assets/grpc go-llama-stable/libbinding.a $(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(shell pwd)/go-llama-stable CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-llama-stable LIBRARY_PATH=$(shell pwd)/go-llama \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama-stable ./cmd/grpc/llama-stable/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama-stable ./backend/go/llm/llama-stable/ backend-assets/grpc/gpt4all: backend-assets/grpc backend-assets/gpt4all gpt4all/gpt4all-bindings/golang/libgpt4all.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/gpt4all/gpt4all-bindings/golang/ LIBRARY_PATH=$(shell pwd)/gpt4all/gpt4all-bindings/golang/ \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt4all ./cmd/grpc/gpt4all/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt4all ./backend/go/llm/gpt4all/ backend-assets/grpc/dolly: backend-assets/grpc go-ggml-transformers/libtransformers.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/dolly ./cmd/grpc/dolly/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/dolly ./backend/go/llm/dolly/ backend-assets/grpc/gpt2: backend-assets/grpc go-ggml-transformers/libtransformers.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt2 ./cmd/grpc/gpt2/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt2 ./backend/go/llm/gpt2/ backend-assets/grpc/gptj: backend-assets/grpc go-ggml-transformers/libtransformers.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptj ./cmd/grpc/gptj/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptj ./backend/go/llm/gptj/ backend-assets/grpc/gptneox: backend-assets/grpc go-ggml-transformers/libtransformers.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptneox ./cmd/grpc/gptneox/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptneox ./backend/go/llm/gptneox/ backend-assets/grpc/mpt: backend-assets/grpc go-ggml-transformers/libtransformers.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/mpt ./cmd/grpc/mpt/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/mpt ./backend/go/llm/mpt/ backend-assets/grpc/replit: backend-assets/grpc go-ggml-transformers/libtransformers.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/replit ./cmd/grpc/replit/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/replit ./backend/go/llm/replit/ backend-assets/grpc/falcon-ggml: backend-assets/grpc go-ggml-transformers/libtransformers.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/falcon-ggml ./cmd/grpc/falcon-ggml/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/falcon-ggml ./backend/go/llm/falcon-ggml/ backend-assets/grpc/starcoder: backend-assets/grpc go-ggml-transformers/libtransformers.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/starcoder ./cmd/grpc/starcoder/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/starcoder ./backend/go/llm/starcoder/ backend-assets/grpc/rwkv: backend-assets/grpc go-rwkv/librwkv.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-rwkv LIBRARY_PATH=$(shell pwd)/go-rwkv \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/rwkv ./cmd/grpc/rwkv/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/rwkv ./backend/go/llm/rwkv backend-assets/grpc/bert-embeddings: backend-assets/grpc go-bert/libgobert.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-bert LIBRARY_PATH=$(shell pwd)/go-bert \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/bert-embeddings ./cmd/grpc/bert-embeddings/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/bert-embeddings ./backend/go/llm/bert/ backend-assets/grpc/langchain-huggingface: backend-assets/grpc - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/langchain-huggingface ./cmd/grpc/langchain-huggingface/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/langchain-huggingface ./backend/go/llm/langchain/ backend-assets/grpc/stablediffusion: backend-assets/grpc if [ ! -f backend-assets/grpc/stablediffusion ]; then \ $(MAKE) go-stable-diffusion/libstablediffusion.a; \ CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-stable-diffusion/ LIBRARY_PATH=$(shell pwd)/go-stable-diffusion/ \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./cmd/grpc/stablediffusion/; \ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./backend/go/image/; \ fi backend-assets/grpc/piper: backend-assets/grpc backend-assets/espeak-ng-data go-piper/libpiper_binding.a CGO_CXXFLAGS="$(PIPER_CGO_CXXFLAGS)" CGO_LDFLAGS="$(PIPER_CGO_LDFLAGS)" LIBRARY_PATH=$(shell pwd)/go-piper \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/piper ./cmd/grpc/piper/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/piper ./backend/go/tts/ backend-assets/grpc/whisper: backend-assets/grpc whisper.cpp/libwhisper.a CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/whisper.cpp LIBRARY_PATH=$(shell pwd)/whisper.cpp \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/whisper ./cmd/grpc/whisper/ + $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/whisper ./backend/go/transcribe/ grpcs: prepare $(GRPC_BACKENDS) diff --git a/pkg/grpc/proto/backend.proto b/backend/backend.proto similarity index 100% rename from pkg/grpc/proto/backend.proto rename to backend/backend.proto diff --git a/backend/cpp/llama/CMakeLists.txt b/backend/cpp/llama/CMakeLists.txt index 7993856667f5..7bf17871cc83 100644 --- a/backend/cpp/llama/CMakeLists.txt +++ b/backend/cpp/llama/CMakeLists.txt @@ -36,7 +36,7 @@ include_directories(${Protobuf_INCLUDE_DIRS}) message(STATUS "Using protobuf version ${Protobuf_VERSION} | Protobuf_INCLUDE_DIRS: ${Protobuf_INCLUDE_DIRS} | CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}") # Proto file -get_filename_component(hw_proto "../../../../../../pkg/grpc/proto/backend.proto" ABSOLUTE) +get_filename_component(hw_proto "../../../../../../backend/backend.proto" ABSOLUTE) get_filename_component(hw_proto_path "${hw_proto}" PATH) # Generated sources diff --git a/cmd/grpc/bert-embeddings/main.go b/backend/go/image/main.go similarity index 71% rename from cmd/grpc/bert-embeddings/main.go rename to backend/go/image/main.go index 90fae8b286ef..425b23404e04 100644 --- a/cmd/grpc/bert-embeddings/main.go +++ b/backend/go/image/main.go @@ -5,7 +5,6 @@ package main import ( "flag" - bert "github.com/go-skynet/LocalAI/pkg/backend/llm/bert" grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) @@ -16,7 +15,7 @@ var ( func main() { flag.Parse() - if err := grpc.StartServer(*addr, &bert.Embeddings{}); err != nil { + if err := grpc.StartServer(*addr, &StableDiffusion{}); err != nil { panic(err) } } diff --git a/pkg/backend/image/stablediffusion.go b/backend/go/image/stablediffusion.go similarity index 98% rename from pkg/backend/image/stablediffusion.go rename to backend/go/image/stablediffusion.go index 50d299da8e35..f04ee1de98ab 100644 --- a/pkg/backend/image/stablediffusion.go +++ b/backend/go/image/stablediffusion.go @@ -1,4 +1,4 @@ -package image +package main // This is a wrapper to statisfy the GRPC service interface // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc) diff --git a/pkg/backend/llm/bert/bert.go b/backend/go/llm/bert/bert.go similarity index 98% rename from pkg/backend/llm/bert/bert.go rename to backend/go/llm/bert/bert.go index 423ff79cac03..4330762ded51 100644 --- a/pkg/backend/llm/bert/bert.go +++ b/backend/go/llm/bert/bert.go @@ -1,4 +1,4 @@ -package bert +package main // This is a wrapper to statisfy the GRPC service interface // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc) diff --git a/cmd/grpc/rwkv/main.go b/backend/go/llm/bert/main.go similarity index 71% rename from cmd/grpc/rwkv/main.go rename to backend/go/llm/bert/main.go index 74724dc287e4..2e582ec5e781 100644 --- a/cmd/grpc/rwkv/main.go +++ b/backend/go/llm/bert/main.go @@ -5,8 +5,6 @@ package main import ( "flag" - rwkv "github.com/go-skynet/LocalAI/pkg/backend/llm/rwkv" - grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) @@ -17,7 +15,7 @@ var ( func main() { flag.Parse() - if err := grpc.StartServer(*addr, &rwkv.LLM{}); err != nil { + if err := grpc.StartServer(*addr, &Embeddings{}); err != nil { panic(err) } } diff --git a/cmd/grpc/dolly/main.go b/backend/go/llm/dolly/main.go similarity index 83% rename from cmd/grpc/dolly/main.go rename to backend/go/llm/dolly/main.go index cc040cb6e6b7..d00a30f0afdd 100644 --- a/cmd/grpc/dolly/main.go +++ b/backend/go/llm/dolly/main.go @@ -5,7 +5,7 @@ package main import ( "flag" - transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers" + transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers" grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) diff --git a/cmd/grpc/falcon-ggml/main.go b/backend/go/llm/falcon-ggml/main.go similarity index 83% rename from cmd/grpc/falcon-ggml/main.go rename to backend/go/llm/falcon-ggml/main.go index 5a99a9159d8c..426ae8233e1c 100644 --- a/cmd/grpc/falcon-ggml/main.go +++ b/backend/go/llm/falcon-ggml/main.go @@ -5,7 +5,7 @@ package main import ( "flag" - transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers" + transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers" grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) diff --git a/cmd/grpc/gpt2/main.go b/backend/go/llm/gpt2/main.go similarity index 83% rename from cmd/grpc/gpt2/main.go rename to backend/go/llm/gpt2/main.go index 42f6ba47f74e..972e39d385d0 100644 --- a/cmd/grpc/gpt2/main.go +++ b/backend/go/llm/gpt2/main.go @@ -5,7 +5,7 @@ package main import ( "flag" - transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers" + transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers" grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) diff --git a/pkg/backend/llm/gpt4all/gpt4all.go b/backend/go/llm/gpt4all/gpt4all.go similarity index 99% rename from pkg/backend/llm/gpt4all/gpt4all.go rename to backend/go/llm/gpt4all/gpt4all.go index 86d4baa32be0..11685b3d3ca5 100644 --- a/pkg/backend/llm/gpt4all/gpt4all.go +++ b/backend/go/llm/gpt4all/gpt4all.go @@ -1,4 +1,4 @@ -package gpt4all +package main // This is a wrapper to statisfy the GRPC service interface // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc) diff --git a/cmd/grpc/gpt4all/main.go b/backend/go/llm/gpt4all/main.go similarity index 69% rename from cmd/grpc/gpt4all/main.go rename to backend/go/llm/gpt4all/main.go index e659fe77d1de..79829efded76 100644 --- a/cmd/grpc/gpt4all/main.go +++ b/backend/go/llm/gpt4all/main.go @@ -5,8 +5,6 @@ package main import ( "flag" - gpt4all "github.com/go-skynet/LocalAI/pkg/backend/llm/gpt4all" - grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) @@ -17,7 +15,7 @@ var ( func main() { flag.Parse() - if err := grpc.StartServer(*addr, &gpt4all.LLM{}); err != nil { + if err := grpc.StartServer(*addr, &LLM{}); err != nil { panic(err) } } diff --git a/cmd/grpc/gptj/main.go b/backend/go/llm/gptj/main.go similarity index 83% rename from cmd/grpc/gptj/main.go rename to backend/go/llm/gptj/main.go index 3530a72764fd..10f16a29c2fc 100644 --- a/cmd/grpc/gptj/main.go +++ b/backend/go/llm/gptj/main.go @@ -5,7 +5,7 @@ package main import ( "flag" - transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers" + transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers" grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) diff --git a/cmd/grpc/gptneox/main.go b/backend/go/llm/gptneox/main.go similarity index 83% rename from cmd/grpc/gptneox/main.go rename to backend/go/llm/gptneox/main.go index d4e8be472b64..450cd5dacbb6 100644 --- a/cmd/grpc/gptneox/main.go +++ b/backend/go/llm/gptneox/main.go @@ -5,7 +5,7 @@ package main import ( "flag" - transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers" + transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers" grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) diff --git a/pkg/backend/llm/langchain/langchain.go b/backend/go/llm/langchain/langchain.go similarity index 98% rename from pkg/backend/llm/langchain/langchain.go rename to backend/go/llm/langchain/langchain.go index 5d5f94bd578f..f1dee3605168 100644 --- a/pkg/backend/llm/langchain/langchain.go +++ b/backend/go/llm/langchain/langchain.go @@ -1,4 +1,4 @@ -package langchain +package main // This is a wrapper to statisfy the GRPC service interface // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc) diff --git a/backend/go/llm/langchain/main.go b/backend/go/llm/langchain/main.go new file mode 100644 index 000000000000..79829efded76 --- /dev/null +++ b/backend/go/llm/langchain/main.go @@ -0,0 +1,21 @@ +package main + +// Note: this is started internally by LocalAI and a server is allocated for each model + +import ( + "flag" + + grpc "github.com/go-skynet/LocalAI/pkg/grpc" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +func main() { + flag.Parse() + + if err := grpc.StartServer(*addr, &LLM{}); err != nil { + panic(err) + } +} diff --git a/pkg/backend/llm/llama-stable/llama.go b/backend/go/llm/llama-stable/llama.go similarity index 99% rename from pkg/backend/llm/llama-stable/llama.go rename to backend/go/llm/llama-stable/llama.go index b9e25754184e..321d476ac003 100644 --- a/pkg/backend/llm/llama-stable/llama.go +++ b/backend/go/llm/llama-stable/llama.go @@ -1,4 +1,4 @@ -package llama +package main // This is a wrapper to statisfy the GRPC service interface // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc) diff --git a/cmd/grpc/llama-stable/main.go b/backend/go/llm/llama-stable/main.go similarity index 61% rename from cmd/grpc/llama-stable/main.go rename to backend/go/llm/llama-stable/main.go index afe98f7df083..a2e61270bb3f 100644 --- a/cmd/grpc/llama-stable/main.go +++ b/backend/go/llm/llama-stable/main.go @@ -3,8 +3,6 @@ package main import ( "flag" - llama "github.com/go-skynet/LocalAI/pkg/backend/llm/llama-stable" - grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) @@ -15,7 +13,7 @@ var ( func main() { flag.Parse() - if err := grpc.StartServer(*addr, &llama.LLM{}); err != nil { + if err := grpc.StartServer(*addr, &LLM{}); err != nil { panic(err) } } diff --git a/pkg/backend/llm/llama/llama.go b/backend/go/llm/llama/llama.go similarity index 99% rename from pkg/backend/llm/llama/llama.go rename to backend/go/llm/llama/llama.go index d461d9f2c2b5..34eb03c28d01 100644 --- a/pkg/backend/llm/llama/llama.go +++ b/backend/go/llm/llama/llama.go @@ -1,4 +1,4 @@ -package llama +package main // This is a wrapper to statisfy the GRPC service interface // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc) diff --git a/cmd/grpc/piper/main.go b/backend/go/llm/llama/main.go similarity index 72% rename from cmd/grpc/piper/main.go rename to backend/go/llm/llama/main.go index d77189ae1562..42868e96ab36 100644 --- a/cmd/grpc/piper/main.go +++ b/backend/go/llm/llama/main.go @@ -1,12 +1,12 @@ package main +// GRPC Falcon server + // Note: this is started internally by LocalAI and a server is allocated for each model import ( "flag" - tts "github.com/go-skynet/LocalAI/pkg/backend/tts" - grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) @@ -17,7 +17,7 @@ var ( func main() { flag.Parse() - if err := grpc.StartServer(*addr, &tts.Piper{}); err != nil { + if err := grpc.StartServer(*addr, &LLM{}); err != nil { panic(err) } } diff --git a/cmd/grpc/mpt/main.go b/backend/go/llm/mpt/main.go similarity index 83% rename from cmd/grpc/mpt/main.go rename to backend/go/llm/mpt/main.go index 6f46d187e5ce..3455078fa158 100644 --- a/cmd/grpc/mpt/main.go +++ b/backend/go/llm/mpt/main.go @@ -5,7 +5,7 @@ package main import ( "flag" - transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers" + transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers" grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) diff --git a/cmd/grpc/replit/main.go b/backend/go/llm/replit/main.go similarity index 83% rename from cmd/grpc/replit/main.go rename to backend/go/llm/replit/main.go index 0f54b6abf013..df605149dc55 100644 --- a/cmd/grpc/replit/main.go +++ b/backend/go/llm/replit/main.go @@ -5,7 +5,7 @@ package main import ( "flag" - transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers" + transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers" grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) diff --git a/backend/go/llm/rwkv/main.go b/backend/go/llm/rwkv/main.go new file mode 100644 index 000000000000..79829efded76 --- /dev/null +++ b/backend/go/llm/rwkv/main.go @@ -0,0 +1,21 @@ +package main + +// Note: this is started internally by LocalAI and a server is allocated for each model + +import ( + "flag" + + grpc "github.com/go-skynet/LocalAI/pkg/grpc" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +func main() { + flag.Parse() + + if err := grpc.StartServer(*addr, &LLM{}); err != nil { + panic(err) + } +} diff --git a/pkg/backend/llm/rwkv/rwkv.go b/backend/go/llm/rwkv/rwkv.go similarity index 99% rename from pkg/backend/llm/rwkv/rwkv.go rename to backend/go/llm/rwkv/rwkv.go index 501e56995ac3..d315fcc2f218 100644 --- a/pkg/backend/llm/rwkv/rwkv.go +++ b/backend/go/llm/rwkv/rwkv.go @@ -1,4 +1,4 @@ -package rwkv +package main // This is a wrapper to statisfy the GRPC service interface // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc) diff --git a/cmd/grpc/starcoder/main.go b/backend/go/llm/starcoder/main.go similarity index 83% rename from cmd/grpc/starcoder/main.go rename to backend/go/llm/starcoder/main.go index c08cef202527..38534d6c09d7 100644 --- a/cmd/grpc/starcoder/main.go +++ b/backend/go/llm/starcoder/main.go @@ -5,7 +5,7 @@ package main import ( "flag" - transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers" + transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers" grpc "github.com/go-skynet/LocalAI/pkg/grpc" ) diff --git a/pkg/backend/llm/transformers/dolly.go b/backend/go/llm/transformers/dolly.go similarity index 100% rename from pkg/backend/llm/transformers/dolly.go rename to backend/go/llm/transformers/dolly.go diff --git a/pkg/backend/llm/transformers/falcon.go b/backend/go/llm/transformers/falcon.go similarity index 100% rename from pkg/backend/llm/transformers/falcon.go rename to backend/go/llm/transformers/falcon.go diff --git a/pkg/backend/llm/transformers/gpt2.go b/backend/go/llm/transformers/gpt2.go similarity index 100% rename from pkg/backend/llm/transformers/gpt2.go rename to backend/go/llm/transformers/gpt2.go diff --git a/pkg/backend/llm/transformers/gptj.go b/backend/go/llm/transformers/gptj.go similarity index 100% rename from pkg/backend/llm/transformers/gptj.go rename to backend/go/llm/transformers/gptj.go diff --git a/pkg/backend/llm/transformers/gptneox.go b/backend/go/llm/transformers/gptneox.go similarity index 100% rename from pkg/backend/llm/transformers/gptneox.go rename to backend/go/llm/transformers/gptneox.go diff --git a/pkg/backend/llm/transformers/mpt.go b/backend/go/llm/transformers/mpt.go similarity index 100% rename from pkg/backend/llm/transformers/mpt.go rename to backend/go/llm/transformers/mpt.go diff --git a/pkg/backend/llm/transformers/predict.go b/backend/go/llm/transformers/predict.go similarity index 100% rename from pkg/backend/llm/transformers/predict.go rename to backend/go/llm/transformers/predict.go diff --git a/pkg/backend/llm/transformers/replit.go b/backend/go/llm/transformers/replit.go similarity index 100% rename from pkg/backend/llm/transformers/replit.go rename to backend/go/llm/transformers/replit.go diff --git a/pkg/backend/llm/transformers/starcoder.go b/backend/go/llm/transformers/starcoder.go similarity index 100% rename from pkg/backend/llm/transformers/starcoder.go rename to backend/go/llm/transformers/starcoder.go diff --git a/backend/go/transcribe/main.go b/backend/go/transcribe/main.go new file mode 100644 index 000000000000..ad77da1a20be --- /dev/null +++ b/backend/go/transcribe/main.go @@ -0,0 +1,21 @@ +package main + +// Note: this is started internally by LocalAI and a server is allocated for each model + +import ( + "flag" + + grpc "github.com/go-skynet/LocalAI/pkg/grpc" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +func main() { + flag.Parse() + + if err := grpc.StartServer(*addr, &Whisper{}); err != nil { + panic(err) + } +} diff --git a/pkg/backend/transcribe/transcript.go b/backend/go/transcribe/transcript.go similarity index 99% rename from pkg/backend/transcribe/transcript.go rename to backend/go/transcribe/transcript.go index e428b324fb80..ebd43eca6b84 100644 --- a/pkg/backend/transcribe/transcript.go +++ b/backend/go/transcribe/transcript.go @@ -1,4 +1,4 @@ -package transcribe +package main import ( "fmt" diff --git a/pkg/backend/transcribe/whisper.go b/backend/go/transcribe/whisper.go similarity index 97% rename from pkg/backend/transcribe/whisper.go rename to backend/go/transcribe/whisper.go index 493d8229ea47..a033afb0cdbc 100644 --- a/pkg/backend/transcribe/whisper.go +++ b/backend/go/transcribe/whisper.go @@ -1,4 +1,4 @@ -package transcribe +package main // This is a wrapper to statisfy the GRPC service interface // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc) diff --git a/backend/go/tts/main.go b/backend/go/tts/main.go new file mode 100644 index 000000000000..faa6d5a582cb --- /dev/null +++ b/backend/go/tts/main.go @@ -0,0 +1,21 @@ +package main + +// Note: this is started internally by LocalAI and a server is allocated for each model + +import ( + "flag" + + grpc "github.com/go-skynet/LocalAI/pkg/grpc" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +func main() { + flag.Parse() + + if err := grpc.StartServer(*addr, &Piper{}); err != nil { + panic(err) + } +} diff --git a/pkg/backend/tts/piper.go b/backend/go/tts/piper.go similarity index 98% rename from pkg/backend/tts/piper.go rename to backend/go/tts/piper.go index 41f10049e699..fb025d82be1a 100644 --- a/pkg/backend/tts/piper.go +++ b/backend/go/tts/piper.go @@ -1,4 +1,4 @@ -package tts +package main // This is a wrapper to statisfy the GRPC service interface // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc) diff --git a/extra/grpc/README.md b/backend/python/README.md similarity index 100% rename from extra/grpc/README.md rename to backend/python/README.md diff --git a/extra/grpc/autogptq/Makefile b/backend/python/autogptq/Makefile similarity index 100% rename from extra/grpc/autogptq/Makefile rename to backend/python/autogptq/Makefile diff --git a/extra/grpc/autogptq/README.md b/backend/python/autogptq/README.md similarity index 100% rename from extra/grpc/autogptq/README.md rename to backend/python/autogptq/README.md diff --git a/extra/grpc/autogptq/autogptq.py b/backend/python/autogptq/autogptq.py similarity index 100% rename from extra/grpc/autogptq/autogptq.py rename to backend/python/autogptq/autogptq.py diff --git a/extra/grpc/autogptq/autogptq.yml b/backend/python/autogptq/autogptq.yml similarity index 100% rename from extra/grpc/autogptq/autogptq.yml rename to backend/python/autogptq/autogptq.yml diff --git a/extra/grpc/autogptq/backend_pb2.py b/backend/python/autogptq/backend_pb2.py similarity index 100% rename from extra/grpc/autogptq/backend_pb2.py rename to backend/python/autogptq/backend_pb2.py diff --git a/extra/grpc/autogptq/backend_pb2_grpc.py b/backend/python/autogptq/backend_pb2_grpc.py similarity index 100% rename from extra/grpc/autogptq/backend_pb2_grpc.py rename to backend/python/autogptq/backend_pb2_grpc.py diff --git a/extra/grpc/autogptq/run.sh b/backend/python/autogptq/run.sh similarity index 100% rename from extra/grpc/autogptq/run.sh rename to backend/python/autogptq/run.sh diff --git a/extra/grpc/bark/Makefile b/backend/python/bark/Makefile similarity index 100% rename from extra/grpc/bark/Makefile rename to backend/python/bark/Makefile diff --git a/extra/grpc/bark/README.md b/backend/python/bark/README.md similarity index 100% rename from extra/grpc/bark/README.md rename to backend/python/bark/README.md diff --git a/extra/grpc/bark/backend_pb2.py b/backend/python/bark/backend_pb2.py similarity index 100% rename from extra/grpc/bark/backend_pb2.py rename to backend/python/bark/backend_pb2.py diff --git a/extra/grpc/bark/backend_pb2_grpc.py b/backend/python/bark/backend_pb2_grpc.py similarity index 100% rename from extra/grpc/bark/backend_pb2_grpc.py rename to backend/python/bark/backend_pb2_grpc.py diff --git a/extra/grpc/bark/run.sh b/backend/python/bark/run.sh similarity index 100% rename from extra/grpc/bark/run.sh rename to backend/python/bark/run.sh diff --git a/extra/grpc/bark/test_ttsbark.py b/backend/python/bark/test_ttsbark.py similarity index 100% rename from extra/grpc/bark/test_ttsbark.py rename to backend/python/bark/test_ttsbark.py diff --git a/extra/grpc/bark/ttsbark.py b/backend/python/bark/ttsbark.py similarity index 100% rename from extra/grpc/bark/ttsbark.py rename to backend/python/bark/ttsbark.py diff --git a/extra/grpc/bark/ttsbark.yml b/backend/python/bark/ttsbark.yml similarity index 100% rename from extra/grpc/bark/ttsbark.yml rename to backend/python/bark/ttsbark.yml diff --git a/extra/grpc/diffusers/Makefile b/backend/python/diffusers/Makefile similarity index 100% rename from extra/grpc/diffusers/Makefile rename to backend/python/diffusers/Makefile diff --git a/extra/grpc/diffusers/README.md b/backend/python/diffusers/README.md similarity index 100% rename from extra/grpc/diffusers/README.md rename to backend/python/diffusers/README.md diff --git a/extra/grpc/diffusers/backend_diffusers.py b/backend/python/diffusers/backend_diffusers.py similarity index 100% rename from extra/grpc/diffusers/backend_diffusers.py rename to backend/python/diffusers/backend_diffusers.py diff --git a/extra/grpc/diffusers/backend_pb2.py b/backend/python/diffusers/backend_pb2.py similarity index 100% rename from extra/grpc/diffusers/backend_pb2.py rename to backend/python/diffusers/backend_pb2.py diff --git a/extra/grpc/diffusers/backend_pb2_grpc.py b/backend/python/diffusers/backend_pb2_grpc.py similarity index 100% rename from extra/grpc/diffusers/backend_pb2_grpc.py rename to backend/python/diffusers/backend_pb2_grpc.py diff --git a/extra/grpc/diffusers/diffusers.yml b/backend/python/diffusers/diffusers.yml similarity index 100% rename from extra/grpc/diffusers/diffusers.yml rename to backend/python/diffusers/diffusers.yml diff --git a/extra/grpc/diffusers/run.sh b/backend/python/diffusers/run.sh similarity index 100% rename from extra/grpc/diffusers/run.sh rename to backend/python/diffusers/run.sh diff --git a/extra/grpc/exllama/Makefile b/backend/python/exllama/Makefile similarity index 100% rename from extra/grpc/exllama/Makefile rename to backend/python/exllama/Makefile diff --git a/extra/grpc/exllama/README.md b/backend/python/exllama/README.md similarity index 100% rename from extra/grpc/exllama/README.md rename to backend/python/exllama/README.md diff --git a/extra/grpc/exllama/backend_pb2.py b/backend/python/exllama/backend_pb2.py similarity index 100% rename from extra/grpc/exllama/backend_pb2.py rename to backend/python/exllama/backend_pb2.py diff --git a/extra/grpc/exllama/backend_pb2_grpc.py b/backend/python/exllama/backend_pb2_grpc.py similarity index 100% rename from extra/grpc/exllama/backend_pb2_grpc.py rename to backend/python/exllama/backend_pb2_grpc.py diff --git a/extra/grpc/exllama/exllama.py b/backend/python/exllama/exllama.py similarity index 100% rename from extra/grpc/exllama/exllama.py rename to backend/python/exllama/exllama.py diff --git a/extra/grpc/exllama/exllama.yml b/backend/python/exllama/exllama.yml similarity index 100% rename from extra/grpc/exllama/exllama.yml rename to backend/python/exllama/exllama.yml diff --git a/extra/grpc/exllama/run.sh b/backend/python/exllama/run.sh similarity index 100% rename from extra/grpc/exllama/run.sh rename to backend/python/exllama/run.sh diff --git a/extra/grpc/huggingface/Makefile b/backend/python/huggingface/Makefile similarity index 100% rename from extra/grpc/huggingface/Makefile rename to backend/python/huggingface/Makefile diff --git a/extra/grpc/huggingface/README.md b/backend/python/huggingface/README.md similarity index 100% rename from extra/grpc/huggingface/README.md rename to backend/python/huggingface/README.md diff --git a/extra/grpc/huggingface/backend_pb2.py b/backend/python/huggingface/backend_pb2.py similarity index 100% rename from extra/grpc/huggingface/backend_pb2.py rename to backend/python/huggingface/backend_pb2.py diff --git a/extra/grpc/huggingface/backend_pb2_grpc.py b/backend/python/huggingface/backend_pb2_grpc.py similarity index 100% rename from extra/grpc/huggingface/backend_pb2_grpc.py rename to backend/python/huggingface/backend_pb2_grpc.py diff --git a/extra/grpc/huggingface/huggingface.py b/backend/python/huggingface/huggingface.py similarity index 100% rename from extra/grpc/huggingface/huggingface.py rename to backend/python/huggingface/huggingface.py diff --git a/extra/grpc/huggingface/huggingface.yml b/backend/python/huggingface/huggingface.yml similarity index 100% rename from extra/grpc/huggingface/huggingface.yml rename to backend/python/huggingface/huggingface.yml diff --git a/extra/grpc/huggingface/run.sh b/backend/python/huggingface/run.sh similarity index 100% rename from extra/grpc/huggingface/run.sh rename to backend/python/huggingface/run.sh diff --git a/extra/grpc/huggingface/test.sh b/backend/python/huggingface/test.sh similarity index 100% rename from extra/grpc/huggingface/test.sh rename to backend/python/huggingface/test.sh diff --git a/extra/grpc/huggingface/test_huggingface.py b/backend/python/huggingface/test_huggingface.py similarity index 100% rename from extra/grpc/huggingface/test_huggingface.py rename to backend/python/huggingface/test_huggingface.py diff --git a/extra/grpc/vall-e-x/Makefile b/backend/python/vall-e-x/Makefile similarity index 100% rename from extra/grpc/vall-e-x/Makefile rename to backend/python/vall-e-x/Makefile diff --git a/extra/grpc/vall-e-x/README.md b/backend/python/vall-e-x/README.md similarity index 100% rename from extra/grpc/vall-e-x/README.md rename to backend/python/vall-e-x/README.md diff --git a/extra/grpc/vall-e-x/backend_pb2.py b/backend/python/vall-e-x/backend_pb2.py similarity index 100% rename from extra/grpc/vall-e-x/backend_pb2.py rename to backend/python/vall-e-x/backend_pb2.py diff --git a/extra/grpc/vall-e-x/backend_pb2_grpc.py b/backend/python/vall-e-x/backend_pb2_grpc.py similarity index 100% rename from extra/grpc/vall-e-x/backend_pb2_grpc.py rename to backend/python/vall-e-x/backend_pb2_grpc.py diff --git a/extra/grpc/vall-e-x/run.sh b/backend/python/vall-e-x/run.sh similarity index 100% rename from extra/grpc/vall-e-x/run.sh rename to backend/python/vall-e-x/run.sh diff --git a/extra/grpc/vall-e-x/ttsvalle.py b/backend/python/vall-e-x/ttsvalle.py similarity index 100% rename from extra/grpc/vall-e-x/ttsvalle.py rename to backend/python/vall-e-x/ttsvalle.py diff --git a/extra/grpc/vall-e-x/ttsvalle.yml b/backend/python/vall-e-x/ttsvalle.yml similarity index 100% rename from extra/grpc/vall-e-x/ttsvalle.yml rename to backend/python/vall-e-x/ttsvalle.yml diff --git a/extra/grpc/vllm/Makefile b/backend/python/vllm/Makefile similarity index 100% rename from extra/grpc/vllm/Makefile rename to backend/python/vllm/Makefile diff --git a/extra/grpc/vllm/README.md b/backend/python/vllm/README.md similarity index 100% rename from extra/grpc/vllm/README.md rename to backend/python/vllm/README.md diff --git a/extra/grpc/vllm/backend_pb2.py b/backend/python/vllm/backend_pb2.py similarity index 100% rename from extra/grpc/vllm/backend_pb2.py rename to backend/python/vllm/backend_pb2.py diff --git a/extra/grpc/vllm/backend_pb2_grpc.py b/backend/python/vllm/backend_pb2_grpc.py similarity index 100% rename from extra/grpc/vllm/backend_pb2_grpc.py rename to backend/python/vllm/backend_pb2_grpc.py diff --git a/extra/grpc/vllm/backend_vllm.py b/backend/python/vllm/backend_vllm.py similarity index 100% rename from extra/grpc/vllm/backend_vllm.py rename to backend/python/vllm/backend_vllm.py diff --git a/extra/grpc/vllm/run.sh b/backend/python/vllm/run.sh similarity index 100% rename from extra/grpc/vllm/run.sh rename to backend/python/vllm/run.sh diff --git a/extra/grpc/vllm/test_backend_vllm.py b/backend/python/vllm/test_backend_vllm.py similarity index 100% rename from extra/grpc/vllm/test_backend_vllm.py rename to backend/python/vllm/test_backend_vllm.py diff --git a/extra/grpc/vllm/vllm.yml b/backend/python/vllm/vllm.yml similarity index 100% rename from extra/grpc/vllm/vllm.yml rename to backend/python/vllm/vllm.yml diff --git a/cmd/grpc/langchain-huggingface/main.go b/cmd/grpc/langchain-huggingface/main.go deleted file mode 100644 index 938908d2af22..000000000000 --- a/cmd/grpc/langchain-huggingface/main.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -// Note: this is started internally by LocalAI and a server is allocated for each model - -import ( - "flag" - - langchain "github.com/go-skynet/LocalAI/pkg/backend/llm/langchain" - - grpc "github.com/go-skynet/LocalAI/pkg/grpc" -) - -var ( - addr = flag.String("addr", "localhost:50051", "the address to connect to") -) - -func main() { - flag.Parse() - - if err := grpc.StartServer(*addr, &langchain.LLM{}); err != nil { - panic(err) - } -} diff --git a/cmd/grpc/llama/main.go b/cmd/grpc/llama/main.go deleted file mode 100644 index 442f84ef309c..000000000000 --- a/cmd/grpc/llama/main.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -// GRPC Falcon server - -// Note: this is started internally by LocalAI and a server is allocated for each model - -import ( - "flag" - - llama "github.com/go-skynet/LocalAI/pkg/backend/llm/llama" - - grpc "github.com/go-skynet/LocalAI/pkg/grpc" -) - -var ( - addr = flag.String("addr", "localhost:50051", "the address to connect to") -) - -func main() { - flag.Parse() - - if err := grpc.StartServer(*addr, &llama.LLM{}); err != nil { - panic(err) - } -} diff --git a/cmd/grpc/stablediffusion/main.go b/cmd/grpc/stablediffusion/main.go deleted file mode 100644 index d12fe3c50691..000000000000 --- a/cmd/grpc/stablediffusion/main.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -// Note: this is started internally by LocalAI and a server is allocated for each model - -import ( - "flag" - - image "github.com/go-skynet/LocalAI/pkg/backend/image" - - grpc "github.com/go-skynet/LocalAI/pkg/grpc" -) - -var ( - addr = flag.String("addr", "localhost:50051", "the address to connect to") -) - -func main() { - flag.Parse() - - if err := grpc.StartServer(*addr, &image.StableDiffusion{}); err != nil { - panic(err) - } -} diff --git a/cmd/grpc/whisper/main.go b/cmd/grpc/whisper/main.go deleted file mode 100644 index 4896c4f9bf6d..000000000000 --- a/cmd/grpc/whisper/main.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -// Note: this is started internally by LocalAI and a server is allocated for each model - -import ( - "flag" - - transcribe "github.com/go-skynet/LocalAI/pkg/backend/transcribe" - - grpc "github.com/go-skynet/LocalAI/pkg/grpc" -) - -var ( - addr = flag.String("addr", "localhost:50051", "the address to connect to") -) - -func main() { - flag.Parse() - - if err := grpc.StartServer(*addr, &transcribe.Whisper{}); err != nil { - panic(err) - } -} diff --git a/extra/requirements.txt b/extra/requirements.txt deleted file mode 100644 index fb3cc0122a9b..000000000000 --- a/extra/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -sentence_transformers -grpcio -google -protobuf -six -omegaconf -compel \ No newline at end of file diff --git a/pkg/grpc/proto/backend.pb.go b/pkg/grpc/proto/backend.pb.go index a9ac60571d01..259f8e0a60f0 100644 --- a/pkg/grpc/proto/backend.pb.go +++ b/pkg/grpc/proto/backend.pb.go @@ -2,7 +2,7 @@ // versions: // protoc-gen-go v1.26.0 // protoc v4.23.4 -// source: pkg/grpc/proto/backend.proto +// source: backend/backend.proto package proto diff --git a/pkg/grpc/proto/backend_grpc.pb.go b/pkg/grpc/proto/backend_grpc.pb.go index 1c3cd8f53f3b..5c97691df86a 100644 --- a/pkg/grpc/proto/backend_grpc.pb.go +++ b/pkg/grpc/proto/backend_grpc.pb.go @@ -2,7 +2,7 @@ // versions: // - protoc-gen-go-grpc v1.2.0 // - protoc v4.23.4 -// source: pkg/grpc/proto/backend.proto +// source: backend/backend.proto package proto @@ -453,5 +453,5 @@ var Backend_ServiceDesc = grpc.ServiceDesc{ ServerStreams: true, }, }, - Metadata: "pkg/grpc/proto/backend.proto", + Metadata: "backend/backend.proto", }