Some API changes (#125)

Some quality of life changes to API server. return number of tokens, token logprob and whether it finishes the request (in streaming). update dockerfile for better caching. formatting done with black. --------- Co-authored-by: hiworldwzj <[email protected]>
ModelTC · Sep 18, 2023 · 82cb0a4 · 82cb0a4
1 parent 87557d1
commit 82cb0a4
Show file tree

Hide file tree

Showing 5 changed files with 218 additions and 103 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -11,26 +11,26 @@ ENV PATH=/opt/conda/bin:$PATH \
     CONDA_PREFIX=/opt/conda
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-        ca-certificates \
-        libssl-dev \
-        curl \
-        g++ \
-        make \
-        git && \
-        rm -rf /var/lib/apt/lists/*
+    ca-certificates \
+    libssl-dev \
+    curl \
+    g++ \
+    make \
+    git && \
+    rm -rf /var/lib/apt/lists/*
 
 RUN case ${TARGETPLATFORM} in \
-         "linux/arm64")  MAMBA_ARCH=aarch64  ;; \
-         *)              MAMBA_ARCH=x86_64   ;; \
+    "linux/arm64")  MAMBA_ARCH=aarch64  ;; \
+    *)              MAMBA_ARCH=x86_64   ;; \
     esac && \
     curl -fsSL -o ~/mambaforge.sh -v "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \
     bash ~/mambaforge.sh -b -p /opt/conda && \
     rm ~/mambaforge.sh
 
 RUN case ${TARGETPLATFORM} in \
-         "linux/arm64")  exit 1 ;; \
-         *)              /opt/conda/bin/conda update -y conda &&  \
-                        /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -c "${CUDA_CHANNEL}" -y "python=${PYTHON_VERSION}" pytorch==$PYTORCH_VERSION "pytorch-cuda=$(echo $CUDA_VERSION | cut -d'.' -f 1-2)" -c anaconda -c conda-forge ;; \
+    "linux/arm64")  exit 1 ;; \
+    *)              /opt/conda/bin/conda update -y conda &&  \
+    /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -c "${CUDA_CHANNEL}" -y "python=${PYTHON_VERSION}" pytorch==$PYTORCH_VERSION "pytorch-cuda=$(echo $CUDA_VERSION | cut -d'.' -f 1-2)" -c anaconda -c conda-forge ;; \
     esac && \
     /opt/conda/bin/conda clean -ya
 
@@ -46,6 +46,8 @@ RUN mkdir ~/cuda-nvcc && cd ~/cuda-nvcc && \
 
 WORKDIR /root
 
+COPY ./requirements.txt /lightllm/requirements.txt
+RUN pip install -r /lightllm/requirements.txt --no-cache-dir
+
 COPY . /lightllm
-RUN pip install -r /lightllm/requirements.txt --no-cache-dir && \
-    pip install -e /lightllm --no-cache-dir
+RUN pip install -e /lightllm --no-cache-dir
diff --git a/build_and_upload_docker.sh b/build_and_upload_docker.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Build and push docker image to AWS ECR.
+
+set -eo pipefail
+
+if [ -z "$1" ]; then
+  echo "Must supply AWS account ID"
+  exit 1;
+fi
+
+if [ -z "$2" ]; then
+  echo "Must supply the image tag"
+  exit 1;
+fi
+
+IMAGE_TAG=$2
+ACCOUNT=$1
+aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin $ACCOUNT.dkr.ecr.us-west-2.amazonaws.com
+DOCKER_BUILDKIT=1 docker build -t $ACCOUNT.dkr.ecr.us-west-2.amazonaws.com/lightllm:$IMAGE_TAG .
+docker push $ACCOUNT.dkr.ecr.us-west-2.amazonaws.com/lightllm:$IMAGE_TAG