From 03a6677a6ae19b88ac5b8363544e339ba01a054d Mon Sep 17 00:00:00 2001 From: FangYin Cheng Date: Sat, 16 Sep 2023 23:58:47 +0800 Subject: [PATCH 1/2] chore: add autodl setup script --- scripts/setup_autodl_env.sh | 94 ++++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 11 deletions(-) diff --git a/scripts/setup_autodl_env.sh b/scripts/setup_autodl_env.sh index 9ec27e0c8..e2594a5c2 100644 --- a/scripts/setup_autodl_env.sh +++ b/scripts/setup_autodl_env.sh @@ -1,19 +1,91 @@ #!/bin/bash -eval "$(conda shell.bash hook)" +# This script is used for setting up the environment required for DB-GPT on https://www.autodl.com/ -source ~/.bashrc +# Usage: source /etc/network_turbo && curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/eosphoros-ai/DB-GPT/main/scripts/setup_autodl_env.sh | bash -# source /etc/network_turbo -# unset http_proxy && unset https_proxy -conda create -n dbgpt python=3.10 -y +DEFAULT_PROXY="true" +USE_PROXY=$DEFAULT_PROXY -conda activate dbgpt +initialize_conda() { + eval "$(conda shell.bash hook)" + source ~/.bashrc + if [[ $USE_PROXY == "true" ]]; then + source /etc/network_turbo + # unset http_proxy && unset https_proxy + fi +} -apt-get update -y && apt-get install git-lfs -y +setup_conda_environment() { + conda create -n dbgpt python=3.10 -y + conda activate dbgpt +} -cd /root && git clone https://github.com/eosphoros-ai/DB-GPT.git +install_sys_packages() { + apt-get update -y && apt-get install git-lfs -y +} -mkdir -p /root/DB-GPT/models && cd /root/DB-GPT/models +clone_repositories() { + cd /root && git clone https://github.com/eosphoros-ai/DB-GPT.git + mkdir -p /root/DB-GPT/models && cd /root/DB-GPT/models + git clone https://huggingface.co/GanymedeNil/text2vec-large-chinese + git clone https://huggingface.co/THUDM/chatglm2-6b-int4 + rm -rf /root/DB-GPT/models/text2vec-large-chinese/.git + rm -rf /root/DB-GPT/models/chatglm2-6b-int4/.git +} -git clone https://huggingface.co/GanymedeNil/text2vec-large-chinese -git clone https://huggingface.co/THUDM/chatglm2-6b-int4 \ No newline at end of file +install_dbgpt_packages() { + conda activate dbgpt && cd /root/DB-GPT && pip install -e . && cp .env.template .env + cp .env.template .env && sed -i 's/LLM_MODEL=vicuna-13b-v1.5/LLM_MODEL=chatglm2-6b-int4/' .env + +} + +clean_up() { + rm -rf `pip cache dir` + apt-get clean + rm ~/.bash_history + history -c +} + +usage() { + echo "USAGE: $0 [--use-proxy]" + echo " [--use-proxy] Use proxy settings (Optional)" + echo " [-h|--help] Usage message" +} + +# Command line arguments parsing +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --use-proxy) + USE_PROXY="true" + shift + ;; + -h|--help) + help="true" + shift + ;; + *) + usage + exit 1 + ;; + esac +done + +if [[ $help ]]; then + usage + exit 0 +fi + +# Main execution + +if [[ $USE_PROXY == "true" ]]; then + echo "Using proxy settings..." + source /etc/network_turbo +fi + +initialize_conda +setup_conda_environment +install_sys_packages +clone_repositories +install_dbgpt_packages +clean_up From 71b745aa0244774015600dc7add1434fcbd6d698 Mon Sep 17 00:00:00 2001 From: FangYin Cheng Date: Sun, 17 Sep 2023 17:00:00 +0800 Subject: [PATCH 2/2] chore: add autodl online image address --- README.md | 2 +- README.zh.md | 2 +- pilot/embedding_engine/loader/ppt_loader.py | 2 +- scripts/setup_autodl_env.sh | 16 +++++++++++++++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 84a554006..882570c21 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ The core capabilities mainly consist of the following parts: - [DB-GPT-Web](https://github.com/eosphoros-ai/DB-GPT-Web) ChatUI for DB-GPT ## Image -๐ŸŒ [AutoDL Image](https://www.codewithgpu.com/i/csunny/DB-GPT/dbgpt-0.3.1-v2) +๐ŸŒ [AutoDL Image](https://www.codewithgpu.com/i/eosphoros-ai/DB-GPT/dbgpt) ## Install ![Docker](https://img.shields.io/badge/docker-%230db7ed.svg?style=for-the-badge&logo=docker&logoColor=white) diff --git a/README.zh.md b/README.zh.md index e54120c58..abebc1c9e 100644 --- a/README.zh.md +++ b/README.zh.md @@ -156,7 +156,7 @@ DB-GPTๅŸบไบŽ [FastChat](https://github.com/lm-sys/FastChat) ๆž„ๅปบๅคงๆจกๅž‹่ฟ ## Image -๐ŸŒ [AutoDL้•œๅƒ](https://www.codewithgpu.com/i/csunny/DB-GPT/dbgpt-0.3.1-v2) +๐ŸŒ [AutoDL้•œๅƒ](https://www.codewithgpu.com/i/eosphoros-ai/DB-GPT/dbgpt) ๐ŸŒ [้˜ฟ้‡Œไบ‘้•œๅƒ](http://dbgpt.site/web/#/p/dc4bb97e0bc15302dbf3a5d5571142dd) diff --git a/pilot/embedding_engine/loader/ppt_loader.py b/pilot/embedding_engine/loader/ppt_loader.py index 04f5e5c93..0bfaf9679 100644 --- a/pilot/embedding_engine/loader/ppt_loader.py +++ b/pilot/embedding_engine/loader/ppt_loader.py @@ -19,7 +19,7 @@ def load(self) -> List[Document]: docs = [] for slide in pr.slides: for shape in slide.shapes: - if hasattr(shape, "text") and shape.text is not "": + if hasattr(shape, "text") and shape.text: docs.append( Document( page_content=shape.text, metadata={"source": slide.slide_id} diff --git a/scripts/setup_autodl_env.sh b/scripts/setup_autodl_env.sh index e2594a5c2..2eafbcebb 100644 --- a/scripts/setup_autodl_env.sh +++ b/scripts/setup_autodl_env.sh @@ -3,10 +3,17 @@ # Usage: source /etc/network_turbo && curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/eosphoros-ai/DB-GPT/main/scripts/setup_autodl_env.sh | bash +# autodl usage: +# conda activate dbgpt +# cd /root/DB-GPT +# bash scripts/examples/load_examples.sh +# dbgpt start webserver --port 6006 + DEFAULT_PROXY="true" USE_PROXY=$DEFAULT_PROXY initialize_conda() { + conda init bash eval "$(conda shell.bash hook)" source ~/.bashrc if [[ $USE_PROXY == "true" ]]; then @@ -42,10 +49,17 @@ install_dbgpt_packages() { clean_up() { rm -rf `pip cache dir` apt-get clean - rm ~/.bash_history + rm -f ~/.bash_history history -c } +clean_local_data() { + rm -rf /root/DB-GPT/pilot/data + rm -rf /root/DB-GPT/pilot/message + rm -f /root/DB-GPT/logs/* + rm -f /root/DB-GPT/logsDbChatOutputParser.log +} + usage() { echo "USAGE: $0 [--use-proxy]" echo " [--use-proxy] Use proxy settings (Optional)"