diff --git a/custom-server/deepseek-v2-5-instruct-sglang/config.yaml b/custom-server/deepseek-v2-5-instruct-sglang/config.yaml new file mode 100644 index 00000000..e85af908 --- /dev/null +++ b/custom-server/deepseek-v2-5-instruct-sglang/config.yaml @@ -0,0 +1,18 @@ +base_image: + image: lmsysorg/sglang:v0.4.0.post1-cu124 +model_metadata: + repo_id: deepseek-ai/DeepSeek-V2.5-1210 +docker_server: + start_command: sh -c "HF_TOKEN=$(cat /secrets/hf_access_token) python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V2.5-1210 --port 8000 --tp 8 --trust-remote-code" + readiness_endpoint: /health + liveness_endpoint: /health + predict_endpoint: /v1/completions + server_port: 8000 +resources: + accelerator: H100:8 + use_gpu: true +runtime: + predict_concurrency : 32 +model_name: DeepSeek V2.5 1210 SGLang +environment_variables: + hf_access_token: null