llama-2-7b-chat-hf.yaml

port: 8000
name: llama-2-7b-chat-hf
route_prefix: /llama-2-7b-chat-hf
num_replicas: 1
device: cpu
cpus_per_worker: 24
vllm:
  enabled: true
  precision: bf16
model_description:
  model_id_or_path: NousResearch/Llama-2-7b-chat-hf
  tokenizer_name_or_path: NousResearch/Llama-2-7b-chat-hf
  config:
    use_auth_token: ''