mistral-7b-v0.1.yaml

port: 8000
name: mistral-7b-v0.1
route_prefix: /mistral-7b-v0.1
num_replicas: 1
device: cpu
cpus_per_worker: 24
vllm:
  enabled: true
  precision: bf16
model_description:
  model_id_or_path: mistralai/Mistral-7B-v0.1
  ipexllm: false
  tokenizer_name_or_path: mistralai/Mistral-7B-v0.1
  config:
    trust_remote_code: true
  chat_template: "llm_on_ray/inference/models/templates/template_mistral.jinja"