serve.yml

resources:
  cloud: azure
  accelerators: {A100-80GB:2, A100-80GB:4, A100:8, A100-80GB:8}
  ports: 8000

setup: |
  conda activate mixtral
  if [ $? -ne 0 ]; then
	conda create -n mixtral -y python=3.10
	conda activate mixtral
  fi
  pip install transformers==4.38.0
  pip install vllm==0.3.2
  pip list | grep megablocks || pip install megablocks

run: |
  conda activate mixtral
  export PATH=$PATH:/sbin
  python -u -m vllm.entrypoints.openai.api_server \
            	--host 0.0.0.0 \
            	--model mistralai/Mixtral-8x7B-Instruct-v0.1 \
            	--tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE | tee ~/openai_api_server.log