Testing Llama Stack 🦙

test_llama_stack.sh

# 0a. Download, install and activate Miniconda
# https://docs.anaconda.com/miniconda/
mkdir -p ~/miniconda3
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda3/miniconda.sh
bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
rm ~/miniconda3/miniconda.sh
~/miniconda3/bin/conda init bash
source ~/.bashrc

# 0b. Install Llama Stack
pip install llama-toolchain

# 1. Install distribution
llama distribution install --spec local --name local_llama_8b_instruct

# 2. Download model
llama download --source huggingface --model-id  Meta-Llama3.1-8B-Instruct --hf-token $HF_TOKEN

# 3. Start distribution
llama distribution start --name local_llama_8b_instruct --port 5000 --disable-ipv6

# 4. Test it!
conda activate conda activate local_llama_8b_instruct
python -m  llama_toolchain.inference.client localhost 5000