Run Llama-2-13B-chat locally on your M1/M2 Mac with GPU inference. Uses 10GB RAM

llama2-mac-gpu.sh

# Clone llama.cpp
git clone https://github.com/ggerganov/llama.cpp.git
cd llama.cpp

# Build it
LLAMA_METAL=1 make

# Download model
export MODEL=llama-2-13b-chat.ggmlv3.q4_0.bin
wget "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/resolve/main/${MODEL}"

# Run
echo "Prompt: " \
    && read PROMPT \
    && ./main \
        -t 8 \
        -ngl 1 \
        -m ${MODEL} \
        --color \
        -c 2048 \
        --temp 0.7 \
        --repeat_penalty 1.1 \
        -n -1 \
        -p "### Instruction: ${PROMPT} \n### Response:"