Basic INTERACTIVE install of text-generation-webui to host LLM on Ubuntu - Launch for 24GB GPU such as AWS EC2 G4 or G5 box

install_text_gen.sh

#!/bin/bash

GPU_GB=${1:-24}

#sudo apt update
#sudo apt install python3-pip

git clone https://github.com/oobabooga/text-generation-webui.git

pushd text-generation-webui

# python3 download-model.py    TheBloke/CodeLlama-13B-GGUF
python3 download-model.py    mistralai/Mistral-7B-v0.1

#python server.py --auto-devices --gpu-memory 24 --share

# IS interactive - so NOT suitable for auto install via User Data
GPU_CHOICE=A USE_CUDA118=FALSE LAUNCH_AFTER_INSTALL=TRUE INSTALL_EXTENSIONS=TRUE ./start_linux.sh --public-api --extensions openai --api-port 5000 --auto-devices --gpu-memory $GPU_GB --share