Running LLM on AWS

running_llm_on_aws.sh

#!/bin/sh
# The cheapest CUDA instance on AWS is Graviton2 based g5g instances
# 1) Install right CUDA Drivers on the system. Whilst this script has been tested on Ubuntu 22.04. It should work on others
# 2) Install torch with GPU support. Compiling from source works the best 
# 3) Preferably Quantize the models - https://huggingface.co/docs/transformers/main/en/main_classes/quantization
#
# Author: Supreet Sethi <[email protected]>
# Web: https://www.linkedin.com/in/djinn
# 
# Step 1) is here 

ARCH='sbsa'
NAME='ubuntu'
REL=`lsb_release -rs|tr -d '.'`
RELEASE="${NAME}${REL}"
DEVICE=`lspci |grep -i nvidia|cut -d':' -f 3`

sudo apt install -y linux-headers-$(uname -r)
wget https://developer.download.nvidia.com/compute/cuda/repos/${RELEASE}/${ARCH}/cuda-keyring_1.0-1_all.deb && sudo dpkg -i cuda-keyring_1.0-1_all.deb
sudo echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/${RELEASE}/${ARCH}/ /" | sudo tee /etc/apt/sources.list.d/cuda-${RELEASE}-${ARCH}.list
wget https://developer.download.nvidia.com/compute/cuda/repos/${RELEASE}/${ARCH}/cuda-${RELEASE}.pin
sudo mv cuda-${RELEASE}.pin /etc/apt/preferences.d/cuda-repository-pin-600
sudo apt update && sudo apt -y install cuda nvidia-cuda-toolkit