Setup for lambdalabs.com boxes

!lambdalabs-setup.md

After booting a new instance:

curl -H 'Cache-Control: no-cache' https://gist.githubusercontent.com/aidando73/2876aabaae7ded0dec68d04d34b70086/raw/new_instance.bash | bash

If you want to use conda:

source ~/miniconda3/bin/activate

New attached storage:

mkdir -p "$STORAGE_DIR/.ssh"
ssh-keygen -t ed25519 -C "[email protected]" -f $STORAGE_DIR/.ssh/id_ed25519
eval "$(ssh-agent -s)"
echo -e "Host github.com\n\tIdentityFile $STORAGE_DIR/.ssh/id_ed25519" >> $STORAGE_DIR/.ssh/config
echo -e "Host hf.co\n\tIdentityFile $STORAGE_DIR/.ssh/id_ed25519" >> $STORAGE_DIR/.ssh/config
echo -e "Go to https://github.com/settings/ssh/new and add:\n\tkeyname: lambdalabs.com/$(basename $STORAGE_DIR)\n\tvalue:   $(cat $STORAGE_DIR/.ssh/id_ed25519.pub)"
echo -e "Go to https://huggingface.co/settings/keys/add?type=ssh and add:\n\tkeyname: lambdalabs.com/$(basename $STORAGE_DIR)\n\tvalue:   $(cat $STORAGE_DIR/.ssh/id_ed25519.pub)"

.bashrc

# Logs start and end timestamps
# Outputs stdin and stderr into specially named log file to differentiate different boxes
# (e.g., when committing to same repo)
track() {
  {
    echo "Start time: $(date)";
    echo "Running command: $*";
    time $*;
    echo "End time: $(date)";
  } 2>&1 | tee -a log-$(hostname)
} 

# Wrapper around nohup
run_nohup() {
  echo "Start time: $(date)" >> "log-$(hostname)";
  echo "Running command: $*" >> "log-$(hostname)";
  nohup bash -c "time $*"'; echo "end: $(date)"' >> "log-$(hostname)" 2>&1 &
  tail -n 2 -f "log-$(hostname)"
}

eval "$(direnv hook bash)"

new_instance.bash

cd ~
storage_dir=~/"$(ls -1 ~ | grep -E 'us-east-1|1xa100-2|us-west-3|us-south-2|us-west-1')"

echo "downloading .bashrc file"
wget -O $storage_dir/.bashrc https://gist.githubusercontent.com/aidando73/2876aabaae7ded0dec68d04d34b70086/raw/.bashrc

echo "Keep hugging face cache inside attached storage"
echo "" >> "${storage_dir}/.bashrc"
echo "export HF_HUB_CACHE=$storage_dir/.cache/huggingface/hub/" >> "${storage_dir}/.bashrc"

echo "Load storage dir .bashrc on startup"
echo "export STORAGE_DIR=$storage_dir" >> "${storage_dir}/.bashrc"
echo "source ${storage_dir}/.bashrc" >> ~/.bashrc

echo "setting llama-stack specific environment variables"
echo 'export LLAMA_STACK_CONFIG_DIR=$STORAGE_DIR/.llama/' >> "${storage_dir}/.bashrc"

echo "setting ollama models folder"
echo 'export OLLAMA_MODELS=$STORAGE_DIR/.ollama/models/' >> "${storage_dir}/.bashrc"

echo "Setting up ssh config"
echo -e "Include $storage_dir/.ssh/config\n" >> ~/.ssh/config

echo "Setting up git config"
git config --global user.email "[email protected]"
git config --global user.name "Aidan Do"
git config --global core.editor "vim"
git config --global init.defaultBranch main

echo "Install direnv"
sudo apt install direnv
mkdir -p ~/.config/direnv
echo -e "[global]\nload_dotenv=true" > ~/.config/direnv/direnv.toml

echo "Installing conda"
mkdir -p ~/miniconda3
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda3/miniconda.sh
bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
rm ~/miniconda3/miniconda.sh

echo "loading ~/.bashrc"
source ~/.bashrc
echo "done ✅"

vllm-setup.bash

# Setup
conda create --prefix ./vllm python=3.12 -y
conda activate ./vllm
pip install vllm
pip install bitsandbytes>=0.45.0

# Run
vllm serve unsloth/Llama-3.3-70B-Instruct-bnb-4bit \
    --enable-lora \
    --lora-modules v1=aidando73/llama-3.3-70b-instruct-code-agent-fine-tune-v1 \
    --port 8000 \
    --quantization bitsandbytes \
    --dtype bfloat16 \
    --trust-remote-code \
    --max-model-len 50_000 \
    --load-format bitsandbytes