start-script for running a locally built version of ollama but mounted into a docker container

startollama.sh

#!/bin/bash

cd $(dirname $0) || exit 1

OLLAMA_MODELDIR="${HOME}/ollama_models"
OLLAMA_RUNDIR="${HOME}/ollama"
OLLAMA_ALLOW_LOCAL_EXECUTABLE=1
OLLAMA_NUM_PARALLEL=2

echo $0 :: PWD: $(pwd)
echo $0 :: OLLAMA_RUNDIR: ${OLLAMA_RUNDIR}
echo $0 :: OLLAMA_MODELDIR: ${OLLAMA_MODELDIR}
echo $0 :: OLLAMA_ALLOW_LOCAL_EXECUTABLE: ${OLLAMA_ALLOW_LOCAL_EXECUTABLE}


# https://github.com/ollama/ollama/blob/main/README.md#quickstart
# https://hub.docker.com/r/ollama/ollama

# curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
# curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
# sudo apt-get update
# sudo apt-get install -y nvidia-container-toolkit
# sudo nvidia-ctk runtime configure --runtime=docker
# sudo systemctl restart docker

docker_cmd="docker run --gpus=all --rm -v ${OLLAMA_MODELDIR}:/ollama_models -p 11434:11434 -e OLLAMA_MODELS=/ollama_models -e OLLAMA_NUM_PARALLEL=${OLLAMA_NUM_PARALLEL}"
#-e OLLAMA_LLAMA_EXTRA_ARGS='--override-kv tokenizer.ggml.pre=str:llama3'"
# https://github.com/ollama/ollama/pull/4120#issuecomment-2094747527

if [ -x ${OLLAMA_RUNDIR}/ollama ] ; then
	if [[ ${OLLAMA_ALLOW_LOCAL_EXECUTABLE} == 1 ]] ; then
		echo "found locally compiled ollama -> using that one (${OLLAMA_RUNDIR}/ollama)" 
		docker_cmd="${docker_cmd} -v ${OLLAMA_RUNDIR}/ollama:/usr/bin/ollama -e GIN_MODE=release"
	else
		echo "found locally compiled ollama -> but disabled using that..." 
	fi
fi

if [ -t 0 ] ; then
  echo $0 :: TTY detected starting in FG
  echo $docker_cmd -it --name ollama ollama/ollama # serve --override-kv tokenizer.ggml.pre=str:llama3
  $docker_cmd -it --name ollama ollama/ollama
else
  echo $0 :: no TTY detected starting in BG
  echo $docker_cmd -d --name ollama ollama/ollama # serve --override-kv tokenizer.ggml.pre=str:llama
  $docker_cmd -d --name ollama ollama/ollama
fi