Convert Finnish-NLP/Ahma-3B to gguf format and install it to ollama

ollama-ahma-3b-convert.sh

#!/usr/bin/env bash

# Check if Git LFS is installed
if ! command -v git-lfs &> /dev/null; then
    echo "Error: git-lfs is not installed. Please install it first." >&2
    exit 1
fi
git lfs install

# Clone model repo
git clone https://huggingface.co/Finnish-NLP/Ahma-3B || { echo "Failed to clone Ahma-3B"; exit 1; }

# Clone llama.cpp
git clone https://github.com/ggerganov/llama.cpp || { echo "Failed to clone llama.cpp"; exit 1; }

# Install llama.cpp dependencies using pipx (if supported by pipx)
if ! command -v pipx &> /dev/null; then
    echo "Error: pipx is not installed. Please install it first." >&2
    exit 1
fi

# Install llama.cpp tools with pipx
pipx install llama.cpp || { echo "Failed to install llama.cpp with pipx"; exit 1; }

# Convert model
cd llama.cpp || exit 1
python convert-hf-to-gguf.py --outtype f16 ../Ahma-3B/ || { echo "Model conversion failed"; exit 1; }

# Define model path
MODEL_PATH="$(pwd)/ahma_3b_1.gguf"

# Check available GPU memory and set num_ctx dynamically
if command -v nvidia-smi &> /dev/null; then
    TOTAL_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | awk '{print $1}')
elif command -v rocm-smi &> /dev/null; then
    TOTAL_MEM=$(rocm-smi --showmeminfo vram | grep "Total Memory" | awk '{print $3}')
elif [[ $(uname) == "Darwin" ]]; then
    TOTAL_MEM=$(sysctl -n hw.memsize)
    TOTAL_MEM=$((TOTAL_MEM / 1024 / 1024)) # Convert bytes to MB
else
    echo "No supported GPU detected. Using default num_ctx=2048."
    TOTAL_MEM=0
fi

# Adjust num_ctx based on available memory
if [[ $TOTAL_MEM -ge 24000 ]]; then
    NUM_CTX=4096
elif [[ $TOTAL_MEM -ge 16000 ]]; then
    NUM_CTX=3072
elif [[ $TOTAL_MEM -ge 8000 ]]; then
    NUM_CTX=2048
else
    NUM_CTX=1024
fi

# Verify model integrity
cd ../Ahma-3B || exit 1
if wget -q --spider https://huggingface.co/Finnish-NLP/Ahma-3B/SHA256SUMS; then
    wget -O SHA256SUMS https://huggingface.co/Finnish-NLP/Ahma-3B/SHA256SUMS
    sha256sum -c SHA256SUMS --ignore-missing
    if [[ $? -ne 0 ]]; then
        echo "Model file is corrupted! Exiting..."
        exit 1
    fi
else
    echo "Warning: No checksum file found for verification."
fi

# Setup custom model file
cat <<EOF > Modelfile
FROM "$MODEL_PATH"
SYSTEM """Olet tekoälyavustaja..."""
PARAMETER num_ctx $NUM_CTX
TEMPLATE "[INST] <<SYS>>{{ .System }}<</SYS>>

{{ .Prompt }} [/INST]"
PARAMETER stop [INST]
PARAMETER stop [/INST]
EOF

# Check if ollama is installed
if ! command -v ollama &> /dev/null; then
    echo "Error: ollama is not installed. Please install it first." >&2
    exit 1
fi

# Install the model to Ollama
ollama create Ahma-3B -f ./Modelfile || { echo "Failed to create model in Ollama"; exit 1; }

echo "Setup completed successfully!"

# Ask if the user wants to test the model
read -p "Do you want to test the model with Ollama? (y/n): " TEST_MODEL
if [[ "$TEST_MODEL" == "y" ]]; then
    echo "Testing Ahma-3B model..."
    ollama run Ahma-3B <<< "Mikä on Suomen pääkaupunki?"
fi