llama权重下载脚本

llama_download.sh

PRESIGNED_URL="https://agi.gpt4.org/llama/LLaMA/*"

TARGET_FOLDER="./"             # where all files should end up

declare -A N_SHARD_DICT

N_SHARD_DICT["7B"]="0"
N_SHARD_DICT["13B"]="1"
N_SHARD_DICT["30B"]="3"
N_SHARD_DICT["65B"]="7"

if [ $# -eq 0 ]
  then
    echo "Please specify the model size you wish to download. Choose from 7B, 13B, 30B, or 65B."
    exit
fi

MODEL_SIZE="$1"

if [ -z "${N_SHARD_DICT[$MODEL_SIZE]}" ]
  then
    echo "Invalid model size. Choose from 7B, 13B, 30B, or 65B."
    exit
fi

echo "Downloading tokenizer"
wget ${PRESIGNED_URL/'*'/"tokenizer.model"} -O ${TARGET_FOLDER}"/tokenizer.model"
wget ${PRESIGNED_URL/'*'/"tokenizer_checklist.chk"} -O ${TARGET_FOLDER}"/tokenizer_checklist.chk"

(cd ${TARGET_FOLDER} && md5sum -c tokenizer_checklist.chk)

echo "Downloading ${MODEL_SIZE}"
mkdir -p ${TARGET_FOLDER}"/${MODEL_SIZE}"
for s in $(seq -f "0%g" 0 ${N_SHARD_DICT[$MODEL_SIZE]})
do
    wget ${PRESIGNED_URL/'*'/"${MODEL_SIZE}/consolidated.${s}.pth"} -O ${TARGET_FOLDER}"/${MODEL_SIZE}/consolidated.${s}.pth"
done
wget ${PRESIGNED_URL/'*'/"${MODEL_SIZE}/params.json"} -O ${TARGET_FOLDER}"/${MODEL_SIZE}/params.json"
wget ${PRESIGNED_URL/'*'/"${MODEL_SIZE}/checklist.chk"} -O ${TARGET_FOLDER}"/${MODEL_SIZE}/checklist.chk"
echo "Checking checksums"
(cd ${TARGET_FOLDER}"/${MODEL_SIZE}" && md5sum -c checklist.chk)