hfget.py

hfget.py

import os
import argparse
import hashlib
import hf_transfer # py -m pip install hf-transfer

CHUNK_SIZE = 10_485_760

parser = argparse.ArgumentParser(description='HuggingFace fast model downloader')
parser.add_argument('--url', type=str, help='file url')
args = parser.parse_args()
fn = args.url

prefix = 'https://huggingface.co/'
assert args.url.startswith(prefix)

parts = args.url[len(prefix):].split('/')
assert len(parts) >= 3

base_dir = '/'.join([parts[0], parts[1]])
os.makedirs(base_dir, exist_ok=True)

fn = '/'.join([base_dir, parts[-1]])

hf_transfer.download(
    # url='https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF/resolve/main/mistral-7b-instruct-v0.1.Q5_K_M.gguf',
    # filename='mistral-7b-instruct-v0.1.Q5_K_M.gguf',
    url=args.url,
    filename=fn,
    max_files=1000,
    chunk_size=CHUNK_SIZE,
    parallel_failures=100,
    max_retries=100
)

print(f'file has been downloaded successfully: {fn}')

with open(fn, 'rb', buffering=0) as f:
    print(f"SHA256: {hashlib.file_digest(f, 'sha256').hexdigest()}")