HuggingFace starter

huggingface_starter.py

# Set `HF_TOKEN` environment variable
# !! Not in your code !!
#
# in jupyter nb:
# %env HF_TOKEN=hfdjakslfhjasdlkfhajslfhdasjl
#
# in bash:
# HF_TOKEN=hfdjakslfhjasdlkfhajslfhdasjl
#
# using colab secrets tab
# from google.colab import userdata
# token=userdata.get('HF_TOKEN')

# Imports
import os
import huggingface_hub
from huggingface_hub import login, hf_hub_download, HfApi

# Login
login(token=os.environ["HF_TOKEN"])

# Manage Repositories
from huggingface_hub import create_repo
create_repo("Gholamreza/test-private", repo_type="dataset", private=True)
delete_repo(repo_id="Gholamreza/test-private", repo_type="dataset")

# Download a file
hf_hub_download(
    repo_id="google/pegasus-xsum",
    filename="pytorch_model.bin",
    repo_type='model') # model/dataset/space

# Upload a file to repo (dont forget `repo_type`)
api.upload_file(
    path_or_fileobj="/content/sample_data/mnist_test.csv",
    path_in_repo="mnist.csv",
    repo_id="Gholamreza/testds",
    repo_type="dataset", # model/dataset/space
    commit_message="uploaded mnist.csv",
    commit_description="new file uploaded")

import pickle
def save_pickle(obj, name):
    with open(name, 'wb') as f:
        pickle.dump(obj, f)
        
        
##################### Colab Notebook Cell ###################


# Huggingface login and helper functions
HF_REPO_ID = "Gholamreza/conditional_gan_mnist"

from google.colab import userdata
import huggingface_hub
from huggingface_hub import login, hf_hub_download, HfApi

def save_to_hub(file_name, file_in_repo_name=None, repo_id=HF_REPO_ID):
    if file_in_repo_name == None:
        file_in_repo_name = file_name

    api = HfApi()
    api.upload_file(
        path_or_fileobj=file_name,
        path_in_repo=file_in_repo_name,
        repo_id=repo_id,
        repo_type="model",
    )
    print(f">> Uploaded {file_name} to {repo_id}/{file_in_repo_name}")
    

def save_folder_to_hub(folder_name, repo_id=HF_REPO_ID):
    api = HfApi()
    api.upload_folder(
        folder_path=folder_name,
        repo_id=repo_id,
        repo_type="model",
    )
    print(f">> Uploaded {folder_name} to {repo_id}/{folder_name}")

def load_from_hub(file_in_repo_name, repo_id=HF_REPO_ID):
    return hf_hub_download(repo_id=repo_id, filename=file_in_repo_name)

login(token=userdata.get('HF_TOKEN'))