Philipp Schmid philschmid

Hugging Face Transformers FSDP & bitsandbytes AdamWInt8 optimizer

process dataset

python process_dataset.py

run training

	import gradio as gr

	theme = gr.themes.Monochrome(
	primary_hue="indigo",
	secondary_hue="blue",
	neutral_hue="slate",
	radius_size="radius_sm",
	font=[gr.themes.GoogleFont('Open Sans'), 'ui-sans-serif', 'system-ui', 'sans-serif'],
	).set(
	shadow_drop='*button_shadow'

	from transformers import AutoModelForSeq2SeqLM
	from peft import PeftModel

	# load PeftModel just with base model
	base_model_id = "google/flan-t5-xxl"
	model = AutoModelForSeq2SeqLM.from_pretrained(base_model_id)

	# loads wrapper without adapters
	model = PeftModel.from_pretrained(model)

	{
	"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
	"contentVersion": "1.0.0.0",
	"metadata": {
	"_generator": {
	"name": "bicep",
	"version": "0.12.40.16777",
	"templateHash": "4423847801202994493"
	}
	},

	import tensorflow as tf
	from tensorflow.keras.optimizers import Adam
	from transformers import TFAutoModelForSequenceClassification,AutoTokenizer
	from datasets import load_dataset

	# load model and tokenizer
	model_id = "distilbert-base-uncased"
	model = TFAutoModelForSequenceClassification.from_pretrained(model_id, num_labels=5)
	tokenizer = AutoTokenizer.from_pretrained(model_id)

	# Setup Ubuntu
	sudo apt update --yes
	sudo apt upgrade --yes

	# Get Miniconda and make it the main Python interpreter
	wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
	bash ~/miniconda.sh -b -p ~/miniconda
	rm ~/miniconda.sh
	export PATH=~/miniconda/bin:$PATH

	# usage:
	# deepspeed --num_gpus 8 bloom-ds-inference.py --name bigscience/bloom
	#
	# to run benchmarks:
	# deepspeed --num_gpus 8 bloom-ds-inference.py --name bigscience/bloom --benchmark
	#


	# This is going to improve, but at the moment, the process is a bit cumbersome - we first use
	# 1. use Deepspeed-ZeRO to instantiate the model on GPUs, w/o loading the checkpoints,