Brandon Lockaby brandon-lockaby

thistleknot / dataset_distillation.py

Last active December 6, 2023 05:19

Dataset Distillation v3

	import torch
	import torch.nn.functional as F
	from transformers import GPTNeoForCausalLM, AutoTokenizer
	from datasets import load_dataset
	from sklearn.model_selection import train_test_split
	import pandas as pd
	import numpy as np
	import random

	# Parameters

younesbelkada / bnb-serialization.py

Created December 25, 2023 18:10

push bnb 4 bit models on the hub

	# pip install -U bitsandbytes
	# pip install -U git+https://github.com/huggingface/transformers.git
	from transformers import AutoModelForCausalLM, AutoTokenizer

	model_id = "mistralai/Mixtral-8x7B-Instruct-v0.1"
	path_to_hub = XXX
	tokenizer = AutoTokenizer.from_pretrained(model_id)

	model = AutoModelForCausalLM.from_pretrained(model_id, load_in_4bit=True)
	model.push_to_hub(path_to_hub)

silphendio / exl_slice_test.py

Last active February 15, 2024 23:55

Create LLM slices at runtime with exllamav2

	# to use this, first install python and exllamav2 (https://github.com/turboderp/exllamav2)
	# load a model, rearrange the layers as you like, set generation parameters, and run it
	# duplicate layers share tensors, but still need extra memory for the cache
	# thanks to @dnhkng for showing that the cache needs to be re-created
	# licensed under WTFPL (http://www.wtfpl.net/about/) - Silphendio


	from exllamav2 import *
	from exllamav2.generator import *
	import sys, torch