david l euler davideuler

Which GGUF is right for me? (Opinionated)

Good question! I am collecting human data on how quantization affects outputs. See here for more information: ggerganov/llama.cpp#5962

In the meantime, use the largest that fully fits in your GPU. If you can comfortably fit Q4_K_S, try using a model with more parameters.

sudo apt update && sudo apt upgrade

	from diffusers import FluxPipeline, AutoencoderKL
	from diffusers.image_processor import VaeImageProcessor
	from transformers import T5EncoderModel, T5TokenizerFast, CLIPTokenizer, CLIPTextModel
	import torch
	import gc


	def flush():
	gc.collect()
	torch.cuda.empty_cache()

	from collections import defaultdict
	import random
	from huggingface_hub import hf_hub_download
	from datasets import Dataset
	import numpy as np
	import pandas as pd
	from transformers import AutoTokenizer
	from rich.console import Console
	from rich.table import Table
	from trl import DPOTrainer

	from transformers import AutoModelForCausalLM, AutoTokenizer, StaticCache
	import torch
	from typing import Optional
	device = "cuda"

	# Copied from the gpt-fast repo
	def multinomial_sample_one_no_sync(probs_sort): # Does multinomial sampling without a cuda synchronization
	q = torch.empty_like(probs_sort).exponential_(1)
	return torch.argmax(probs_sort / q, dim=-1, keepdim=True).to(dtype=torch.int)

	#!/bin/bash

	### steps ####
	# verify the system has a cuda-capable gpu
	# download and install the nvidia cuda toolkit and cudnn
	# setup environmental variables
	# verify the installation
	###

	### to verify your gpu is cuda enable check

	# Credit 🙏: I just used the example from langchain docs and it works quite well: https://python.langchain.com/en/latest/use_cases/question_answering.html
	# Note 2: The Arxiv -> PDF logic is a bit messy, I'm sure it can be done better
	# Note 3: Please install the following:

	# To run:

	# Save this in a `app.py`
	# pip install arxiv PyPDF2 langchain chromadb

	# The chat feature was shipped in H2O nightly this week, we will need to install from nightly link:

	#Spectrogram is a 2D numpy array
	print(type(mel_sgram), mel_sgram.shape)
	# <class 'numpy.ndarray'> (128, 134)

	# use the decibel scale to get the final Mel Spectrogram
	mel_sgram = librosa.amplitude_to_db(mel_scale_sgram, ref=np.min)
	librosa.display.specshow(mel_sgram, sr=sample_rate, x_axis='time', y_axis='mel')
	plt.colorbar(format='%+2.0f dB')