Skip to content

Instantly share code, notes, and snippets.

View CoffeeVampir3's full-sized avatar

Z CoffeeVampir3

View GitHub Profile
from transformers import AutoTokenizer
import json
import sys
model = "/home/blackroot/Desktop/llama3-8b/llama-3-8b"
max_tokens = 8192
def count_tokens_hf(text: str, model_name: str) -> int:
tokenizer = AutoTokenizer.from_pretrained(model_name)
encoded_input = tokenizer.encode(text)
@CoffeeVampir3
CoffeeVampir3 / bitnetting.ipynb
Last active April 27, 2024 21:11
Bitnet 1.58 MLP Example
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
def load_cfg_from_json(json_file):
with open(json_file, "r", encoding="utf-8") as reader:
text = reader.read()
return json.loads(text)
def load_cfg(model_id, cfg_path):
hf_config = load_cfg_from_json(cfg_path)
if 'pretrained_cfg' not in hf_config:
# old form, pull pretrain_cfg out of the base dict
pretrained_cfg = hf_config
@CoffeeVampir3
CoffeeVampir3 / dora_train.sh
Last active April 3, 2024 15:26
Dora test train
#!/bin/bash
# Config Start
# Configurations
ckpt="/home/blackroot/Desktop/SD/sd-scripts/models/model.safetensors" # base checkpoint to finetune
image_dir="/home/blackroot/Desktop/SD/sd-scripts/train_data" # folder containing folders with repeats_conceptname
reg_dir="" #optional, just point this to an empty folder if you don't care
output="/home/blackroot/Desktop/SD/sd-scripts/outputs" # safetensors output folder
@CoffeeVampir3
CoffeeVampir3 / example.py
Created March 20, 2024 08:21
exl-with-stopbuffering
def per_polygon_translate(generate_fn, input):
prompt = (
f"""::JAPANESE TEXT::
むし、 ある ところ に
[[0.0, 8.0], [252.0, 8.0], [252.0, 27.0], [0.0, 26.0]]
おじいさん と おばあさん が いました。
[[0.0, 33.0], [289.0, 32.0], [289.0, 50.0], [0.0, 50.0]]
おじいさん が 山(やま) へ 木(き) を きり に いけば、
[[0.0, 57.0], [416.0, 56.0], [417.0, 74.0], [0.0, 75.0]]
おばあさん は 川(かわ)へ せんたく に でかけます。
@CoffeeVampir3
CoffeeVampir3 / hi-res-upscale-example.py
Created March 15, 2024 02:40
hi-res-upscale diffusers example
from diffusers import StableDiffusionXLPipeline, StableDiffusionXLImg2ImgPipeline
from diffusers import EulerAncestralDiscreteScheduler
import torch
import torch.nn.functional
import gc
from PIL import Image
from mechanisms.tokenizer_utils import encode_from_pipe
@torch.no_grad()
import re, os, random, csv
class WildCLIP:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"text": ("STRING", {"multiline": True}), "clip": ("CLIP", )
}
}
// Fill out your copyright notice in the Description page of Project Settings.
#include "Ai/Behaviours/NecroAiBasicCombatant.h"
#include "NavigationSystem.h"
#include "NecroGameplayTags.h"
#include "AbilitySystem/NecroAbilitySystem.h"
#include "ActorInterfaces/CombatInterface.h"
#include "GameFramework/Character.h"
@CoffeeVampir3
CoffeeVampir3 / vae_preview_test.ipynb
Created February 19, 2024 21:30
vae-preview for SDXL
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
@CoffeeVampir3
CoffeeVampir3 / Aesthetic_interval_filter.py
Created February 14, 2024 16:04
Anti jpeg data collator
import json
def filter_images_by_score_interval(jsonl_file_path, target_interval, output_file_path):
min_score = None
matching_entries = []
# Find the minimum score in the file
with open(jsonl_file_path, 'r') as file:
for line in file:
json_object = json.loads(line)