luistung

#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <unordered_map>
#include <boost/algorithm/string.hpp>
#include <utf8proc.h>

//https://unicode.org/reports/tr15/#Norm_Forms
//https://ssl.icu-project.org/apiref/icu4c/uchar_8h.html

madaan

from safetensors.torch import load_file
from glob import glob
import torch
from tqdm import tqdm


def main(base_path: str):
    """
    Convert safetensors files to pytorch checkpoints files.