SinclairCoder

import argparse
import os
import random
from datasets import Dataset
from datatrove.pipeline.readers import JsonlReader, ParquetReader
from tqdm import tqdm
from transformers import AutoTokenizer
from vllm import LLM, SamplingParams
from data_gen.configs import GentaskConfig
from utils.data_utils import get_adapter_func, split_into_batches

SinclairCoder

import pandas as pd
import numpy as np
import math
import matplotlib.pyplot as plt
plt.rcParams['figure.dpi'] = 1000 #分辨率

class ComplexRadar():
    """
    Create a complex radar chart with different scales for each variable
    Parameters

SinclairCoder

🤗Huggingface Model Downloader

Update: The previous version has a bug. When resuming from a breakpoint, there may be an issue causing incomplete files. Please update to the latest version!!!

Considering the lack of multi-threaded download support in the official huggingface-cli, and the inadequate error handling in hf_transfer， this command-line tool smartly utilizes wget or aria2 for LFS files and git clone for the rest.

Features

• ⏯️ Resume from breakpoint: You can re-run it or Ctrl+C anytime.
• 🚀 Multi-threaded Download: Utilize multiple threads to speed up the download process.
• 🚫 File Exclusion: Use --exclude to skip specific files, save time for models with duplicate formats (e.g., .bin and .safetensors).
• 🔐 Auth Support: For gated models that require Huggingface login, use --hf_username and --hf_token to authenticate.

SinclairCoder

from pytorch_lightning import seed_everything
from transformers import AdamW, T5ForConditionalGeneration, T5Tokenizer, AutoConfig, BartTokenizer, BartForConditionalGeneration
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
import pytorch_lightning as pl
import os



class MyDataset(Dataset):

SinclairCoder

from pytorch_lightning import seed_everything
from transformers import AdamW, T5ForConditionalGeneration, T5Tokenizer, AutoConfig, BartTokenizer, BartForConditionalGeneration
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
import pytorch_lightning as pl
import os



class MyDataset(Dataset):

SinclairCoder

from pytorch_lightning import seed_everything
from transformers import AdamW, T5ForConditionalGeneration, T5Tokenizer, AutoConfig, BartTokenizer, BartForConditionalGeneration
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
import pytorch_lightning as pl
import os



class MyDataset(Dataset):