ddh0

# serve.py
# Python 3.12.11

import os
import re
import sys
import math
import datetime

from typing import Optional, Literal

ddh0

diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index d6807b6dd..a050211b1 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -568,6 +568,7 @@ extern "C" {
         GGML_OP_RWKV_WKV7,
         GGML_OP_SOLVE_TRI,
         GGML_OP_GATED_DELTA_NET,
+        GGML_OP_LIGHTNING_INDEXER,
 

ddh0

warn: LLAMA_ARG_MMAP environment variable is set, but will be overwritten by command line argument --no-mmap
warn: LLAMA_ARG_BATCH environment variable is set, but will be overwritten by command line argument -b
warn: LLAMA_ARG_UBATCH environment variable is set, but will be overwritten by command line argument -ub
0.00.022.162 W main: parameter override: warmup = false
0.00.022.549 I system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 890 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
0.00.022.559 I common_init_result: fitting params to device memory ...
0.00.022.559 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)
0.00.022.804 I common_params_fit_impl: getting device memory data for initial parameters:

ddh0

A Brief Guide to GGML

DISCLAIMER: I intend for this guide to be helpful and correct, but I make no guarantees. If you find something is inaccurate or missing, please let me know!

Intro

GGML is a C/C++ tensor library for machine learning. It implements:

• A set of tensor operations
• Automatic differentiation

ddh0

# chat_template_txt.py
# Python 3.12.3

import sys
import argparse
import datetime

from typing import TextIO, Optional, Literal
from transformers import AutoTokenizer

ddh0

#include "arg.h"
#include "common.h"
#include "log.h"
#include "llama.h"
#include "ggml.h"

#include <map>
#include <cstdio>
#include <string>
#include <vector>

ddh0

/// adaptive-p: select tokens near a configurable target probability over time.
///
/// the adaptive-p sampler transforms the token probability distribution to favor tokens
/// that fall near a user-configurable probability target.
///
/// internally, the sampler maintains an exponential moving average of the *ORIGINAL*
/// probabilities of selected tokens at each sampling step. it uses this EMA to compute an
/// adapted target probability at each sampling step, thus maintaining the desired target
/// probability over time.
///

ddh0

# train_tokenizer2.py
# Python 3.12.3

"""Train a `PreTrainedTokenizerFast` given parquet files and a vocabulary size."""

import sys
import argparse
import tempfile
import shutil
import os

ddh0

import os
import sys
import ctypes

import numpy as np

from easy_llama import libllama as lib

# -------------------------------------------------------------------------------------------- #

ddh0

Tensor Type Testing

Tip

Skip to the bottom of this document for a TL;DR

For more info, see llama.cpp #12511: Handle user-defined quantization levels for additional tensors by @EAddario

Testing done by @ddh0 using this branch as of committ 5a304b8. Using libllama built for Linux CUDA.

Quantization naming scheme