Skip to content

Instantly share code, notes, and snippets.

@ochafik
Last active April 22, 2024 10:28
Show Gist options
  • Select an option

  • Save ochafik/e5b1731e4e574ac1aed2e49e5f9c5899 to your computer and use it in GitHub Desktop.

Select an option

Save ochafik/e5b1731e4e574ac1aed2e49e5f9c5899 to your computer and use it in GitHub Desktop.

Context:

hyperfine --warmup 1 --runs 5 \
  -L branch grammar-reps,master \
  --setup 'git checkout {branch} && \
             make clean && \
             make -j LLAMA_CURL=1 main && \
             rm -f issue4218.bin && \
             ./issue4218.sh precache' \
  './issue4218.sh run {branch}'
#!/bin/bash
set -euo pipefail

commits=(
  f608415de06fc914568d7ffc84e293d0a94bd16a # grammars: cache decoded tokens
  98f33bae767dd19e213ef663b22ad99979ca71d7 # grammars: early exit when no next_candidates to reject
  # ab9a3240a9da941fdef5cd4a25f2b97c2f5a67aa # JSON schema conversion: ⚡️ faster repetitions, min/maxLength for strings, cap number length (#6555)
  04a5ac211ef40936295980b7cdf0ba6e97093146 # Optimization: eliminate addition of redundant stacks when advancing grammar. (#6616)
  cbaadc92942c50aab599a9e4c163afc1f44f7c26 # grammars: 1.5x faster inference w/ complex grammars (vector reserves / reuses) (#6609)
)

# 160 sec -> cbaadc92942c50aab599a9e4c163afc1f44f7c26 -> 96 sec
# 96 sec  -> 04a5ac211ef40936295980b7cdf0ba6e97093146 -> 6.6 sec
# 6.6 sec -> f608415de06fc914568d7ffc84e293d0a94bd16a -> 3.5 sec

versions=()
for commit in "${commits[@]}"; do
  versions+=( "$commit^" "$commit" )
done

echo '{"items": {"type": "number"}, "minItems": 10, "maxItems": 100}' | \
  python examples/json_schema_to_grammar.py - > grammar.gbnf

hyperfine --warmup 1 --runs 5 \
    --export-json bench-commits.json \
    --export-markdown bench-commits.md \
    -L branch "$( IFS=$','; echo "${versions[*]}" )" \
    --setup 'git checkout {branch} && make clean && make -j LLAMA_CURL=1 main' \
    'BRANCH={branch} \
        ./main --seed 12345 \
            -mu https://huggingface.co/TheBloke/phi-2-GGUF/resolve/main/phi-2.Q4_K_M.gguf \
            -m models/phi-2.Q4_K_M.gguf \
            --grammar-file grammar.gbnf \
            -p "JSON list of first 50 integers"'
root ::= [ \t\n]* exp
ws ::= [ \t\n]+
w ::= [ \t]*
comment ::= "#" [^#]* "#" [ \t]+ [\n]? [ \t]*
### Expressions
exp ::= comment* sequence-exp
sequence-exp ::= tuple-exp (w ";" ws tuple-exp)*
tuple-exp ::= cons-exp (w "," ws cons-exp)*
cons-exp ::= binary-exp (w "::" w binary-exp)*
binary-exp ::= unary-exp (ws binary-op ws unary-exp)*
unary-exp ::= unary-op* function-app-exp
function-app-exp ::= primary-exp (w "(" w exp w ")" w)*
primary-exp ::= bool |
integer |
float |
string |
variable |
"()" |
"[]" |
constructor |
constructor-app |
parenthesized-exp |
list-exp |
let-exp |
if-exp |
case-exp |
test-exp |
type-alias |
fun
constructor-app ::= constructor "(" w exp w ")"
parenthesized-exp ::= "(" w exp w ")"
list-exp ::= "[" exp ("," ws exp)* "]"
let-exp ::= "let" ws pat ws "=" ws exp ws "in" ws exp
if-exp ::= "if" ws exp ws "then" ws exp ws "else" ws exp
case-exp ::= "case" ws exp (ws "|" ws pat ws "=>" ws exp)+ ws "end"
test-exp ::= "test" ws exp ws "end"
type-alias ::= "type" ws constructor ws "=" ws typ ws "in" ws exp
fun ::= "fun" ws pat ws "->" ws exp
type-variable ::= [a-z][A-Za-z0-9_]*
constructor ::= [A-Z][A-Za-z0-9_]*
variable ::= ([_a-bdg-hj-kn-qu-z][A-Za-z0-9_.]*)|(("s" ([.0-9A-Z_a-su-z][A-Za-z0-9_.]*)?)|("st" ([.0-9A-Z_a-qs-z][A-Za-z0-9_.]*)?)|("str" ([.0-9A-Z_a-tv-z][A-Za-z0-9_.]*)?)|("stru" ([.0-9A-Z_a-bd-z][A-Za-z0-9_.]*)?)|("struc" ([.0-9A-Z_a-su-z][A-Za-z0-9_.]*)?)|("struct" [A-Za-z0-9_.]+)|("c" ([.0-9A-Z_b-z][A-Za-z0-9_.]*)?)|("ca" ([.0-9A-Z_a-rt-z][A-Za-z0-9_.]*)?)|("cas" ([.0-9A-Z_a-df-z][A-Za-z0-9_.]*)?)|("case" [A-Za-z0-9_.]+)|("i" ([.0-9A-Z_a-mo-z][A-Za-z0-9_.]*)?)|("in" [A-Za-z0-9_.]+)|("r" ([.0-9A-Z_a-df-z][A-Za-z0-9_.]*)?)|("re" ([.0-9A-Z_a-bd-z][A-Za-z0-9_.]*)?)|("rec" [A-Za-z0-9_.]+)|("t" ([.0-9A-Z_a-df-z][A-Za-z0-9_.]*)?)|("te" ([.0-9A-Z_a-rt-z][A-Za-z0-9_.]*)?)|("tes" ([.0-9A-Z_a-su-z][A-Za-z0-9_.]*)?)|("test" [A-Za-z0-9_.]+)|("l" ([.0-9A-Z_a-df-z][A-Za-z0-9_.]*)?)|("le" ([.0-9A-Z_a-su-z][A-Za-z0-9_.]*)?)|("let" [A-Za-z0-9_.]+)|("m" ([.0-9A-Z_b-z][A-Za-z0-9_.]*)?)|("ma" ([.0-9A-Z_a-su-z][A-Za-z0-9_.]*)?)|("mat" ([.0-9A-Z_a-bd-z][A-Za-z0-9_.]*)?)|("matc" ([.0-9A-Z_a-gi-z][A-Za-z0-9_.]*)?)|("match" [A-Za-z0-9_.]+)|("f" ([.0-9A-Z_a-tv-z][A-Za-z0-9_.]*)?)|("fu" ([.0-9A-Z_a-mo-z][A-Za-z0-9_.]*)?)|("fun" [A-Za-z0-9_.]+)|("e" ([.0-9A-Z_a-mo-z][A-Za-z0-9_.]*)?)|("en" ([.0-9A-Z_a-ce-z][A-Za-z0-9_.]*)?)|("end" [A-Za-z0-9_.]+))
bool ::= "true" | "false"
integer ::= [0-9]+
float ::= [0-9]* "." [0-9]+
string ::= "\"" [^"]* "\""
unary-op ::= "-" | "!"
binary-op-int ::= "+" | "-" | "*" | "/" | "<" | ">" | "<=" | ">=" | "==" | "!="
binary-op-float ::= "+." | "-." | "*." | "/." | "<." | ">." | "<=." | ">=." | "==." | "!=."
binary-op-string ::= "$==" | "@"
binary-op-logic ::= "&&"
binary-op ::= binary-op-int | binary-op-float | binary-op-string | binary-op-logic
### Patterns
pat ::= type-ascription-pat
type-ascription-pat ::= tuple-pat (w ":" ws typ)*
tuple-pat ::= cons-pat (w "," ws cons-pat)*
cons-pat ::= primary-pat (w "::" w primary-pat)*
primary-pat ::=
bool |
integer |
float |
string |
variable |
"()" |
"[]" |
"_" |
constructor |
constructor-app-pat |
parenthesized-pat |
list-pat
constructor-app-pat ::= constructor "(" w pat w ")"
parenthesized-pat ::= "(" w pat w ")"
list-pat ::= "[" pat (w "," ws pat)* "]"
### Types
typ ::= arrow-typ
arrow-typ ::= tuple-typ (ws "->" ws tuple-typ)*
tuple-typ ::= primary-typ (w "," ws primary-typ)*
primary-typ ::=
"Unit" |
"Int" |
"Float" |
"Bool" |
"String" |
type-variable |
constructor |
constructor-def (ws "+" ws constructor-def)+ |
parenthesized-typ |
list-typ
parenthesized-typ ::= "(" w typ w ")"
list-typ ::= "[" w typ w "]"
constructor-def ::= constructor | constructor "(" w typ w ")"
#!/bin/bash
#
set -euo pipefail
args=(
# -m ../models/codellama-34b.Q5_K_M.gguf \
# -m /Users/ochafik/AI/Models/Hermes-2-Pro-Mistral-7B.Q4_K_M.gguf
-mu https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF/resolve/main/Hermes-2-Pro-Mistral-7B.Q8_0.gguf
-t 10
-ngl 64
-b 512
--color -c 3400
--temp 0.7
--repeat_penalty 1.1
--seed 12345
--grammar-file issue4218.gbnf
-f issue4218.txt
--prompt-cache issue4218.bin
)
if [[ "$1" == "precache" ]]; then
./main "${args[@]}" -n 1 --log-disable
elif [[ "$1" == "run" ]]; then
./main "${args[@]}" \
--prompt-cache-ro \
--no-display-prompt \
-n 128
else
echo "Expected command precache or run, got ${1:-?}" >&2
exit 1
fi
### Option ###
# Represent values that may or may not exist. #
type Option =
+ Some(?)
+ None
in
# Compare if two Options are equal #
# equal: ((?, ?) -> Bool) -> (Option, Option) -> Bool #
let equal: ((?, ?) -> Bool) -> (Option, Option) -> Bool =
fun eq, os ->
case os
| Some(x), Some(y) => eq(x, y)
| None, None => True
| _, _ => False
end
in
### Result ###
# A Result is either Ok meaning the computation succeeded, #
# or it is an Err meaning that there was some failure. #
type Result =
+ Ok(a)
+ Err(b)
in
# Compare if two Results are equal #
# equal: ((a, a) -> Bool) -> (Result, Result) -> Bool #
let equal: ((a, a) -> Bool) -> (Result, Result) -> Bool =
fun eq, rs ->
case rs
| Ok(e1), Ok(e2) => eq(e1, e2)
| Error(e1), Error(e2) => e1 $== e2
| _ => false
end
in
### JSON ###
# This module helps you convert between Hazel values and JSON values. #
# A JSON value type #
type Value =
+ Object([(String, Value)])
+ Array([Value])
+ Str(String)
+ Number(Float)
+ Boolean(Bool)
+ Null
in
# Check if two JSON values are equal #
# equal : (Value,Value) -> Bool #
let equal : (Value,Value) -> Bool =
fun a, b ->
case (a, b)
| Object(o1), Object(o2) => List.equal(
fun (s1, v1), (s2, v2) ->
s1 $== s2 && equal(v1, v2), o1, o2)
| Array(a1), Array(a2) => List.equal(equal, a1, a2)
| Str(s1), Str(s2) => s1 $== s2
| Number(n1), Number(n2) => n1 ==. n2
| Boolean(b1), Boolean(b2) => if b1 then b2 else !b2
| Null, Null => true
| _ => false
end
in
# JSON Encoder #
# Convert a string to a JSON string #
# value_of_string : String -> Value #
let value_of_string : String -> Value =
fun s -> Str(s)
in
# Convert an integer to a JSON integer #
# value_of_int : Int -> Value #
let value_of_int : Int -> Value =
fun i -> Number(float_of_int(i))
in
# Convert a float to a JSON float #
# value_of_float : Float -> Value #
let value_of_float : Float -> Value =
fun f -> Number(f)
in
# Convert a boolean to a JSON boolean #
# value_of_bool : Bool -> Value #
let value_of_bool : Bool -> Value =
fun b -> if b then Boolean(true) else Boolean(false)
in
# Convert a null to a JSON null #
# value_of_null : Value #
let value_of_null : Value = Null in
# Convert a list of JSON values to a JSON array #
# value_of_list : (a -> Value, [a]) -> Value #
let value_of_list : (a -> Value, [a]) -> Value =
fun (func, entries) ->
Array(
List.rev(List.fold_left(
fun l, e-> func(e)::l, [], entries)))
in
# Convert a dictionary of JSON values to a JSON object #
# value_of_object : [(String, Value)] -> Value #
let value_of_object : [(String, Value)] -> Value =
fun entries -> Object(entries)
in
# JSON Decoder #
# A Decoder decodes a JSON value into a Hazel value, or return an Err on failure. #
type Decoder = Value -> Result in
# Decodes a JSON string into a string #
# string_of_value : Decoder #
let string_of_value : Decoder =
fun v ->
case v
| Str(s) => Ok(s)
| _ => Err("Cannot unpack value as a String")
end
in
# Decodes a JSON boolean into a boolean #
# bool_of_value : Decoder #
let bool_of_value : Decoder =
fun v ->
case v
| Boolean(b) => Ok(b)
| _ => Err("Cannot unpack value as a Bool")
end
in
# Decodes a JSON integer into an integer #
# int_of_value : Decoder #
let int_of_value : Decoder =
fun v ->
case v
| Number(n) =>
if floor(n) ==. n then
# n is a whole number #
Ok(floor(n))
else
# n is a floating point #
Err("Cannot unpack a float value as an Int")
| _ => Err("Cannot unpack value as an Int")
end
in
# Decodes a JSON float into a float #
# float_of_value : Decoder #
let float_of_value : Decoder =
fun v ->
case v
| Number(n) => Ok(floor(n))
| _ => Err("Cannot unpack value as a Float")
end
in
# Decodes a JSON null into a null #
# null_of_value : Decoder #
let null_of_value : Decoder =
fun v ->
case v
| Null => Ok(None)
| _ => Err("Cannot unpack value as a None")
end
in
# Parsers #
# Try a bunch of different decoders. #
# This can be useful if the JSON may come in a couple different formats. #
# one_of : [Decoder] -> Decoder #
let one_of : [Decoder] -> Decoder =
fun decoders -> fun v ->
case decoders
| decoder::decoders =>
result_map_err(fun _ -> one_of(decoders)(v), decoder(v))
| [] => Err("one_of failed to decode value")
end
in
# Transform a decoder. #
# map : ((a -> b), Decoder) -> Decoder #
let map : ((a -> b), Decoder) -> Decoder =
fun (func, decoder) -> fun v ->
case decoder(v)
| Err(e) => Err(e)
| Ok(o) => func(o)
in
# Create decoders that depend on previous results. #
# and_then: ((a -> Decoder), Decoder) -> Decoder #
let and_then: ((a -> Decoder), Decoder) -> Decoder =
fun (func, decoder) ->
fun v ->
case decoder(v)
| Err(e) => Err(e)
| Ok(o)=> func(o)(v)
end
in
# Decode a nullable JSON value into a Hazel value. #
# nullable : Decoder -> Decoder #
let nullable : Decoder -> Decoder =
fun decoder ->
one_of([
map(fun s -> Some(s), decoder),
null_of_value
])
in
# Decode a JSON array into a Hazel List. #
# list : Decoder -> Decoder #
let list : Decoder -> Decoder =
fun elem_decoder ->
fun v ->
case v
| Array(arr) =>
case arr
| head::tail =>
case elem_decoder(head)
| Ok(hd) => map(fun tl -> hd::tl, list(elem_decoder))(Array(tail))
| Err(e) => Err(e)
end
| [] => Ok([])
end
| _ => Err("Cannot unpack value as a List")
end
in
# Decode a JSON object into a Hazel dictionary. #
# For now, a dictionary is just a list of key-value pairs #
# dict : Decoder -> Decoder #
let dict : Decoder -> Decoder =
fun value_decoder ->
fun v ->
case v
| Object(pairs) =>
case pairs
| (key, value)::tail =>
case value_decoder(value)
| Ok(hd)=> map(fun tl -> (key, hd)::tl, dict(value_decoder))(Object(tail))
| Err(e) => Err(e)
end
| [] => Ok([])
end
| _ => Err("Cannot unpack value as a dict")
end
in
### List ###
# Add an element to the front of a list. #
# cons: (a, [a]) -> [a] #
let cons: (a, [a]) -> [a] = fun x, xs -> x::xs in
# Determine the length of a list. #
# length: [a] -> Int #
let length: [a] -> Int =
fun xs ->
case xs
| [] => 0
| _::xs => 1 + length(xs) end in
# Extract the first element of a list. #
# hd: [a] -> Option #
let hd: [a] -> Option =
fun l ->
case l
| [] => None
| x::xs => Some(x) end in
# Extract the rest of the list. #
# tl: [a] -> [a] #
let tl: [a] -> [a] =
fun l ->
case l
| [] => []
| x::xs => xs end in
# Determine if a list is empty. #
# is_empty: [a] -> Bool #
let is_empty: [a] -> Bool =
fun xs ->
case xs
| [] => true
| _::_ => false end in
# Return the element at the index. #
# nth: ([a], Int) -> Option #
let nth: ([a], Int) -> Option =
fun xs, n ->
case xs, n
| x::_, 0 => Some(x)
| _::xs, n => nth(xs, n - 1)
| [], _ => None end in
# Reverse a List. #
# rev: [a] -> [a] #
let rev: [a] -> [a] =
fun l ->
let go: ([a], [a]) -> [a] =
fun xs, acc ->
case xs
| [] => acc
| x::xs => go(xs, x::acc) end in
go(l, []) in
# Check if two lists are equal #
# equal: ((a, a) -> Bool, [a], [a]) -> Bool #
let equal: ((a, a) -> Bool, [a], [a]) -> Bool =
fun p, xs, ys ->
case xs, ys
| [], [] => true
| x::xs, y::ys => p(x, y) && equal(p, xs, ys)
| _ => false end
in
# Initialize a list with a given length using an initializer function #
# init: (Int, Int -> a) -> [a] #
let init: (Int, Int -> a) -> [a] =
fun len, f ->
let go: (Int, [a]) -> [a] =
fun idx, xs ->
if idx < len
then go(idx + 1, xs @ [f(idx)])
else xs
in
go(0, [])
in
# Reduce a list from the left. #
# fold_left: ((b, a) -> b, b, [a]) -> b #
let fold_left: ((b, a) -> b, b, [a]) -> b =
fun f, acc, xs ->
case xs
| [] => acc
| hd::tl => fold_left(f, f(acc, hd), tl) end in
# Reduce a list from the right. #
# fold_right: ((a, b) -> b, [a], b) -> b #
let fold_right: ((a, b) -> b, [a], b) -> b =
fun f, xs, acc ->
case xs
| [] => acc
| hd::tl => f(hd, fold_right(f, tl, acc)) end in
# A simplified lambda calculus expression containing variables, lambdas, and applications #
type Exp =
+ Var(String)
+ Lam(String, Exp)
+ Ap(Exp, Exp)
in
# Evaluation can result in either an Exp or an Error #
# Evaluation by substitution #
# eval: Exp -> Result #
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment