rerank test

rerank.py

import httpx

MAX_TOKENS = 2048
truncated_snippets = []

print('[INFO] Reading file...')
snippets = []
with open('./pacmanlog.txt', 'r') as f:
    snippets = f.readlines()

print('[INFO] Chunking content...')
client = httpx.Client(
    base_url="https://url.com",
    headers={
        "Authorization": "Bearer mysecretkey",
    },
    timeout=120
)

try:
    response = client.get("/v1/models")
    response = response.json()
except Exception:
    print('[ERROR] Could not reach server.')
    exit(1)

tok_count = 0
for idx, doc in enumerate(snippets, start=1):
    print(f'[INFO] Tokenizing/detokenizing line {idx}/{len(snippets)}')
    tok_response = client.post(
        '/tokenize',
        json={"content": doc}
    )
    tokens = tok_response.json()['tokens']
    tok_count += len(tokens)
    print(tok_count)
    if tok_count > MAX_TOKENS:
        break
    else:
        truncated_snippets.append(doc)

json = None
print('[INFO] Reranking...')
try:
    response = client.post(
        "/rerank",
        json={
            "model": "qwen3-reranker-0.6b-q8_0.gguf",
            "query": "When did I mess with nvim?",
            "documents": truncated_snippets,
            "top_n": 2,
        },
    )
    json = response.json()
except Exception as e:
    print(f"[ERROR] {e}")
    exit(1)

for result in json["results"]:
    print(truncated_snippets[result['index']])

embeddings: llama-server -m embeddinggemma-300M-Q8_0.gguf --embeddings --pooling mean -c 2048 -b 2048 -ub 2048 -np 1

reranking: llama-server -m ./qwen3-reranker-0.6b-q8_0.gguf --reranking -c 2048 -b 2048 -ub 2048 --pooling rank