logprobs

hyperbolic_logprobs.py

import json

import httpx
import requests
from openai import OpenAI

func_name = "chat/completions"

model = "NousResearch/Hermes-3-Llama-3.1-70B"


addr = hyper_addr = "https://api.hyperbolic.xyz/v1/"
api_key = hyper_key = (
    "YOUR_API_KEY"
)

client = OpenAI(
    api_key=api_key,
    base_url=addr,
    timeout=httpx.Timeout(10.0, read=15.0),
)

response = client.chat.completions.create(
    model=model,
    messages=[
        {"role": "system", "content": "You are a helpful and polite assistant."},
        {"role": "user", "content": "What is Chinese hotpot?"},
    ],
    temperature=0.4,
    stream=True,  # this time, we set stream=True
    logprobs=True,
    top_logprobs=2,
    n=1,
)

for chunk in response:
    print(chunk.choices[0])