vllm request

main.py

import openai
from pydantic import BaseModel, Field
import json

class ResponseStructure(BaseModel):
    bookname: str = Field(..., title="Book Name")
    author: str = Field(..., title="Author")
    reading_list_name: str = Field(..., title="Reading List Name")
    
    
client = openai.Client(
    api_key="ollama",
    base_url="http://gandalf:8000/v1"
)


messages = [
    {"role": "system", "content": "You are a helpful assistant designed to output JSON."},
    {"role": "user", "content": "Add the book '1984' by 'George Orwell' to the 'Currently Reading' list."},
]

chat_response = client.chat.completions.create(
    model="solidrust/Hermes-2-Pro-Mistral-7B-AWQ",
    messages=messages,# type: ignore
    response_format={"type": "json_object", "schema": ResponseStructure.model_json_schema()}, # type: ignore
    temperature=0.0,
)

# json_obj = json.loads(chat_response.choices[0].message.content) # type: ignore
# print(json_obj)

print(chat_response.choices[0].message.content)