Using the Guidance AI to access an LLM on a vLLM server

vllm_guidance.py

import guidance
import requests

base_url = "http://localhost:9999/v1"
x = requests.get(base_url + "/models")
model = str(x.json()["data"][0]["id"])

guidance.llm = guidance.llms.OpenAI(
	model=model, 
	endpoint=base_url,
	api_key="***",
	chat_mode=False,
	encoding_name="cl100k_base"
)

program = guidance(
	"""What are the top five fun facts about {{country}}? Provide a one-liner with a description of each of the facts.

    Here are the fun facts: 
    {{~#geneach 'facts' num_iterations=5}}
    [{{@index}}]: "{{gen 'this' stop='"' max_tokens=50 ~}}"
    {{/geneach}}
	"""
)

result = program(country="France")
print(result["facts"])
print("===========================================")
print(result)