test.py

import openai
from pdb import set_trace
import time
start = time.monotonic()
openai.api_base = "http://34.16.147.202/v1"
openai.api_key = "test"

# prompt = "Apakah ibu negara malaysia?" # warm up prompt
prompt = "Siapakah perdana menteri malaysia sekarang?"
messages = [
    {"role": "user", "content": prompt},
]
stop = ["###","\n"]
response = openai.ChatCompletion.create(
    model="xtau",
    messages=messages,
    temperature=0,
    max_tokens=128,
    stream=True,
    stop=stop,
)
for chunk in response:
    chunk_message = chunk['choices'][0]['delta'].to_dict() 
    if "content" in chunk_message:
        print(chunk_message["content"],end="",flush=True)
    else:
        if len(chunk_message) == 0:
            break
        print(chunk_message)
print()
end = time.monotonic() - start
print("Time taken:",end)