local request to llvm served instance

gistfile1.txt

require 'net/http'
require 'json'

uri = URI('http://localhost:8000/v1/chat/completions')
request = Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json')
request.body = {
  model: 'meta-llama/Llama-3.2-1B',
  messages: [{ role: 'user', content: ARGV[0] }],
  chat_template: "{% for message in messages %}{{'\u003chuman\u003e' if message.role == 'user' else '\u003cassistant\u003e'}}{{ message.content }}{% if not loop.last %}{{ '\\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}\\n\u003cassistant\u003e{% endif %}",
  max_tokens: 100,
  stop: ['</assistant>', '<assistant>', "\n"]
}.to_json

response = Net::HTTP.start(uri.hostname, uri.port) do |http|
  http.request(request)
end

puts JSON.parse(response.body)['choices'][0]['message']['content']