Quick Chatbot Deployment

gistfile1.txt

python server.py

Quick_Chatbot_Deployment.txt

pip install huggingface-hub gradio llama-cpp-python \\n--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
huggingface-cli login
mkdir models
cd models
huggingface-cli download Qwen/Qwen2-0.5B-Instruct-GGUF \\nqwen2-0_5b-instruct-q5_k_m.gguf \\n--local-dir . --local-dir-use-symlinks False
cd ..
touch server.py

server.py

import gradio as gr
from llama_cpp import Llama

llm = Llama(
    model_path="./models/qwen2-0_5b-instruct-q5_k_m.gguf",
    verbose=True,
)

def predict(message, history):
    messages = [{"role": "system", "content": "You are a helpful assistant."}]
    for user_message, bot_message in history:
        if user_message:
            messages.append({"role": "user", "content": user_message})
        if bot_message:
            messages.append({"role": "assistant", "content": bot_message})
    messages.append({"role": "user", "content": message})

    response = ""
    for chunk in llm.create_chat_completion(
        stream=True,
        messages=messages,
    ):
        part = chunk["choices"][0]["delta"].get("content", None)
        if part:
            response += part
        yield response

demo = gr.ChatInterface(predict)

if __name__ == "__main__":
    demo.launch(share=True)