Didier Lacroix cnndabbler

Part 1/2 : Set up an OpenAI compatible CPU-based local LLM model

What is llama-cpp-python
- llama-cpp-python offers a web server which aims to act as a drop-in replacement for the OpenAI API. This allows you to use llama.cpp compatible models with any OpenAI compatible client (language libraries, services, etc).
- https://github.com/abetlen/llama-cpp-python
Installation

	#!/usr/bin/env python3
	import asyncio
	import logging
	import sys
	import os
	from pathlib import Path
	from contextlib import AsyncExitStack
	from typing import Optional, List, Dict, Any

	import gradio as gr

	#!/usr/bin/env python3
	from mcp.server import Server, NotificationOptions
	from mcp.server.models import InitializationOptions
	from mcp.server.fastmcp import FastMCP
	import mcp.server.stdio
	import mcp.types as types
	from youtube_transcript_api import YouTubeTranscriptApi
	from youtube_transcript_api._errors import NoTranscriptFound
	from urllib.parse import urlparse, parse_qs
	import logging

	# ---
	# BENTOAI:
	# input as prompts_file: image_prompts.txt
	# output-directory: "./flux/frame_XXXX.png"
	# ---

	# BENTOAI: This guide is based on Modal excellent examples and tutorials: https://modal.com/docs/example


	# In this guide, we'll run Flux to generate images from prompts using [Hugging Face's Diffusers](https://github.com/huggingface/diffusers).

	"""
	Restaurant recommendation system demonstrating context features with multiple customers.
	Shows how context helps personalize and improve recommendations over time.
	"""

	from typing import List, Optional, Dict, Literal
	from pydantic import BaseModel, Field
	import asyncio
	from datetime import datetime
	from pydantic_ai import Agent, RunContext

	from openai import OpenAI
	from rich import print
	from rich.console import Console
	from rich.panel import Panel
	from rich.prompt import Prompt
	from rich.markdown import Markdown
	from rich.text import Text
	from rich.style import Style
	from swarm import Swarm, Agent
	from swarm.types import Response

	from openai import OpenAI
	from rich import print

	client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")

	# model="qwq"
	model="qwen2.5-coder:32b"

	# TESTING INFERENCE

	"""
	Movie Purchase System with Multi-Agent Orchestration

	This script implements a sophisticated movie purchase system using OpenAI's Swarm framework
	for multi-agent orchestration. The system handles the entire flow from movie selection to
	shipping using specialized AI agents working in concert.

	Key Components:
	- Supervisor Agent: Orchestrates the overall purchase flow
	- Buy Movie Agent: Handles pricing and purchase transactions