Crawl4AI example

main.py

from crawl4ai import WebCrawler
from crawl4ai.extraction_strategy import LLMExtractionStrategy
from pydantic import BaseModel, Field

class OpenAIModelFee(BaseModel):
    model_name: str
    input_fee: str
    output_fee: str

url = 'https://openai.com/api/pricing/'
crawler = WebCrawler()
crawler.warmup()
result = crawler.run(
    url=url,
    extraction_strategy=LLMExtractionStrategy(
        provider="openai/gpt-4o",
        api_token='your_api_key',
        schema=OpenAIModelFee.schema(),
        instruction="Extract all model names and their fees."
    )
)
print(result.extracted_content)