Extract highlights from a piece of content.

highlights.py

import outlines
from pydantic import BaseModel
from transformers import AutoTokenizer
from rich import print

model_string = "HuggingFaceTB/SmolLM2-1.7B-Instruct"
model = outlines.models.transformers(model_string)
tokenizer = AutoTokenizer.from_pretrained(model_string)

class Highlights(BaseModel):
  highlights: list[str]
    
# Adding reasoning and observations to assist model performance.
# class Highlights(BaseModel):
#   observations: list[str]
#   reasoning: list[str]
#   highlights: list[str]

generator = outlines.generate.json(model, Highlights)

article = """
Red Hat, Inc., the world's leading provider of open source solutions, today announced that it has completed its acquisition of Neural Magic, a pioneer in software and algorithms that accelerate generative AI (gen AI) inference workloads. With Neural Magic, Red Hat adds expertise in inference performance engineering and model optimization, helping further the company’s vision of high-performing AI workloads that directly map to unique customer use cases, wherever needed across the hybrid cloud.

> By adding Neural Magic’s expertise in gen AI performance engineering and optimization to Red Hat AI, we’re furthering our commitment to a gen AI that answers customers’ unique needs, from where workloads run to how they are tuned and trained.
> Matt Hicks -- president and CEO, Red Hat

The large language models (LLMs) underpinning today’s gen AI use cases, while innovative, are often too expensive and resource-intensive for most organizations to use effectively. To address these challenges, Red Hat views smaller, optimized and open source-licensed models driven by open innovation across compute architectures and deployment environments as key to the future success of AI strategies.
"""

templated = tokenizer.apply_chat_template(
    conversation=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": f"""
         Please provide a list of highlights and key takeaways from the article. 
         Respond in JSON format with the following schema: {Highlights.model_json_schema()}. 
         \n\nArticle: {article}
         """},
    ],
    tokenize=False,
    add_generation_prompt=True,
)

print(templated)

highlights = generator(templated)

print(highlights)
# highlights=['Red Hat completes acquisition of Neural Magic, a pioneer in software and algorithms accelerating gen AI workloads', "Neural Magic's addition brings expertise in inference performance engineering and model optimization to enhance gen AI resources and customer experiences", 'Matt Hicks, president and CEO of Red Hat, emphasizes the importance of open source non-traditional models and innovative approaches for AI success']