| # Related to issue at https://github.com/vllm-project/vllm/issues/17448 | |
| import os | |
| from vllm import LLM, SamplingParams | |
| from vllm.inputs import TokensPrompt | |
| # Get your Huging face access token at https://huggingface.co/settings/tokens | |
| HF_TOKEN = os.getenv("HF_TOKEN") | |
| llm = LLM(model="google/gemma-3-1b-it") | |
| token_ids = [2, 236772, 35559, 236772, 107, 3048, 659, 496, 9550, 7820, 34799, 236761, 5180, 4209, 563, 531, 18840, 496, 2238, 1816, 2354, 236764, 14846, 7798, 531, 18698, 14050, 236764, 532, 8430, 784, 3629, 9994, 1534, 20880, 6950, 2351, 506, 1816, 236761, 30235, 22525, 580, 21558, 532, 22454, 1239, 9994, 236764, 2440, 4889, 1003, 506, 6638, 20880, 236761, 108, 236772, 59881, 236772, 107, 236770, 236761, 8847, 506, 2744, 1816, 13058, 236761, 49644, 14491, 529, 20880, 568, 6265, 236764, 3328, 236768, 600, 659, 808, 181670, 4596, 236829, 2721, 580, 506, 1938, 6212, 528, 506, 1816, 236761, 41152, 1800, 20880, 528, 496, 6727, 8386, 531, 886, 529, 506, 9155, 2165, 11128, 236779, 9846, 21233, 107, 236778, 2367 |
When running FastAPI app, all the logs in console are from Uvicorn and they do not have timestamp and other useful information. As Uvicorn applies python logging module, we can override Uvicorn logging formatter by applying a new logging configuration.
Meanwhile, it's able to unify the your endpoints logging with the Uvicorn logging by configuring all of them in the config file log_conf.yaml.
Before overriding:
uvicorn main:app --reload
| # brew install gifsicle | |
| gifsicle -O3 --lossy=200 -o output.gif input.gif |
| import argparse | |
| from getpass import getpass | |
| from mistralai import Mistral | |
| MISTRAL_MODEL_NAME = "mistral-small-latest" # uses "mistral-small-2503" | |
| LANGUAGE_PROMPTS = { | |
| "en": "Summarize this paragraph: Supreme Court sides with administration over Education Department grants. The U.S. Supreme Court on Friday sided with the Trump administration, at least for now, in a dispute over the Department of Education's freeze of DEI-related grants. The administration has taken several grievances to the high court recently, but this was the first of its legal theories to stick.By a 5-4 vote, the justices allowed the administration to keep frozen $65 million for teacher training and professional development, halting a lower court order that had temporarily reinstated the grants.", | |
| "fr": "Résumé ce paragraphe: Selon une enquête de Mediapart, révélée vendredi 4 avril 2025, une enseignante du collège privé catholique Saint-Joseph de Plabennec (Finistère) publiait depuis plusieurs années, sous un pseudonyme, des prop |
| +-----------------------+ HTTP Requests +---------------------------------+ Trace Data +-------------------+ | |
| | Client de Load Test | -------------------> | vLLM OpenAI API Endpoint | -------------------> | Langfuse Cloud | | |
| | (k6, Locust, script) | (Load Profile) | (Tournant sur H100 ou L40) | (SDK/API Call) | (UI & API) | | |
| | | | (Modèle: Qwen / Llama3) | | | | |
| | - Génère QPS | <------------------- | | <------------------- | - Visualisation | | |
| | - Mesure Latence (Client) | Responses | - Traite les requêtes | User Query | - Analyse | | |
| | - Mesure Taux Succès | | - Mesure Latence (Server) | | - Export | | |
| +-----------------------+ | - Mesure Tokens In/Out | +------------------ |
| GRAPH_EXTRACTION_PROMPT = """ | |
| -Goal- | |
| Given a text document that is potentially relevant to this activity and a list of entity types, identify all entities of those types from the text and all relationships among the identified entities. | |
| -Steps- | |
| 1. Identify all entities. For each identified entity, extract the following information: | |
| - entity_name: Name of the entity, capitalized | |
| - entity_type: One of the following types: [{entity_types}] | |
| - entity_description: Comprehensive description of the entity's attributes and activities | |
| Format each entity as ("entity"{{tuple_delimiter}}<entity_name>{{tuple_delimiter}}<entity_type>{{tuple_delimiter}}<entity_description>) |
| # Start vLLM OpenAI compatible server | |
| docker run --runtime nvidia --gpus all \ | |
| -v ~/.cache/huggingface:/root/.cache/huggingface \ | |
| --env "HUGGING_FACE_HUB_TOKEN=<secret>" \ | |
| -p 8000:8000 \ | |
| --ipc=host \ | |
| vllm/vllm-openai:latest \ | |
| --api-key your-secret-key \ | |
| --tokenizer-mode "mistral" \ | |
| --model mistralai/Mistral-Small-24B-Instruct-2501 |
| #!/bin/bash | |
| VIDEO_SOURCE="video.mp4" | |
| RTSP_URL="rtsp://machine-ip:8554/stream" | |
| ffmpeg \ | |
| -stream_loop -1 \ | |
| -re \ | |
| -i "$VIDEO_SOURCE" \ | |
| -c:v libx264 \ |
| #!/bin/bash | |
| sudo apt update -y | |
| # Install oh my bash | |
| bash -c "$(curl -fsSL https://raw.githubusercontent.com/ohmybash/oh-my-bash/master/tools/install.sh)" | |
| # Set up SSH key for GitHub | |
| echo "Setting up SSH key for GitHub..." | |
| ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -N "" || { echo "SSH key generation failed"; exit 1; } |
| #!/bin/bash | |
| # This script initializes a GPU machine to start a vLLM server | |
| # Ensure the script is run as root | |
| if [[ $EUID -ne 0 ]]; then | |
| echo "This script must be run as root" | |
| exit 1 | |
| fi |