Big Boss 0xBigBoss



sudo ln -s /data2/k3s-rancher-etc/ /etc/rancher
sudo ln -s /data2/k3s/ /run/k3s
sudo ln -s /data2/k3s-kubelet/ /var/lib/kubelet
sudo ln -s /data2/k3s-rancher/ /var/lib/rancher

Run vLLM in Distributed Mode with Ray

Prerequisites

A docker image with the vLLM server installed.

export DOCKER_IMAGE=docker.io/fxnlabs/vllm-openai
# or use the following if you want to use the latest version

Demo a Simple LLM

A simple LLM and tokenizor for demonstrating KV Cache and inference using the transformer architecture(generating text).

Need PyTorch installed.

$ python ./demo_llm.py
Demonstrating KV cache

	# train_grpo.py
	import re
	import torch
	from datasets import load_dataset, Dataset
	from transformers import AutoTokenizer, AutoModelForCausalLM
	from peft import LoraConfig
	from trl import GRPOConfig, GRPOTrainer

	# Load and prep dataset

	-- Find indexes on the table
	SELECT indexname, tablename
	FROM pg_indexes
	WHERE tablename = '' \|\| ${table_name} \|\| '';

	-- Find foreign key relationships
	SELECT
	tc.table_schema,
	tc.constraint_name,
	tc.table_name,

	curl -sfL https://get.k3s.io \| sh -s - \
	--bind-address=0.0.0.0 \
	--flannel-iface=zt12345678 \
	--node-ip=10.0.0.2 \
	--cluster-init

	# from control plane
	K3S_URL=https://10.0.0.2:6443
	K3S_TOKEN=$(sudo cat /var/lib/rancher/k3s/server/node-token)

	async function callDebugRpcMethod(method: string, params: any[]) {
	const response = await fetch("http://localhost:3030/rpc", {
	headers: {
	"content-type": "application/json",
	},
	referrer: "http://localhost:3000/",
	body: JSON.stringify({
	jsonrpc: "2.0",
	id: 1,
	method: method,

	name: pytorch
	channels:
	- https://repo.anaconda.com/pkgs/main
	- https://repo.anaconda.com/pkgs/r
	dependencies:
	- python=3.12.8
	- cmake
	- ninja
	- libjpeg-turbo
	- libpng

	macos-option-as-alt = true
	keybind = cmd+right=text:\x05
	keybind = cmd+left=text:\x01
	keybind = alt+left=esc:b
	keybind = alt+right=esc:f

	#!/bin/bash

	set -eo pipefail

	user_input="$*"

	if [[ -z "$user_input" ]]; then
	user_input="beginner shell commands"
	echo "Using default input: $user_input"
	fi