vadimkantorov

alias cmdline='ps --no-headers -o comm,args -p'
cmdline 1

# can produce JSON, but without process args
alias notbrokenjson='ps --no-headers -o '\''{"pcpu": %C, "group": "%G", "PPID": %P, "user": "%U", "comm": "%c", "rgroup": "%g", "nice": "%n", "pid": %p, "pgid": %r, "etime": "%t", "ruser": "%u", "time": "%x", "tty": "%y", "vsz": %z}'\'' -p'
notbrokenjson 1 > notbroken.json

# does not remove escape sequences or quotes in args - this will break JSON format for some processes
alias brokenjson='ps --no-headers -o '\''{"pcpu": %C, "group": "%G", "PPID": %P, "user": "%U", "args": "%a", "comm": "%c", "rgroup": "%g", "nice": "%n", "pid": %p, "pgid": %r, "etime": "%t", "ruser": "%u", "time": "%x", "tty": "%y", "vsz": %z}'\'' -p'
brokenjson 1 > brokenjson.json

vadimkantorov

# git clone https://gist.github.com/vadimkantorov/fe63f8628ff6cad460e934e1d7ed650b
# cd fe63f8628ff6cad460e934e1d7ed650b
# uv venv
# uv sync

# https://github.com/vllm-project/vllm/pull/20358#issuecomment-3247178818
# https://github.com/vllm-project/vllm/issues/9244
# https://github.com/astral-sh/uv/issues/8082
# https://github.com/vllm-project/vllm/issues/24126

vadimkantorov

# https://stackoverflow.com/questions/71944328/how-to-implement-json-format-logs-in-python

import json
import logging

# https://docs.python.org/3/library/logging.html#logrecord-attributes
formatter = type('JsonFormatter', (logging.Formatter, ), dict(format = lambda self, record: json.dumps(dict(time = self.formatTime(record), level = record.levelname, message = record.getMessage(), module = record.module, lineno = record.lineno)) ))()

# simpler version below does not escape quotes in message and does not delete newlines in message
# formatter = logging.Formatter('{\"time\": \"%(asctime)-s\", \"level\": \"%(levelname)-s\", \"message\": \"%(message)s\", \"module\": \"%(module)s\", \"lineno\": %(lineno)d}')

vadimkantorov

echo '{"site": {"hello": "world"}}' > foo.json
echo 'Hello {{ site.hello }}' > foo.txt.j2

bash j2.sh -t foo.txt.j2 -i foo.json -o foo.txt
cat foo.txt

source function.sh
j2 -t foo.txt.j2 -i foo.json -o foo.txt
cat foo.txt
unset -f j2

vadimkantorov

# Extracted and simplified the two-level aggregation approach (first, parallel aggregation in blocks, then final sequential aggregation) from https://github.com/volcengine/verl/blob/main/verl/utils/kernel/kernels.py
# Examples of single-level sequential, online aggregation approaches:
# - https://github.com/linkedin/Liger-Kernel/blob/main/src/liger_kernel/ops/cross_entropy.py
# - https://github.com/Dao-AILab/flash-attention/blob/main/flash_attn/ops/triton/cross_entropy.py
# logsumexp_torch has some eager pseudo/code in PyTorch which emulates what Triton does, except that BLOCK_SIZE_M equials to M
# tl.program_id(axis=0).to(tl.int64) is used for https://arxiv.org/abs/2410.10989 and https://github.com/linkedin/Liger-Kernel/blob/05b43a14913ced3776aa3fc50020089b8c0d63c1/src/liger_kernel/ops/cross_entropy.py#L77-L79
# sample_verl.pt is derived from the inputs (logits = torch.matmul(hidden, weights) uploaded by @WindowsXP-Beta in https://github.com/volcengine/verl/issues/2656#issuecomment-3131136498 )
# created for

vadimkantorov

nvidia-smi -q -x | grep "</pid>" | tr -d "</pid>\t"

vadimkantorov

# https://github.com/pytorch/pytorch/issues/158710
# https://github.com/pytorch/pytorch/issues/158698
# https://github.com/pytorch/pytorch/issues/69431

import torch

def to_(tensor1d, dtype, *, chunks = 0, split_size = 0):
    # TODO: instead of clone() maybe could copy_ into a buffer, clone() does not allow using a buffer
    # TODO: unclear if these codes can support autograd, and if so, will it remember too much in saved_for_backward 
    

vadimkantorov

# https://en.wikipedia.org/wiki/Base64
# 00123456  00ABCDEF  00abcdef  00uvwxyz
# 123456AB  CDEFabcd  efuvwxyz
# this code does not support batches. adapting for e.g. concatenated varlen format is possible, but need to handle/preserve varlen information and paddings in some way

import torch

def base64_encode_padded(input_as_uint8_tensor):
    base64_alphabet, base64_pad = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '='
    device = input_as_uint8_tensor.device

vadimkantorov

# Nebius s3
# sudo apt-get install s3cmd # export the vars below or configure access_key / secret_key
# export AWS_ACCESS_KEY_ID=...
# export AWS_SECRET_ACCESS_KEY=...
# https://s3tools.org/usage
# s3cmd -c ~/.nebius.s3cfg ls

[default]
# access_key=...
# secret_key=...

vadimkantorov

# PYTHONPATH=. python ...

import os
import torch

def cuda_oom_hook(device, alloc, device_alloc, device_free, info = dict(counter = 0), snapshot_dump_file_pattern = './memory_snapshot_{pid}_{oom_counter}.pt'):
    memory_summary = torch.cuda.memory_summary(device = device)
    memory_snapshot = torch.cuda.memory._snapshot(device = device)
    pid = os.getpid()
    print('device:', device, 'oom#:', info['oom_counter'], 'pid:', pid, 'alloc:', alloc, 'device_alloc:', device_alloc, 'device_free:', device_free)