how to sample from logits in huggingface transformers

hf_sample.py

# how to sample from logits in huggingface transformers

from transformers.generation.utils import MinPLogitsWarper, LogitNormalization
logits_processors = [
    MinPLogitsWarper(min_p=0.1),
    LogitNormalization() # alaways need this last
]

logits = o.logits[:, -1].clone()
# logits[:, banned_token_ids] = -float("inf")
for proc in logits_processors:
    logits = proc(input_ids, logits)

logp = logits.log_softmax(dim=-1)

if do_sample:
    new_token_id = torch.multinomial(logp.exp(), num_samples=1)
else:
    new_token_id = logp.argmax(dim=-1, keepdim=True)#.unsqueeze(1)

see

• sample: https://github.com/huggingface/transformers/blob/main/src/transformers/generation/utils.py#L2788
• minp: huggingface/transformers#27670