Example of creating a Gist using Python

intro-zero_hugging-face-course-tokenizers.py

import torch
from transformers import AutoTokenizer

checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = "I've been waiting for a HuggingFace course my whole life."

tokens = tokenizer.tokenize(sequence)
print("Tokens:", tokens)
ids = tokenizer.convert_tokens_to_ids(tokens)
input_ids = torch.tensor([ids])
print("Input IDs:", input_ids)

final_inputs = tokenizer.prepare_for_model(ids)
print("Final Inputs:", final_inputs)

decoded_inputs = tokenizer.decode(final_inputs['input_ids'])
print("Decoded Inputs:", decoded_inputs)