See tutorial: https://github.com/huggingface/tokenizers/tree/91f602f744f7fee72f2f5fa8d6c7c48bb1d72d3b/bindings/python

nlp_datasets_hfds_tokenizer_training.py

with open("imdb_train_plain_lines.txt",'w') as f:
    for examples in ds2_train:
        f.write(examples['text'])
        f.write('\n')

from tokenizers import Tokenizer, models, pre_tokenizers, decoders, trainers, processors, BertWordPieceTokenizer

# Initialize a tokenizer
tokenizer = BertWordPieceTokenizer()

# Customize pre-tokenization and decoding
tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()
tokenizer.decoder = decoders.WordPiece()

# And then train
tokenizer.train([
	"imdb_train_plain_lines.txt"
], vocab_size=max_features, min_frequency=1)