Demonstration of XLNet with Classification head on top, implementation of XLNet follows huggingface's pytorch build.

XLNet_sequence_classification

from transformers import XLNetTokenizer, XLNetForSequenceClassification
import torch
tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased')
model = XLNetForSequenceClassification.from_pretrained('xlnet-large-cased')
input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
outputs = model(input_ids, labels=labels)
loss, logits = outputs[:2]
print(f'Current Loss at -- {loss.tolist()}')
# Current Loss at -- 1.1906177997589111
print(f'Output labels are -- {logits.tolist()[0]}')
# Output labels are -- [0.613148033618927, -0.21501630544662476]