Houssem96

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datasets import list_datasets, load_dataset
import torch
from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification, Trainer, TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, classification_report

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

Houssem96

AG_news_dataset = load_dataset(dataset_name)
AG_news_dataset.set_format(type="pandas")
df = AG_news_dataset["train"][:]
def label_int2str(row, split):
    return AG_news_dataset[split].features["label"].int2str(row)

df["label_name"] = df["label"].apply(label_int2str, split="train")
df["label_name"].value_counts(ascending=True).plot.barh()
plt.title("Category Counts")

Houssem96

num_labels = 4
model_name = "distilbert-base-uncased"

AG_news_dataset_train = load_dataset(dataset_name, split='train[:8000]')
AG_news_dataset_validation = load_dataset(dataset_name, split='train[-2000:]')

tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name).to(device)
def tokenize(batch):
    return tokenizer(batch["text"], padding=True, truncation=True)

Houssem96

dataset_name = 'ag_news'
AG_news_dataset = load_dataset(dataset_name)
AG_news_dataset