transformers_linear_regression

f1.py

gpu_info = !nvidia-smi
gpu_info = '\n'.join(gpu_info)
if gpu_info.find('failed') >= 0:
  print('Not connected to a GPU')
else:
  print(gpu_info)

f10.py

test_embeddings = tokenizer(test_reviews_3000.reviews.astype(str).tolist(), truncation=True, padding=True, max_length=126)
test_dataset = MakeTorchData(test_embeddings, test_reviews_3000.Rating.astype(float))
bert_trainer.eval_dataset = test_dataset
bert_trainer.evaluate()

f11.py

{'epoch': 5.0,
 'eval_accuracy': 0.6603333333333333,
 'eval_loss': 0.48832669854164124,
 'eval_mae': 0.45958763360977173,
 'eval_mse': 0.4883267283439636,
 'eval_r2': 0.6103774787626888,
 'eval_rmse': 0.6988037824630737,
 'eval_runtime': 13.5094,
 'eval_samples_per_second': 222.068,
 'eval_steps_per_second': 11.103}

f2.py

import torch, gc, random, datasets
from transformers.file_utils import is_tf_available, is_torch_available
from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments
%load_ext memory_profiler
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score, mean_squared_error, mean_absolute_error
import pandas as pd
import numpy as np

f3.py

# Make data
X = Data
y = Target


# Split Data
X_train, X_test, y_train, y_test = train_test_split(X.tolist(), y, test_size=test_size)

# Call the Tokenizer
tokenizer = AutoTokenizer.from_pretrained(model_name)

# Encode the text
train_encodings = tokenizer(X_train, truncation=True, padding=True, max_length=max_length)
valid_encodings = tokenizer(X_test, truncation=True, padding=True, max_length=max_length)



class MakeTorchData(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {k: torch.tensor(v[idx]) for k, v in self.encodings.items()}
        item["labels"] = torch.tensor([self.labels[idx]])
        item["labels"] = float(item["labels"])
        return item

    def __len__(self):
        return len(self.labels)

# convert our tokenized data into a torch Dataset
train_dataset = MakeTorchData(train_encodings, y_train.ravel())
valid_dataset = MakeTorchData(valid_encodings, y_test.ravel())

f4.py

model = AutoModelForSequenceClassification.from_pretrained(model_name, 
                                                           num_labels = 1).to("cuda")

f5.py

def compute_metrics_for_regression(eval_pred):
    logits, labels = eval_pred
    labels = labels.reshape(-1, 1)

    mse = mean_squared_error(labels, logits)
    rmse = mean_squared_error(labels, logits, squared=False)
    mae = mean_absolute_error(labels, logits)
    r2 = r2_score(labels, logits)
    smape = 1/len(labels) * np.sum(2 * np.abs(logits-labels) / (np.abs(labels) + np.abs(logits))*100)

    return {"mse": mse, "rmse": rmse, "mae": mae, "r2": r2, "smape": smape}

f6.py

# Specifiy the arguments for the trainer  
training_args = TrainingArguments(
    output_dir ='./results',          
    num_train_epochs = num_epochs,     
    per_device_train_batch_size = 64,   
    per_device_eval_batch_size = 20,   
    weight_decay = 0.01,               
    learning_rate = 2e-5,
    logging_dir = './logs',            
    save_total_limit = 10,
    load_best_model_at_end = True,     
    metric_for_best_model = 'rmse',    
    evaluation_strategy = "epoch",
    save_strategy = "epoch",
) 

# Call the Trainer
trainer = Trainer(
    model = model,                         
    args = training_args,                  
    train_dataset = train_dataset,         
    eval_dataset = valid_dataset,          
    compute_metrics = compute_metrics_for_regression,     
)

# Train the model
trainer.train()

# Call the summary
trainer.evaluate()

f7.csv

f8.py

def compute_metrics_for_regression(eval_pred):
  
  ...
  ...
  
  single_squared_errors = ((logits - labels).flatten()**2).tolist()
  accuracy = sum([1 for e in single_squared_errors if e < 0.25]) / len(single_squared_errors)
  
  return {"mse": mse, "rmse": rmse, "mae": mae, "r2": r2, "accuracy": accuracy}

f9.csv