Handle outliers using IQR and eliminate rows

handle_outliers_eliminate_rows.py

import numpy as np

# data as pandas DataFrame
train = ...

# Variables to store out bounds by column value
upper_bounds = {}
lower_bounds = {}

# Best practice
train_copy = train.copy()

for c in train_copy.columns:
    # Take inter quartile boundary range
    q25, q75 = np.percentile(train_copy[c], [25, 75])
    intr_qr = q75 - q25

    # Set upper/lower bounds (arbitrary)
    upper = q75 + 1.5 * intr_qr
    lower = q25 - 1.5 * intr_qr

    # Store bounds
    upper_bounds[c] = upper
    lower_bounds[c] = lower

    # Eliminate any row that doesn't fit within the required bounds
    train_copy = train_copy[(train_copy[c] >= lower) & (train_copy[c] <= upper)].reset_index(drop=True)

train_copy.shape