tommydangerous

from sklearn.preprocessing import OneHotEncoder

categorical_columns = ['Pclass', 'Sex', 'Embarked', 'cabin_letter']
categorical_encoder = OneHotEncoder(handle_unknown='ignore')
categorical_encoder.fit(df[categorical_columns])

# Add the new columns to the data
new_column_names = []
for idx, cat_column_name in enumerate(categorical_columns):
    values = categorical_encoder.categories_[idx]

tommydangerous

from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
df.loc[:, ['Age']] = scaler.fit_transform(df[['Age']])

tommydangerous

from sklearn.impute import SimpleImputer

print(f'Missing values in "Cabin": {len(df[df["Cabin"].isna()].index)}')
df.loc[df['Cabin'].isna(), 'Cabin'] = 'somewhere out of sight'
df.loc[df['cabin_letter'].isna(), 'cabin_letter'] = 'ZZZ'

print(f'Missing values in "Age": {len(df[df["Age"].isna()].index)}')
age_imputer = SimpleImputer(strategy='median')
df.loc[:, ['Age']] = age_imputer.fit_transform(df[['Age']])

tommydangerous

df = df.drop(columns=['Name', 'PassengerId'])

# Name and PassengerId is no longer a column
df.columns.tolist()

tommydangerous

df = X_train_raw.copy()

# Add a column to determine if the person can vote
df['can_vote'] = df['Age'].apply(lambda age: 1 if age >= 18 else 0)

# 892 passengers can vote; aka they are 18 or older
df['can_vote'].value_counts()

# Cabin letter: a cabin can be denoted as B123. The cabin letter will be B.
df.loc[:, 'cabin_letter'] = df['Cabin'].apply(

tommydangerous

X_train_raw, X_test_raw, y_train, y_test = train_test_split(
    X,
    y,
    stratify=y,
    test_size=0.2,
)

tommydangerous

from sklearn.model_selection import train_test_split
import pandas as pd

df = pd.read_csv('/content/titanic_survival.csv')
label_feature_name = 'Survived'

X = df.drop(columns=[label_feature_name])
y = df[label_feature_name]

tommydangerous

from pyspark.sql import SparkSession
from pyspark.sql.functions import pandas_udf, PandasUDFType
from pyspark.sql.types import (
    IntegerType,
    StringType,
    StructField,
    StructType,
)

tommydangerous

from pyspark.sql.functions import pandas_udf, PandasUDFType
from pyspark.sql.types import (
    IntegerType,
    StringType,
    StructField,
    StructType,
)


"""

tommydangerous

from pyspark.sql.functions import pandas_udf, PandasUDFType
from pyspark.sql.types import (
    IntegerType,
    StringType,
    StructField,
    StructType,
)


"""