One-Hot Encoding

merge_columns.py

def merge_columns(df: pd.DataFrame) -> pd.DataFrame:
    unique_columns = df.columns.unique()
    total_n_columns = len(df.columns)
    unique_n_columns = len(unique_columns)

    print(f"There are {total_n_columns} columns, {unique_n_columns} are unique")

    # if there are duplicates at all
    if total_n_columns != unique_n_columns:
        # get each unique colummn name
        for column in unique_columns:
            # If the "shape" array contains 2 elements, it means that
            # there is more than one column with the same name
            if len(df[column].shape) == 2:
                column_control = f"{column}_control"
                # sum the value of all the duplicate columns for each row
                # and assign it to a new control column which will then be used as the
                # final (merged) column for the duplicates
                df = pd.concat([
                    df,
                    pd.Series(
                        df[column].sum(axis=1),
                        name=column_control
                    )
                ], axis='columns')

                df = df.drop(columns=[column]).rename(columns={column_control: column})

            count = count + 1

    return df

onehot_encoding.py

onehot_encodings = []

# For each column
for column in df.columns:
    print(column)
    # Splits the values and expands them in multiple numbered columns
    temp_df = df[column].str.split("|", expand=True)
    # One-Hot encodes all the values for each column
    temp_df = pd.get_dummies(temp_df).astype('uint8')
    # Removes the "N_" prefixes for each column to expose duplicates
    temp_df = remove_prefixes(temp_df)
    # Merges the duplicate columns
    temp_df = merge_columns(temp_df)
    # Appends the one-hot encoded dataframe to the list
    onehot_encodings.append(temp_df)

onehot_encodings = pd.concat(onehot_encodings, axis='columns')

remove_prefixes.py

def remove_prefixes(df: pd.DataFrame) -> pd.DataFrame:
    # iterate over all the columns to remove the prefix,
    # to expose the columns with the same name
    columns = np.array([], dtype='str')
    
    for column in df.columns:
        # split the string by "_" and get the second token which corresponds
        # to the name of the column (e.g. N_something)
        columns = np.append(columns, column.split('_')[1])

    # assign the new column names back to the dataframe
    df.columns = columns

    return df