flatten_df.py

import itertools
import pandas as pd

def flatten_df(df, list_col, elem_col_name="elem"):
    """Convert a series of list to individual rows, within a dataframe.
    Adapted from https://stackoverflow.com/a/48532692
    This function can be used on a dask dataframe:
    
    ```python
    df.map_partitions(lambda x: flatten_df(x, "list_col", elem_col_name="elem")).clear_divisions()
    // I am not sure if the clear_divisions is requied
    ```
    """
    
    len_lists = df[list_col].str.len()
    return pd.DataFrame({
        **{col: df[col].repeat(len_lists) for col in df.columns.drop(list_col)},
        elem_col_name: list(itertools.chain.from_iterable(df[list_col].values))})