Organize links in an advertools crawl DataFrame

crawl_link_summary.py

def link_summary(crawldf, internal_url_regex=None):
    """Get a DataFrame summary of links from a crawl DataFrame
    
    Parameters:
    -----------
    crawldf : DataFrame
      A DataFrame of a website crawled with advertools.
    internal_url_regex : str
      A regular expression for identifying if a link is internal or not.
      For example if your website is example.com, this would be "example.com".
    
    Returns:
    --------
    link_df : pandas.DataFrame
    """
    link_df = pd.merge(
        crawldf[['url']],
        crawldf.filter(regex='^links_').apply(lambda s: s.str.split('@@').explode()),
        left_index=True, right_index=True)
    link_df['links_nofollow'] = link_df['links_nofollow'].replace({
        'True': True, 'False': False, pd.NA: False})
    if internal_url_regex is not None:
        link_df['internal'] = link_df['links_url'].fillna('').str.contains(internal_url_regex, regex=True)
    link_df.columns = ['url', 'link', 'text', 'nofollow', 'internal']
    return link_df