dreyco676

import requests
from bs4 import BeautifulSoup


# Get top influencers from WeFollow
def get_influencers(topic, min_rank, max_rank):
    url = 'http://wefollow.com/interest/'
    req_url = url + topic + '/' + str(min_rank) + '-' + str(max_rank)
    response = requests.get(req_url)
    soup = BeautifulSoup(response.content)

dreyco676

from pyspark.sql.functions import split, explode, lit, coalesce, first

# split 'ROOF' column by comma
df = df.withColumn('roof_list', split(df['ROOF'], ', '))

# explode each value to a new record
ex_df = df.withColumn('ex_roof_list', explode(df['roof_list']))

# create a new record to agg by later
ex_df = ex_df.withColumn('constant_val', lit(1))