Python We Follow Scraper

getWeFollow.py

import requests
from bs4 import BeautifulSoup


# Get top influencers from WeFollow
def get_influencers(topic, min_rank, max_rank):
    url = 'http://wefollow.com/interest/'
    req_url = url + topic + '/' + str(min_rank) + '-' + str(max_rank)
    response = requests.get(req_url)
    soup = BeautifulSoup(response.content)
    # get the blocks of html that contain the information we care about
    influencer_soup = soup.findAll("div", {"class": "interest-list-user" })

    topic_influencers = []
    for influencer in influencer_soup:
        handle = influencer.findAll("p", {"class": "user-username"})[0].string
        rank = influencer.findAll("p", {"class": "rank-big"})[0].string
        topic_influencers.append((topic, handle, rank))

    return topic_influencers

# example
# print(get_influencers('hadoop',80,100))