fanpagelist-Eshikhon_Class_22.py

#!/usr/bin/env python3

import sys
import requests as rq

from bs4 import BeautifulSoup

url = "https://fanpagelist.com/category/celebrities"
pagination = "/view/list/sort/fans/page"

f = open('data.txt', "w", encoding="utf-8")

def get_info(data_obj):
    rank_list = []
    name_list = []
    for data in data_obj:
        rank = data.find('div', {'class': 'rank_number'}
                         ).text[1:].strip('tweet this')
        rank_list.append(rank)
        # print(rank)
        name = data.find('span', {'class': 'title'}).text
        # print(name.text)
        name_list.append(name)
    return rank_list, name_list


def grab_names():
    res = rq.get(url)
    data = BeautifulSoup(res.text, "html.parser")
    rank, name = get_info(data.find_all('li', {'class': 'ranking_results'}))

    for i in range(len(rank)):
        f.write(f"{rank[i]} - {name[i]}\n")

    for i in range(2, 10):
        print("Page: ", i)
        res = rq.get(url + pagination + str(i), allow_redirects=False)
        if res.status_code == 200:
            data = BeautifulSoup(res.text, "html.parser")
            rank, name = get_info(data.find_all(
                'li', {'class': 'ranking_results'}))
            for i in range(len(rank)):
                f.write(rank[i] + " - " + name[i] + "\n")
        else:
            print("Page Limit Exceeded")
            break

grab_names()
f.close()