Scrap popular posts

popular_posts_scraping.py

from python_graphql_client import GraphqlClient
import time


client = GraphqlClient(endpoint="https://medium.com/_/graphql")


def get_posts(data, posts):
    for post in data["data"]["topic"]["latestPosts"]["postPreviews"]:
        title = ""
        author = ""
        author_username = ""
        url = ""
        domain = ""

        if 'title' in post['post']:
            title = post['post']['title']
        if 'creator' in  post['post']:
            creator = post['post']['creator'] or []
            if 'username' in creator:
                author = post['post']['creator']['name']
                author_username = post['post']['creator']['username']
        if 'mediumUrl' in post['post']:
            url = post['post']['mediumUrl']
        if 'collection' in post['post']:
            collection = post['post']['collection'] or []
            if 'domain' in collection:
                domain = post['post']['collection']['domain']
        post = {
            "title": title, 
            "author": author,
            "author_username": author_username,
            "url": url,
            "domain": domain
        }
        if post not in posts:
            posts.append(post)
    return posts
    
    
posts = []
nb_h_per_day = 6
for i in range(365 * 24 // nb_h_per_day):
    variables = {"topicSlug": "popular", "feedPagingOptions": {"limit": 25, "to": f"{int(time.time() * 1000) - 3600000 * nb_h_per_day * i}"}, "sidebarPagingOptions": {"limit": 1}}
    data = client.execute(query=query, variables=variables)
    posts = get_posts(data, posts)