Script to gather games from epic game store using burp as proxy to bypass anti-bot measures

crawl.py

import json
import random
import time

import requests

HEADERS = {
    "Cookie": "xxxx",
    "X-XSRF-TOKEN": "xxxx",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
    "Host": "www.epicgames.com",
    "Referer": "https://www.epicgames.com/",
    "Content-Type": "application/json;charset=utf-8",
}
ORDERS = set()


def set_convert(obj):
    if isinstance(obj, set):
        return list(obj)


def query_next_page(next_page_token=None):
    params = {"count": 10, "sortDir": "DESC", "sortBy": "DATE"}
    if next_page_token is not None:
        params["nextPageToken"] = next_page_token
    res = requests.get(
        "https://www.epicgames.com/account/v2/payment/ajaxGetOrderHistory",
        headers=HEADERS,
        params=params,
        proxies={
            "https": "http://localhost:8080",
            "http": "http://localhost:8080",
        },
        verify=False,
    )
    data = res.json()
    return data


def save():
    with open("orders.json", "w") as f:
        json.dump(ORDERS, f, default=set_convert, ensure_ascii=False)


def process_orders(data):
    for order in data["orders"]:
        for item in order["items"]:
            ORDERS.add(item["description"])


def main():
    data = query_next_page()
    process_orders(data)
    next_page_token = data["nextPageToken"]

    while True:
        data = query_next_page(next_page_token)
        process_orders(data)
        save()
        next_page_token = data["nextPageToken"]
        if next_page_token is None:
            break
        time.sleep(random.uniform(0.4, 1))


if __name__ == "__main__":
    main()