Created
September 30, 2019 22:03
-
-
Save MetroWind/d49ae1fa840d73cbd27cca58a0329d77 to your computer and use it in GitHub Desktop.
Scrap the game list JSON from Steam profile page.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
import sys, os | |
import json | |
import time | |
from tqdm import tqdm | |
import requests | |
import lxml.html | |
with open("games.json", 'r') as f: | |
GamesRaw = json.load(f) | |
Cookies = {"birthtime": "502272001"} | |
for Game in tqdm(GamesRaw): | |
# print('.', file=sys.stderr, end='', flush=True) | |
URI = "https://store.steampowered.com/app/" + str(Game["appid"]) | |
Res = requests.get(URI, cookies=Cookies) | |
if Res.status_code != 200: | |
while Res.status_code == 500: | |
time.sleep(1) | |
Res = requests.get(URI, cookies=Cookies) | |
else: | |
Res.raise_for_status() | |
Root = lxml.html.fromstring(Res.text) | |
Containers = Root.xpath('//div[@id="game_highlights"]/div/div[@class="glance_ctn"]') | |
if len(Containers) == 0: | |
# The game might not exist anymore. Example: Test Drive unlimited 2 (9930). | |
print("-", Game["name"]) | |
elif Containers[0].xpath('div[@class="glance_details"]'): | |
pass | |
else: | |
print("- [{}]({})".format(Game["name"], URI)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment