Skip to content

Instantly share code, notes, and snippets.

@MetroWind
Created September 30, 2019 22:03
Show Gist options
  • Save MetroWind/d49ae1fa840d73cbd27cca58a0329d77 to your computer and use it in GitHub Desktop.
Save MetroWind/d49ae1fa840d73cbd27cca58a0329d77 to your computer and use it in GitHub Desktop.
Scrap the game list JSON from Steam profile page.
#!/usr/bin/env python3
import sys, os
import json
import time
from tqdm import tqdm
import requests
import lxml.html
with open("games.json", 'r') as f:
GamesRaw = json.load(f)
Cookies = {"birthtime": "502272001"}
for Game in tqdm(GamesRaw):
# print('.', file=sys.stderr, end='', flush=True)
URI = "https://store.steampowered.com/app/" + str(Game["appid"])
Res = requests.get(URI, cookies=Cookies)
if Res.status_code != 200:
while Res.status_code == 500:
time.sleep(1)
Res = requests.get(URI, cookies=Cookies)
else:
Res.raise_for_status()
Root = lxml.html.fromstring(Res.text)
Containers = Root.xpath('//div[@id="game_highlights"]/div/div[@class="glance_ctn"]')
if len(Containers) == 0:
# The game might not exist anymore. Example: Test Drive unlimited 2 (9930).
print("-", Game["name"])
elif Containers[0].xpath('div[@class="glance_details"]'):
pass
else:
print("- [{}]({})".format(Game["name"], URI))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment