Dump/Archive/Export Ryver.com forum chat history

dump_ryver.py

#! /usr/bin/env python3
# Usage: dump_ryver_forum.py <PHPSESSID> <forum name, like "foo.ryver.com"> <forum ids, e.g. 1217356 ...>
# Requires Python 3.6 and the `requests` library:
#   pip install requests
import requests
import sys
import time

session_id = sys.argv[1]
forum_name = sys.argv[2]
forum_ids = sys.argv[3:]

cookies = dict(PHPSESSID=session_id)
headers = {
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'en',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)',
    'Content-Type': 'application/json',
    'Accept': 'application/json, */*',
    'X-Requested-With': 'XMLHttpRequest',
    'Connection': 'keep-alive',
    'Authorization-Mode': 'no-challenge',
}

def get_params(id_):
    return tuple(filter(None, (
        ('$format', 'json'),
        ('$top', '50'),
        ('$filter', f"id lt '{id_}'") if id_ else None,
        ('$orderby', 'when desc'),
        ('$inlinecount', 'allpages'),
        ('_', str(int(1000 * time.time()))),
    )))

def dump_forum(session, forum_name, forum_id):
    url = f'https://{forum_name}/api/1/odata.svc/forums({forum_id})/Chat.History()'
    id_ = None
    for i in range(1000): # limit to a maximum of ~50k posts per forum
        f = f'{forum_name}_{forum_id}_{i}.json'
        print(url, f'page={i}')
        response = session.get(url, headers=headers, params=get_params(id_), cookies=cookies)
        j = response.json()
        if not j or not j['d']['results']: break
        id_ = j['d']['results'][-1]['id']
        with open(f, 'wb') as fh:
            fh.write(response.content)


if __name__ == '__main__':
    with requests.Session() as session:
        for forum_id in forum_ids:
            dump_forum(session, forum_name, forum_id)