hearth_log_data_dump.py

#!/bin/env python
#
# This program downloads data from hearthlog and stores it in a
# single JSON file stored in AppData/Roaming/Hearth Log/Uploaded
# It has requires the requests library and has been tested with 
# Python 2.7. It is licensed under the WTFPL version 2.

import os, requests, re, json, datetime

username = 'AnnanFay'
data_file = 'hl_dump-%s.json'

hl_path =  os.path.join(os.getenv('APPDATA'), 'Hearth Log', 'Uploaded')
dl_link = 'http://www.hearthlog.com/u/%s/%s/replay'

start = r'<script>hearthlog\.replay\('
end = r'\)</script>'
reg = start + r'(.*?)' + end

def js_to_json(js):
    json = re.sub(r'\s*(\w+?):', r'"\1":', js)
    json = re.sub(r',\s*([\]\}\)])', r'\1', json)
    return json

def grab(fn):
    ts = fn[:-4]
    path = dl_link % (username, ts)
    print 'grabbing:', path
    r = requests.get(path)
    m = re.search(reg, r.text, re.DOTALL)
    if m:
        j = json.loads(js_to_json(m.group(1)))
        return j

if __name__ == '__main__':
    data = []
    for (dirpath, dirnames, filenames) in os.walk(hl_path):
        print 'grabbing %s files in %s' % (len(filenames), dirpath)
        data += map(grab, filter(lambda fn: '.hsl' in fn, filenames))

    os.chdir(hl_path)
    with file(data_file % (datetime.datetime.now().strftime('%Y%m%d-%H%M%S')), 'w') as f:
        print 'dumping all data to: ', f.name
        json.dump(data, f, indent=4)