Download comments from projecteuler.net for given problem range, user.

.gitignore

/.cachedir/
/comments-*.json

get-my-projecteuler-comments.py

#!/usr/bin/env python
"""Download comments from projecteuler.net for given problem range, user."""
from __future__ import print_function
import json
import re
import sys
import time
from argparse import ArgumentParser
from getpass import getpass
from netrc import netrc
from timeit import default_timer as timer

import requests  # pip install requests
from bs4 import BeautifulSoup  # pip install beautifulsoup4

try:
    from werkzeug.contrib.cache import FileSystemCache  # pip install werkzeug
except ImportError:
    cache = None
else:
    cache = FileSystemCache('.cachedir', threshold=10000)

__version__ = '0.1.0'
USER_AGENT = 'get-my-projecteuler-comments/' + __version__
CACHE_TIMEOUT = 60 * 60 * 24 * 7  # week
ALL = object()

class Client(object):
    def __init__(self, user, passwd, host, delay=5, verbose=False):
        self._user = user
        self._passwd = passwd
        self._url = 'https://' + host
        self._session = requests.session(headers={'User-Agent': USER_AGENT},
             config={'verbose': verbose and sys.stderr,
                     'danger_mode': True})
        self._delay = delay  # seconds
        self._last_request_time = None

    def login(self):
        self._session.post(self._url + '/login', verify=True,
                           data=dict(username=self._user,
                                     password=self._passwd,
                                     remember=1, login='Login'),
                           allow_redirects=False)
        self._last_request_time = timer()

    def _get_url(self, url):
        text = cache.get(key=url) if cache else None
        if not text:
            time.sleep(self._last_request_time + self._delay - timer())
            r = self._session.get(url)
            self._last_request_time = timer()
            text = r.text
            if cache:
                cache.set(url, text, timeout=CACHE_TIMEOUT)
        return text

    def get_comments(self, problem_id, username=ALL):
        html = self._get_url(self._url + '/thread=%d' % problem_id)
        p = Parser(html)
        for x in p.comments(username):
            yield x

        # get all pages in the thread
        pattern = re.compile(r'thread=%d;page=\d+' % problem_id)
        for page in sorted(p.page_numbers(pattern)):
            html = self._get_url(self._url + '/thread=%d;page=%d' % (
                    problem_id, page))
            for x in Parser(html).comments(username):
                yield x


class Parser(object):
    """Helper object for Client.get_comments()."""
    def __init__(self, html):
        self._soup = BeautifulSoup(html)

    def page_numbers(self, pattern):
        return set(int(a.string) for a in self._soup('a', href=pattern))

    def comments(self, username=ALL):
        for tbl in self._soup.find_all('table', 'forum_table'):
            user = tbl.find('td', 'forum_info').span.string
            message = tbl.find('div', 'forum_message').get_text()
            if username is ALL or user == username:
                yield dict(user=user, message=message, html=tbl.prettify())


def main(argv=None):
    parser = ArgumentParser(prog=sys.argv[0] if not argv else argv[0],
                            description=__doc__)
    gr = parser.add_mutually_exclusive_group()
    gr.add_argument('--netrc', nargs='?', metavar='PATH',
                    help='get credentials from given netrc file')
    gr.add_argument('--user', help='projecteuler username')
    parser.add_argument('--projecteuler-host', default='projecteuler.net')
    gr_a = parser.add_mutually_exclusive_group()
    gr_a.add_argument('--author',
                      help='comments author, default from credentials')
    gr_a.add_argument('--all', action='store_true', help='get all comments')
    parser.add_argument('--verbose', action='store_true')
    parser.add_argument('problem_id',
                        help='problem id(s) a single number or n-m range')
    args = parser.parse_args(sys.argv[1:] if argv is None else argv[1:])

    # get credentials
    if args.user is not None:
        user = args.user
        passwd = getpass('enter password for %s@%s: ' % (
                user, args.projecteuler_host))
    else:  # read credentials from .netrc-file
        config = netrc(args.netrc)
        creds = config.authenticators(args.projecteuler_host)
        if not creds:
            parser.error('no credentials found for %s in the %s netrc file' % (
                    args.projecteuler_host, args.netrc or 'default'))
        user, _, passwd = creds

    if args.all:
        author = ALL
    elif not args.author:
        author = user
    else:
        author = args.author

    # parse problem id range
    start_id, _, end_id = args.problem_id.partition('-')
    start_id = int(start_id)
    end_id = int(end_id) if end_id else start_id

    # save comments
    client = Client(user, passwd, args.projecteuler_host, verbose=args.verbose)
    client.login()
    for problem_id in range(start_id, end_id + 1):
        comments = []
        for x in client.get_comments(problem_id, username=author):
            comments.append(x)
            if args.verbose:
                print(x, file=sys.stderr)
        if comments:
            filename = 'comments%s-%0*d.json' % (
                '' if author is ALL else '-' + author,
                max(3, len(str(end_id))), problem_id)
            with open(filename, 'wb') as file:
                json.dump(comments, file)

if __name__ == "__main__":
    sys.exit(main())