This program downloads logs from 2pktfkt.

2pktfkt-mirror.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# 2pktfkt-mirror  Copyright (C) 2011  Nils Dagsson Moskopp (erlehmann)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.

from sys import argv, exit, stderr
from datetime import date, timedelta
from os import path

import urllib2

try:
    channel = argv[1]
    username = argv[2]
    password = argv[3]
except IndexError:
    stderr.write('This program downloads logs from 2pktfkt.\n\n\
Usage: 2pktfkt-mirror channel username password\n\n\
(channels are specified without leading “#” character.)\n\n')
    exit(1)

prefix = 'http://logs.2pktfkt.de/'

passwdmngr = urllib2.HTTPPasswordMgrWithDefaultRealm()
passwdmngr.add_password(None, prefix, username, password)
authhandler = urllib2.HTTPBasicAuthHandler(passwdmngr)
opener = urllib2.build_opener(authhandler)

urllib2.install_opener(opener)

today = date.today()
day = timedelta(days=1)

logdate = date.today()

while True:
    update_needed = False
    filename = path.join(channel, logdate.isoformat() + '.txt')

    try:
        with open(filename, 'r') as f:
            lines = f.readlines()
            if not lines[-2].startswith('--- Log closed'):
                update_needed = True
                stderr.write('Updating ')
    except IOError:
        update_needed = True
        stderr.write('Fetching ')

    url = prefix + filename

    if update_needed:
        stderr.write('<'+ url + '> …')

        try:
            req = urllib2.Request(url)
            res = urllib2.urlopen(req)

            response = res.read()

            with open(filename, 'w') as f:
                f.write(response)

            stderr.write(' done.\n')
        except urllib2.HTTPError:
            stderr.write(' aborting.\n')
            break

    else:
        stderr.write('Skipping <' + url + '>.\n')


    logdate -= day