rclonereport.py

#!/usr/bin/python

import collections
import datetime
import re
import sys
import time

import pprint

COPIED_RE = '(INFO|ERROR) +: ([^:]+): ([^:]+).*$'
ERROR_RE = '(.*): Error detected after finished.*$'

class BlankLine(Exception):
    pass


def parse_stamp(line):
    #print line.split(' ', 2)
    try:
        date, time, rest = line.split(' ', 2)
    except ValueError:
        raise BlankLine()
    try:
        stamp = datetime.datetime.strptime('%s %s' % (date, time),
                                           '%Y/%m/%d %H:%M:%S')
    except ValueError:
        raise BlankLine()
    return stamp, rest

def read_log(log):
    reasons = collections.defaultdict(list)
    for line in log:
        line = line.strip()
        try:
            stamp, content = parse_stamp(line)
        except BlankLine:
            continue
        copied = re.match(COPIED_RE, content)
        error = re.match(COPIED_RE, content)
        if copied:
            filename = copied.group(2)
            reason = copied.group(3)
            ignored = ['reading', 'excluded',
                       'modify window',
                       'replacing invalid',
                       'saved new token',
                       'waiting for checks',
                       'sizes identical',
                       'token expired',
                       'token refresh successful',
                       'resetting sleep',
                       'waiting for transfers to finish',
                       'http code',
                       'not deleting files']
            collapsed = ['Object not found',
                         'Rate limited',
                         'failed to delete',
                         'no response',
                         'EOF',
                         'connection reset']

            if any([x in reason.lower() for x in ignored]):
                continue

            for r in collapsed:
                if r in reason:
                    reason = r
                    break

            reasons[reason].append(filename)
            #print '%r copied %r because %r' % (stamp, filename, reason)
        elif error:
            filename = error.group(1)
            reasons['ERROR'].append(filename)
            print 'ERROR: %s' % filename

    return reasons

start = time.time()
results = read_log(sys.stdin)
end = time.time()

minutes = (end - start) / 60
hours = minutes / 60
minutes %= 60

print('###### Report ######')
print('Runtime: %ih%02im' % (hours, minutes))
for key in results:
    print('%7i: %s' % (len(results[key]), key))
print('####################\n\n')

results.pop('Unchanged skipping', None)

pprint.pprint(dict(results))

if results:
    # Exit nonzero if we did stuff
    sys.exit(1)
else:
    sys.exit(0)