Pipeline-II logscraper with CSV output

logscrape.py

import sys
import subprocess
import os
import re

"""
Pipeline-II Logscraper

You must run this at SLAC currently.

The Taskpath is the Pipeline-II task path.
The process is the target process in a given task which you want to
scrape files from.

Underneath, it's using the "pipeline find" command to discover working 
directories for a given process.

This script currently only scrapes logfiles of successful jobs. You can
change this by modifying the "--success" flag below.

This logscraper iterates through every line in a logfile attempting
to match the user-supplied regexes to that logfile. If a match is found,
it is put in the matched dictionary, so only the last match in a given file, 
for a given regex, will be retained.

If no match is found, at the end of the file, the CSV should show that
as an empty entry.

Each regex has a key attached to it. If multiple matching groups are supplied
for a given regex, it will output a header with the key name and the
iteration number appended to that keyname for each matching group.

"""

if len(sys.argv) < 3:
    print "Usage: python logscrape.py [taskpath] [process] [key=regexes....]"
    print "Example:"
    print """
python logscrape.py TwinklesDM/processFilter/processVisit processEimage \\
"visit=^VISIT=(\d*)" \\
"filter=^FILTER=(\w*)" \\
"wcs=.*WCS in (\d*) iterations; \
found (\d*) matches with scatter = (?P<scatter>[-\d\.]*) \+- \
(?P<deviation>[-\d\.]*) arcsec"
    """
    sys.exit(1)

task = sys.argv[1]
process = sys.argv[2]

process = subprocess.Popen(["/u/gl/srs/pipeline/prod/pipeline",
                            "find", "--success", task, process],
                            stdout=subprocess.PIPE)

out, err = process.communicate()
errorcode = process.returncode

if errorcode:
    print "Error running script: " + errorcode
    print err
    sys.exit(1)

logpaths = [os.path.join(dir, "logFile.txt") for dir in out.split("\n") if dir]

regex_entries = [(ren.split("=",1)[0], ren.split("=",1)[1])
                 for ren in sys.argv[3:]]

regexes = [(key, re.compile(val)) for key, val in regex_entries]

def columns_for(key, regex):
    gn = {v-1:k for k, v in regex.groupindex.items()}
    if regex.groups > 1:
        return [gn[i] if i in gn else key + str(i) for i in range(regex.groups)]
    return [key]

all_columns = []
for key, regex in regexes:
    all_columns.extend(columns_for(key, regex))

print ",".join(all_columns)

for logpath in logpaths:
    with open(logpath) as logfile:
        matched = {}
        tlist = []
        for line in logfile:
            for key, regex in regexes:
                if regex.match(line):
                    matched[key] = regex.match(line).groups()
        for key, regex in regexes:
            tlist.extend(matched[key] if key in matched
                         else ["" for i in range(regex.groups)])
        print ",".join(tlist)

scraped.csv