make_progress.py

#!/usr/bin/python

from __future__ import print_function

import re
import sys
import json
import collections
import os
try:
    import filetail
except ImportError:
    print("I use the failtail recipe found in https://github.com/ActiveState/code/blob/master/recipes/Python/436477_filetailpy/recipe-436477.py")
    sys.exit(1)
import argparse
import tqdm

PROGRESS_POINT_PATTERN=r'^((MAKE|PREBUILD).*):\s*(\S+) -> (\S+)'
ACE_RPM_PATTERN=r'^--> asn-anap'
POP_RPM_PATTERN=r'^mkdir -p .*pop/install'

patterns = [ PROGRESS_POINT_PATTERN, ACE_RPM_PATTERN, POP_RPM_PATTERN ]
#if elem[0] == 1, then the next tell the match.groups() to use.
#elif elem[1] == 0, then use the strings as keys
compiled_patterns_keys = [ (1,1,3,4) , (0,"RPM-Finished","RPM-Finished","anap"), (0,"RPM-Finished","RPM-Finished","pop") ]
compiled_patterns = []
for i in patterns:
    compiled_patterns.append(re.compile(i))

BRANCH_INFO_STORE=os.path.expanduser("~/ws/.make_progress_store")

def module_factory():
    return collections.defaultdict(int)
def platform_factory():
    return collections.defaultdict(module_factory)

def prepare_init_repo(filename):
    """ Given a make_op filename, it prepares a first time list of progress-points

    :filename: the make_op file to parse
    :returns: dict-of-activies -> dict-of-platforms -> dict-of-modules -> count (should be 1!), total-count """
    all_activities = collections.defaultdict(platform_factory)
    total = 0
    with open(filename,'r') as fd:
        for i in fd:
            for p,key in zip(compiled_patterns,compiled_patterns_keys):
                m = p.search(i)
                if m:
                    if key[0]:
                        activity=m.group(key[1])
                        platform=m.group(key[2])
                        module=m.group(key[3])
                    else:
                        activity=key[1]
                        platform=key[2]
                        module=key[3]
                    all_activities[activity][platform][module] += 1
                    total += 1
                    break
    return all_activities,total

def psExists(pid):
    try:
        os.kill(pid, 0)
    except OSError:
        return False
    else:
        return True


class ProgressChecker:
    def __init__(self, mp, pid, compareActivity, total):
        self.makeOp = mp
        self.make_pid = pid
        self.toCompare = compareActivity
        self.ft = filetail.Tail(self.makeOp, max_sleep=5)
        self.extra_activities = collections.defaultdict(platform_factory)
        self.totalActivities = total
        self.completed = 0

    def __iter__(self):
        return self

    def __next__(self):
        return self.getNextProgressPoint()

    def next(self):
        return self.__next__()

    def getNextLine(self):
        while psExists(self.make_pid):
            line = self.ft.nextline()
            if line:
                return line
            if not psExists(self.make_pid):
                raise StopIteration
            # continue looping . The actual sleeping happens inside the filetail library
        raise StopIteration

    def getNextProgressPoint(self):
        while self.completed < self.totalActivities:
            line = self.getNextLine()
            for p,key in zip(compiled_patterns,compiled_patterns_keys):
                m = p.search(line)
                if m:
                    if key[0]:
                        activity=m.group(key[1])
                        platform=m.group(key[2])
                        module=m.group(key[3])
                    else:
                        activity=key[1]
                        platform=key[2]
                        module=key[3]
                    if activity in self.toCompare:
                        if platform in self.toCompare[activity]:
                            if module in self.toCompare[activity][platform]:
                                self.toCompare[activity][platform][module] -= 1
                                self.completed += 1
                                return (self.completed,activity,platform,module)
                    #hmm.. extra activity
                    print("Hey extra: {0:s},{1:s},{2:s}".format(activity,platform,module))
                    self.extra_activities[activity][platform][module] += 1
                    break
            #dont bother about this line
        raise StopIteration


class UnitTest:
    def __init__(self):
        pass

    def test_prepare_init_repo(self):
        print ("This tests mere generation of init_repo - args: makefile op having progress points")
        p = prepare_init_repo(sys.argv[1])
        js = json.dumps(p,indent=4)
        print(js)

    def testFileTailer(self):
        print ("This tests file tailer library - args, some growing file and a pid to watch")
        print (" For ex, you can do bash -c 'for i in $(seq 100) ; do echo this is line $i ; sleep 1 ; done' > /tmp/a &\n And supply {0} /tmp/a <pid>".format(sys.argv[0]))
        ft = filetail.Tail(sys.argv[1])
        pid = int(sys.argv[2])
        for i in range(1,100):
            if not psExists(pid):
                print ("pid doesn't exist anymore")
                break
            else:
                line = ft.nextline()
                if line:
                    print ("iter:{0:d} got line:{1:s}".format(i,line))

#ut = UnitTest()
#ut.test_prepare_init_repo()
#ut.testFileTailer()
#sys.exit(0)


parser = argparse.ArgumentParser()
parser.add_argument("-p", "--prepare", help="prepare_init_repo", action="store_true")
parser.add_argument("make_op",         help="make-op file to follow (or prepare from)")
parser.add_argument("branch",          help="branch to compare against, (or store for)")
parser.add_argument("pid",             help="pid of makefile", nargs='?', type=int)

parsed_args = parser.parse_args()
filename=os.path.join(BRANCH_INFO_STORE,parsed_args.branch)

if parsed_args.prepare:
    all_activities,tot = prepare_init_repo(parsed_args.make_op)
    if not os.path.exists(BRANCH_INFO_STORE):
        os.makedirs(BRANCH_INFO_STORE)
    with open(filename,"w") as fd:
        a={}
        a["activities"] = all_activities
        a["total"] = tot
        json.dump(a, fd, indent=4)
    print("Wrote branch {0:s} info to {1:s}. Total of {2:d} activities".format(parsed_args.branch, filename, tot))
    sys.exit(0)

if not parsed_args.pid:
    print ("You should supply pid")
    sys.exit(1)

if not os.access(filename,os.F_OK):
    print ("Couldn't spot file {0:s}. First prepare activities list for branch:{1:s}".format(filename, parsed_args.branch))

with open(filename,"r") as fd:
    js = json.load(fd)

if "total" not in js:
    print ("Couldn't find total in file:{0:s}".format(filename))

if not psExists(parsed_args.pid):
    print ("Pid {0:d} doesn't exist".format(parsed_args.pid))
    sys.exit(1)

pc = ProgressChecker(parsed_args.make_op, parsed_args.pid, js["activities"], js["total"])
last = 0;
t = tqdm.tqdm(pc, total=js["total"])
for i in t:
    last = i[0]
    t.set_description("{0:10s}-{1:10s}-{2:20s}".format(i[1],i[2],i[3]))

if last != js["total"]:
    print ("Last:{0:d} is less than expected total:{1:d}".format(last,js["total"]))
    for act in js["activities"]:
        for plat in js["activities"][act]:
            for mod in js["activities"][act][plat]:
                if js["activities"][act][plat][mod] != 0:
                    print(" {0:s},{1:s},{2:s} is not 0, but: {3:d}".format(act,plat,mod,js["activities"][act][plat][mod]))
else:
    print ("Got {0:d} completion points".format(last))